Exemple #1
0
    def ccquery (self, msg, blob = None):
        """Sends query to CC, waits for answer."""
        assert isinstance (msg, BaseMessage)
        if not self.cc: self.connect_cc()

        cmsg = self.xtx.create_cmsg (msg, blob)
        cmsg.send_to (self.cc)

        crep = CCMessage(self.cc.recv_multipart())
        return crep.get_payload(self.xtx)
Exemple #2
0
 def on_recv(self, zmsg):
     """ Got reply from a remote CC, process it. """
     try:
         self.log.trace("%r", zmsg)
         cmsg = CCMessage(zmsg)
         req = cmsg.get_dest()
         if req == "echo.response":
             self.process_response(cmsg)
         else:
             self.log.warn("unknown msg: %s", req)
     except:
         self.log.exception("crashed, dropping msg")
Exemple #3
0
 def on_recv (self, zmsg):
     """ Got reply from a remote CC, process it. """
     try:
         self.log.trace ("%r", zmsg)
         cmsg = CCMessage (zmsg)
         req = cmsg.get_dest()
         if req == "echo.response":
             self.process_response (cmsg)
         else:
             self.log.warn ("unknown msg: %s", req)
     except:
         self.log.exception ("crashed, dropping msg")
Exemple #4
0
 def router_on_recv (self, zmsg):
     """ Got reply from worker via "router" connection """
     cmsg = CCMessage (zmsg)
     data = cmsg.get_payload (self.xtx)
     fi = (data['d_hostname'], data['d_st_dev'], data['d_st_ino'], data['d_filename'])
     fd = self.files[fi]
     if fd.waddr is None:
         fd.waddr = zmsg[0]
         fd.wname = data['worker']
     else:
         assert fd.waddr == zmsg[0] and fd.wname == data['worker']
     fd.atime = time.time()
     fd.count -= 1
     assert fd.count >= 0
Exemple #5
0
 def router_on_recv(self, zmsg):
     """ Got reply from worker via "router" connection """
     cmsg = CCMessage(zmsg)
     data = cmsg.get_payload(self.xtx)
     fi = (data['d_hostname'], data['d_st_dev'], data['d_st_ino'],
           data['d_filename'])
     fd = self.files[fi]
     if fd.waddr is None:
         fd.waddr = zmsg[0]
         fd.wname = data['worker']
     else:
         assert fd.waddr == zmsg[0] and fd.wname == data['worker']
     fd.atime = time.time()
     fd.count -= 1
     assert fd.count >= 0
Exemple #6
0
 def on_recv (self, zmsg):
     """ Got message from remote CC, process it. """
     try:
         # pongs to our pings should come back w/o any routing info
         if self.ping_remote and zmsg[0] == '':
             self.log.trace ("%r", zmsg)
             cmsg = CCMessage (zmsg)
             req = cmsg.get_dest()
             if req == "echo.response":
                 self._recv_pong (cmsg)
             else:
                 self.log.warn ("unknown msg: %s", req)
     except:
         self.log.exception ("crashed")
     finally:
         super(ProxyHandler, self).on_recv(zmsg)
Exemple #7
0
 def on_recv(self, zmsg):
     """ Got message from remote CC, process it. """
     try:
         # pongs to our pings should come back w/o any routing info
         if self.ping_remote and zmsg[0] == '':
             self.log.trace("%r", zmsg)
             cmsg = CCMessage(zmsg)
             req = cmsg.get_dest()
             if req == "echo.response":
                 self._recv_pong(cmsg)
             else:
                 self.log.warn("unknown msg: %s", req)
     except:
         self.log.exception("crashed")
     finally:
         super(ProxyHandler, self).on_recv(zmsg)
Exemple #8
0
    def handle_cc_recv(self, zmsg):
        """Got message from client, pick handler."""

        start = time.time()
        self.stat_inc('count')
        self.log.trace('got msg: %r', zmsg)
        try:
            cmsg = CCMessage(zmsg)
        except:
            self.log.exception('Invalid CC message')
            self.stat_increase('count.invalid')
            return

        try:
            dst = cmsg.get_dest()
            size = cmsg.get_size()
            route = tuple(dst.split('.'))

            # find and run all handlers that match
            cnt = 0
            for n in range(0, 1 + len(route)):
                p = route[:n]
                for h in self.routes.get(p, []):
                    self.log.trace('calling handler %s', h.hname)
                    h.handle_msg(cmsg)
                    cnt += 1
            if cnt == 0:
                self.log.warning('dropping msg, no route: %s', dst)
                stat = 'dropped'
            else:
                stat = 'ok'

        except Exception:
            self.log.exception('crashed, dropping msg: %s', dst)
            stat = 'crashed'

        # update stats
        taken = time.time() - start
        self.stat_inc('bytes', size)
        self.stat_inc('seconds', taken)
        self.stat_inc('count.%s' % stat)
        self.stat_inc('bytes.%s' % stat, size)
        self.stat_inc('seconds.%s' % stat, taken)
        if self.stat_level > 1:
            self.stat_inc('count.%s.msg.%s' % (stat, dst))
            self.stat_inc('bytes.%s.msg.%s' % (stat, dst), size)
            self.stat_inc('seconds.%s.msg.%s' % (stat, dst), taken)
Exemple #9
0
    def handle_cc_recv(self, zmsg):
        """Got message from client, pick handler."""

        start = time.time()
        self.stat_inc("count")
        self.log.trace("got msg: %r", zmsg)
        try:
            cmsg = CCMessage(zmsg)
        except:
            self.log.exception("Invalid CC message")
            self.stat_increase("count.invalid")
            return

        try:
            dst = cmsg.get_dest()
            size = cmsg.get_size()
            route = tuple(dst.split("."))

            # find and run all handlers that match
            cnt = 0
            for n in range(0, 1 + len(route)):
                p = route[:n]
                for h in self.routes.get(p, []):
                    self.log.trace("calling handler %s", h.hname)
                    h.handle_msg(cmsg)
                    cnt += 1
            if cnt == 0:
                self.log.warning("dropping msg, no route: %s", dst)
                stat = "dropped"
            else:
                stat = "ok"

        except Exception:
            self.log.exception("crashed, dropping msg: %s", dst)
            stat = "crashed"

        # update stats
        taken = time.time() - start
        self.stat_inc("bytes", size)
        self.stat_inc("seconds", taken)
        self.stat_inc("count.%s" % stat)
        self.stat_inc("bytes.%s" % stat, size)
        self.stat_inc("seconds.%s" % stat, taken)
        if self.stat_level > 1:
            self.stat_inc("count.%s.msg.%s" % (stat, dst))
            self.stat_inc("bytes.%s.msg.%s" % (stat, dst), size)
            self.stat_inc("seconds.%s.msg.%s" % (stat, dst), taken)
Exemple #10
0
    def handle_cc_recv(self, zmsg):
        """Got message from client, pick handler."""

        start = time.time()
        self.stat_inc ('count')
        self.log.trace('got msg: %r', zmsg)
        try:
            cmsg = CCMessage(zmsg)
        except:
            self.log.exception('Invalid CC message')
            self.stat_increase('count.invalid')
            return

        try:
            dst = cmsg.get_dest()
            size = cmsg.get_size()
            route = tuple(dst.split('.'))

            # find and run all handlers that match
            cnt = 0
            for n in range(0, 1 + len(route)):
                p = route[ : n]
                for h in self.routes.get(p, []):
                    self.log.trace('calling handler %s', h.hname)
                    h.handle_msg(cmsg)
                    cnt += 1
            if cnt == 0:
                self.log.warning('dropping msg, no route: %s', dst)
                stat = 'dropped'
            else:
                stat = 'ok'

        except Exception:
            self.log.exception('crashed, dropping msg: %s', dst)
            stat = 'crashed'

        # update stats
        taken = time.time() - start
        self.stat_inc ('bytes', size)
        self.stat_inc ('seconds', taken)
        self.stat_inc ('count.%s' % stat)
        self.stat_inc ('bytes.%s' % stat, size)
        self.stat_inc ('seconds.%s' % stat, taken)
        if self.stat_level > 1:
            self.stat_inc ('count.%s.msg.%s' % (stat, dst))
            self.stat_inc ('bytes.%s.msg.%s' % (stat, dst), size)
            self.stat_inc ('seconds.%s.msg.%s' % (stat, dst), taken)
Exemple #11
0
 def handle_cc_recv(self, zmsg):
     """Got task, do something with it"""
     try:
         cmsg = CCMessage(zmsg)
         self.launch_task(cmsg)
     except:
         ex = traceback.format_exc()
         self.log.error('crashed, dropping msg: %s', ex)
Exemple #12
0
    def handle_recv_real(self, zmsg):
        """Actual callback that can throw exceptions."""

        cmsg = CCMessage(zmsg)

        route = cmsg.get_route()
        if len(route) != 1:
            self.log.error('Invalid reply route: %r', route)
            return

        qid = route[0]
        if qid not in self.query_cache:
            self.log.error('reply for unknown query: %r', qid)
            return

        msg = cmsg.get_payload(self.xtx)

        qi = self.query_cache[qid]
        qi.launch_cb(msg)
Exemple #13
0
    def handle_recv_real(self, zmsg):
        """Actual callback that can throw exceptions."""

        cmsg = CCMessage(zmsg)

        route = cmsg.get_route()
        if len(route) != 1:
            self.log.error('Invalid reply route: %r', route)
            return

        qid = route[0]
        if qid not in self.query_cache:
            self.log.error('reply for unknown query: %r', qid)
            return

        msg = cmsg.get_payload(self.xtx)

        qi = self.query_cache[qid]
        qi.launch_cb(msg)
Exemple #14
0
 def work(self):
     socks = dict(self.poller.poll(1000))
     if self.master in socks and socks[self.master] == zmq.POLLIN:
         zmsg = self.master.recv_multipart()
     else:  # timeout
         return
     try:
         cmsg = CCMessage(zmsg)
     except:
         self.log.exception("invalid CC message")
     else:
         self.handle_msg(cmsg)
Exemple #15
0
 def work(self):
     socks = dict(self.poller.poll(1000))
     if self.master in socks and socks[self.master] == zmq.POLLIN:
         zmsg = self.master.recv_multipart()
     else:  # timeout
         return
     try:
         cmsg = CCMessage(zmsg)
         self.log.trace('%s', cmsg)
     except:
         self.log.exception("invalid CC message")
         return
     self.process_request(cmsg)
Exemple #16
0
    def work(self):
        socks = dict(self.poller.poll(1000))
        if self.master in socks and socks[self.master] == zmq.POLLIN:
            zmsg = self.master.recv_multipart()
        else:  # timeout
            return
        try:
            cmsg = CCMessage(zmsg)
            self.log.trace('%s', cmsg)
        except:
            self.log.exception("invalid CC message")
            return

        if not self.db:
            self.log.info('connecting to database')
            self.db = skytools.connect_database(self.connstr)
            self.db.set_isolation_level(0)

        self.process_request(cmsg)
Exemple #17
0
 def create_cmsg(self, msg, blob=None):
     if blob is not None and self.sign_name:
         msg.blob_hash = "SHA-1:" + sha1(blob).hexdigest()
     js = msg.dump_json()
     part1 = js
     part2 = ''
     if self.encrypt_name and self.sign_name:
         self.log.trace("encrypt: %s", msg['req'])
         part1 = 'ENC1'
         part2 = self.cms.sign_and_encrypt(js, self.sign_name, self.encrypt_name)
     elif self.encrypt_name:
         raise Exception('encrypt_name without sign_name ?')
     elif self.sign_name:
         self.log.trace("sign: %s", msg['req'])
         part2 = self.cms.sign(js, self.sign_name)
     else:
         self.log.trace("no crypto: %s", msg['req'])
     zmsg = ['', msg.req.encode('utf8'), part1, part2]
     if blob is not None:
         zmsg.append(blob)
     return CCMessage(zmsg)
Exemple #18
0
 def convert_cmsg(zmsg):
     cmsg = CCMessage(zmsg)
     cbfunc(cmsg)