示例#1
0
 def __init__(self,
              ctx,
              device_type=zmq.QUEUE,
              in_type=None,
              out_type=None):
     self.ctx = ctx
     # XXX: super(ThreadDevice, self).__init__(device_type, in_type, out_type)
     #      but ZMQThreadDevice is an old-style class (yay!).
     ZMQThreadDevice.__init__(self, device_type, in_type, out_type)
示例#2
0
class Heart(object):
    """A basic heart object for responding to a HeartMonitor.
    This is a simple wrapper with defaults for the most common
    Device model for responding to heartbeats.
    
    It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using 
    SUB/XREQ for in/out.
    
    You can specify the XREQ's IDENTITY via the optional heart_id argument."""
    device=None
    id=None
    def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.XREQ, heart_id=None):
        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        self.device.daemon=True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, "")
        if heart_id is None:
            heart_id = str(uuid.uuid4())
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id
    
    def start(self):
        return self.device.start()
示例#3
0
class Heart(object):
    """A basic heart object for responding to a HeartMonitor.
    This is a simple wrapper with defaults for the most common
    Device model for responding to heartbeats.
    
    It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using 
    SUB/XREQ for in/out.
    
    You can specify the XREQ's IDENTITY via the optional heart_id argument."""
    device = None
    id = None

    def __init__(self,
                 in_addr,
                 out_addr,
                 in_type=zmq.SUB,
                 out_type=zmq.XREQ,
                 heart_id=None):
        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        self.device.daemon = True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, "")
        if heart_id is None:
            heart_id = str(uuid.uuid4())
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id

    def start(self):
        return self.device.start()
示例#4
0
class Heart(object):
    """A basic heart object for responding to a HeartMonitor.
    This is a simple wrapper with defaults for the most common
    Device model for responding to heartbeats.

    It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using
    SUB/XREQ for in/out.

    You can specify the XREQ's IDENTITY via the optional heart_id argument."""
    device=None
    id=None
    def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.DEALER, heart_id=None):
        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context
        
        self.device.daemon=True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id

    def start(self):
        return self.device.start()
示例#5
0
class Heart(object):
    """A basic heart object for responding to a HeartMonitor.
    This is a simple wrapper with defaults for the most common
    Device model for responding to heartbeats.

    It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using
    SUB/XREQ for in/out.

    You can specify the XREQ's IDENTITY via the optional heart_id argument."""

    device = None
    id = None

    def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.DEALER, heart_id=None):
        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context

        self.device.daemon = True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id

    def start(self):
        return self.device.start()
示例#6
0
def setup_zmq(frontend_port, backend_port):
    device = ThreadDevice(device_type=zmq.QUEUE, in_type=zmq.ROUTER, out_type=zmq.DEALER)
    # Set high water mark to 1 to set constraint on req/rep pattern
    device.setsockopt_in(zmq.SNDHWM, 1)
    device.setsockopt_out(zmq.RCVHWM, 1)
    device.bind_in("tcp://127.0.0.1:{}".format(frontend_port))
    device.bind_out("tcp://127.0.0.1:{}".format(backend_port))
    return device
示例#7
0
class Forwarder(object):
    def __init__(self, input_addr, output_addr):
        self.device = ThreadDevice(zmq.FORWARDER, in_type=zmq.SUB, out_type=zmq.PUB)
        self.device.bind_in(input_addr)
        self.device.bind_out(output_addr)
        self.device.setsockopt_in(zmq.SUBSCRIBE, b"")

    def start(self):
        self.device.start()

    def stop(self):
        self.device._context.term()
        self.device.join()
示例#8
0
 def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.DEALER, heart_id=None):
     self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
     # do not allow the device to share global Context.instance,
     # which is the default behavior in pyzmq > 2.1.10
     self.device.context_factory = zmq.Context
     
     self.device.daemon=True
     self.device.connect_in(in_addr)
     self.device.connect_out(out_addr)
     if in_type == zmq.SUB:
         self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
     if heart_id is None:
         heart_id = uuid.uuid4().bytes
     self.device.setsockopt_out(zmq.IDENTITY, heart_id)
     self.id = heart_id
示例#9
0
def start_proxy():
    logger.info("Starting proxy")
    device = ThreadDevice(zmq.FORWARDER, zmq.XSUB, zmq.XPUB)
    device.bind_in(PROXY_BACKEND)
    device.bind_out(PROXY_FRONTEND)
    device.start()

    return device
示例#10
0
 def __init__(self,
              in_addr,
              out_addr,
              in_type=zmq.SUB,
              out_type=zmq.XREQ,
              heart_id=None):
     self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
     self.device.daemon = True
     self.device.connect_in(in_addr)
     self.device.connect_out(out_addr)
     if in_type == zmq.SUB:
         self.device.setsockopt_in(zmq.SUBSCRIBE, "")
     if heart_id is None:
         heart_id = str(uuid.uuid4())
     self.device.setsockopt_out(zmq.IDENTITY, heart_id)
     self.id = heart_id
示例#11
0
    def start(self):
        # Launch two proxy sockets, one for remote communication, and one for in-process communication.
        self.device_thread = ThreadDevice(zmq.FORWARDER, zmq.XSUB, zmq.XPUB)
        self.device_thread.bind_in("tcp://*:" + str(MessageProxy.SUB_PORT))
        self.device_thread.bind_out("tcp://*:" + str(MessageProxy.PUB_PORT))
        self.device_thread.start()

        # TODO Finish in-process communication
        """
        self.local_frontend = self.context.socket(zmq.XSUB)
        self.local_frontend.bind("inproc://" + MessageProxy.INPROC_SUB_ID)
        self.local_backend = self.context.socket(zmq.XPUB)
        self.local_backend.bind("inproc://" + MessageProxy.INPROC_PUB_ID)
        zmq.device(zmq.FORWARDER, self.local_frontend, self.local_backend)
        """

        self.initialized = True
示例#12
0
文件: broker.py 项目: shenely/PyGS
def main():
    """Main Function"""
    
    device = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
    
    device.bind_in("tcp://127.0.0.1:5555")
    device.bind_out("tcp://127.0.0.1:5556")
    device.setsockopt_in(zmq.SUBSCRIBE,"")
    
    device.start()
    
    router = ThreadRouter()
    router.bind("tcp://127.0.0.1:5560")
    
    router.start()
示例#13
0
 def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.XREQ, heart_id=None):
     self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
     self.device.daemon=True
     self.device.connect_in(in_addr)
     self.device.connect_out(out_addr)
     if in_type == zmq.SUB:
         self.device.setsockopt_in(zmq.SUBSCRIBE, "")
     if heart_id is None:
         heart_id = str(uuid.uuid4())
     self.device.setsockopt_out(zmq.IDENTITY, heart_id)
     self.id = heart_id
示例#14
0
    def __init__(
        self,
        in_addr,
        out_addr,
        mon_addr=None,
        in_type=zmq.SUB,
        out_type=zmq.DEALER,
        mon_type=zmq.PUB,
        heart_id=None,
        curve_serverkey=None,
        curve_secretkey=None,
        curve_publickey=None,
    ):
        if mon_addr is None:
            self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        else:
            self.device = ThreadMonitoredQueue(in_type,
                                               out_type,
                                               mon_type,
                                               in_prefix=b"",
                                               out_prefix=b"")
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context

        self.device.daemon = True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if curve_serverkey:
            self.device.setsockopt_in(zmq.CURVE_SERVERKEY, curve_serverkey)
            self.device.setsockopt_in(zmq.CURVE_PUBLICKEY, curve_publickey)
            self.device.setsockopt_in(zmq.CURVE_SECRETKEY, curve_secretkey)

            self.device.setsockopt_out(zmq.CURVE_SERVERKEY, curve_serverkey)
            self.device.setsockopt_out(zmq.CURVE_PUBLICKEY, curve_publickey)
            self.device.setsockopt_out(zmq.CURVE_SECRETKEY, curve_secretkey)
            if mon_addr is not None:
                self.device.setsockopt_mon(zmq.CURVE_SERVERKEY,
                                           curve_publickey)
                self.device.setsockopt_mon(zmq.CURVE_PUBLICKEY,
                                           curve_publickey)
                self.device.setsockopt_mon(zmq.CURVE_SECRETKEY,
                                           curve_secretkey)

        if mon_addr is not None:
            self.device.connect_mon(mon_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id
示例#15
0
    def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.DEALER, heart_id=None):
        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context

        self.device.daemon = True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id
示例#16
0
    def start(self):
        # Launch two proxy sockets, one for remote communication, and one for in-process communication.
        self.device_thread = ThreadDevice(zmq.FORWARDER, zmq.XSUB, zmq.XPUB)
        self.device_thread.bind_in("tcp://*:" + str(MessageProxy.SUB_PORT))
        self.device_thread.bind_out("tcp://*:" + str(MessageProxy.PUB_PORT))
        self.device_thread.start()

        # TODO Finish in-process communication
        """
        self.local_frontend = self.context.socket(zmq.XSUB)
        self.local_frontend.bind("inproc://" + MessageProxy.INPROC_SUB_ID)
        self.local_backend = self.context.socket(zmq.XPUB)
        self.local_backend.bind("inproc://" + MessageProxy.INPROC_PUB_ID)
        zmq.device(zmq.FORWARDER, self.local_frontend, self.local_backend)
        """

        self.initialized = True
示例#17
0
    def start(self):
        # open log files
        now = datetime.datetime.utcnow()
        self.logDir = os.path.abspath(self.opts.logDir)
        self.messageLogPath = self.readyLog(self.opts.messageLog, now)
        self.messageLog = open(self.messageLogPath, 'a')
        self.consoleLogPath = self.readyLog(self.opts.consoleLog, now)

        rootLogger = logging.getLogger()
        rootLogger.setLevel(logging.DEBUG)
        fmt = logging.Formatter('%(asctime)s - %(levelname)-7s - %(message)s')
        fmt.converter = time.gmtime
        fh = logging.FileHandler(self.consoleLogPath)
        fh.setFormatter(fmt)
        fh.setLevel(logging.DEBUG)
        rootLogger.addHandler(fh)
        if self.opts.foreground:
            ch = logging.StreamHandler()
            ch.setLevel(logging.DEBUG)
            ch.setFormatter(fmt)
            rootLogger.addHandler(ch)

        # daemonize
        if self.opts.foreground:
            logging.info('staying in foreground')
        else:
            logging.info('daemonizing')
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.setsid()
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.chdir('/')
            os.close(1)
            os.close(2)
            nullFd = os.open('/dev/null', os.O_RDWR)
            os.dup2(nullFd, 1)
            os.dup2(nullFd, 2)

        try:
            # set up zmq
            self.context = zmq.Context.instance()
            self.rpcStream = ZMQStream(self.context.socket(zmq.REP))
            self.rpcStream.bind(self.opts.rpcEndpoint)
            self.rpcStream.on_recv(self.handleRpcCall)

            self.forwarder = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
            self.forwarder.setsockopt_in(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_out(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_in(zmq.SUBSCRIBE, '')
            self.forwarder.setsockopt_out(zmq.HWM, self.opts.highWaterMark)
            self.forwarder.bind_in(self.opts.subscribeEndpoint)
            self.forwarder.bind_in(INJECT_ENDPOINT)
            self.forwarder.bind_out(self.opts.publishEndpoint)
            self.forwarder.bind_out(MONITOR_ENDPOINT)
            for entry in self.opts.subscribeTo:
                try:
                    moduleName, endpoint = entry.split('@')
                    endpoint = parseEndpoint(endpoint)
                except ValueError:
                    raise ValueError(
                        '--subscribeTo argument "%s" is not in the format "<moduleName>@<endpoint>"'
                        % entry)
                self.forwarder.connect_in(endpoint)
                self.info[moduleName] = {'module': moduleName, 'pub': endpoint}
            self.forwarder.start()
            time.sleep(0.1)  # wait for forwarder to bind sockets

            self.monStream = ZMQStream(self.context.socket(zmq.SUB))
            self.monStream.setsockopt(zmq.SUBSCRIBE, '')
            self.monStream.connect(MONITOR_ENDPOINT)
            self.monStream.on_recv(self.handleMessages)

            self.injectStream = ZMQStream(self.context.socket(zmq.PUB))
            self.injectStream.connect(INJECT_ENDPOINT)

            self.disconnectTimer = ioloop.PeriodicCallback(
                self.handleDisconnectTimer, 5000)
            self.disconnectTimer.start()

        except:  # pylint: disable=W0702
            errClass, errObject, errTB = sys.exc_info()[:3]
            errText = '%s.%s: %s' % (errClass.__module__, errClass.__name__,
                                     str(errObject))
            logging.error(''.join(traceback.format_tb(errTB)))
            logging.error(errText)
            logging.error('[error during startup -- exiting]')
            sys.exit(1)
示例#18
0
    def run(self, *args, **kwargs):

        frontend_address = kwargs['frontend-address']
        assert frontend_address
        backend_address = kwargs['backend-address']
        assert backend_address

        device = ThreadDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
        device.bind_in(frontend_address)
        device.setsockopt_in(zmq.IDENTITY, 'ROUTER')
        device.bind_out(backend_address)
        device.setsockopt_out(zmq.IDENTITY, 'DEALER')
        device.start()
示例#19
0
def run_proxy(context):   
    proxy=ThreadDevice(zmq.QUEUE, zmq.XSUB,zmq.XPUB )
    proxy.bind_in(ADDR_SUB)
    #proxy.setsockopt_in(zmq.SUBSCRIBE, '')
    proxy.bind_out(ADDR_PUB)
    proxy.start() 
示例#20
0
    def start(self):
        # open log files
        now = datetime.datetime.utcnow()
        self.logDir = os.path.abspath(self.opts.logDir)
        self.messageLogPath = self.readyLog(self.opts.messageLog, now)
        self.messageLog = open(self.messageLogPath, 'a')
        self.consoleLogPath = self.readyLog(self.opts.consoleLog, now)

        rootLogger = logging.getLogger()
        rootLogger.setLevel(logging.DEBUG)
        fmt = logging.Formatter('%(asctime)s - %(levelname)-7s - %(message)s')
        fmt.converter = time.gmtime
        fh = logging.FileHandler(self.consoleLogPath)
        fh.setFormatter(fmt)
        fh.setLevel(logging.DEBUG)
        rootLogger.addHandler(fh)
        if self.opts.foreground:
            ch = logging.StreamHandler()
            ch.setLevel(logging.DEBUG)
            ch.setFormatter(fmt)
            rootLogger.addHandler(ch)

        # daemonize
        if self.opts.foreground:
            logging.info('staying in foreground')
        else:
            logging.info('daemonizing')
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.setsid()
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.chdir('/')
            os.close(1)
            os.close(2)
            nullFd = os.open('/dev/null', os.O_RDWR)
            os.dup2(nullFd, 1)
            os.dup2(nullFd, 2)

        try:
            # set up zmq
            self.context = zmq.Context.instance()
            self.rpcStream = ZMQStream(self.context.socket(zmq.REP))
            self.rpcStream.bind(self.opts.rpcEndpoint)
            self.rpcStream.on_recv(self.handleRpcCall)

            self.forwarder = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
            self.forwarder.setsockopt_in(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_out(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_in(zmq.SUBSCRIBE, '')
            self.forwarder.setsockopt_out(zmq.HWM, self.opts.highWaterMark)
            self.forwarder.bind_in(self.opts.subscribeEndpoint)
            self.forwarder.bind_in(INJECT_ENDPOINT)
            self.forwarder.bind_out(self.opts.publishEndpoint)
            self.forwarder.bind_out(MONITOR_ENDPOINT)
            for entry in self.opts.subscribeTo:
                try:
                    moduleName, endpoint = entry.split('@')
                    endpoint = parseEndpoint(endpoint)
                except ValueError:
                    raise ValueError('--subscribeTo argument "%s" is not in the format "<moduleName>@<endpoint>"' % entry)
                self.forwarder.connect_in(endpoint)
                self.info[moduleName] = {'module': moduleName,
                                         'pub': endpoint}
            self.forwarder.start()
            time.sleep(0.1)  # wait for forwarder to bind sockets

            self.monStream = ZMQStream(self.context.socket(zmq.SUB))
            self.monStream.setsockopt(zmq.SUBSCRIBE, '')
            self.monStream.connect(MONITOR_ENDPOINT)
            self.monStream.on_recv(self.handleMessages)

            self.injectStream = ZMQStream(self.context.socket(zmq.PUB))
            self.injectStream.connect(INJECT_ENDPOINT)

            self.disconnectTimer = ioloop.PeriodicCallback(self.handleDisconnectTimer, 5000)
            self.disconnectTimer.start()

        except:  # pylint: disable=W0702
            errClass, errObject, errTB = sys.exc_info()[:3]
            errText = '%s.%s: %s' % (errClass.__module__,
                                     errClass.__name__,
                                     str(errObject))
            logging.error(''.join(traceback.format_tb(errTB)))
            logging.error(errText)
            logging.error('[error during startup -- exiting]')
            sys.exit(1)
示例#21
0
class ZmqCentral(object):
    def __init__(self, opts):
        self.opts = opts
        self.info = {}

    def announceConnect(self, moduleName, params):
        logging.info('module %s connected', moduleName)
        self.injectStream.send('central.connect.%s:%s'
                               % (moduleName, json.dumps(params)))

    def announceDisconnect(self, moduleName):
        logging.info('module %s disconnected', moduleName)
        self.injectStream.send('central.disconnect.%s:%s'
                               % (moduleName,
                                  json.dumps({'timestamp': str(getTimestamp())})))

    def logMessage(self, msg, posixTime=None, attachmentDir='-'):
        mlog = self.messageLog
        mlog.write('@@@ %d %d %s ' % (getTimestamp(posixTime), len(msg), attachmentDir))
        mlog.write(msg)
        mlog.write('\n')

    def logMessageWithAttachments0(self, msg):
        parsed = parseMessage(msg)
        posixTime = time.time()

        # construct attachment directory
        dt = datetime.datetime.utcfromtimestamp(posixTime)
        dateText = dt.strftime('%Y-%m-%d')
        timeText = dt.strftime('%H-%M-%S') + '.%06d' % dt.microsecond
        uniq = '%08x' % random.getrandbits(32)
        attachmentSuffix = os.path.join('attachments',
                                        dateText,
                                        timeText,
                                        parsed['topic'],
                                        uniq)
        attachmentPath = os.path.join(self.logDir, attachmentSuffix)
        os.makedirs(attachmentPath)

        # write attachments to attachment directory
        for attachment in parsed['attachments']:
            fullName = os.path.join(attachmentPath, attachment.get_filename())
            open(fullName, 'wb').write(attachment.get_payload())

        # log message with a pointer to the attachment directory
        self.logMessage(':'.join((parsed['topic'], parsed['json'])),
                        posixTime,
                        attachmentSuffix)

    def logMessageWithAttachments(self, msg):
        try:
            return self.logMessageWithAttachments0(msg)
        except:  # pylint: disable=W0702
            self.logException('logging message with attachments')

    def handleHeartbeat(self, params):
        moduleName = params['module'].encode('utf-8')
        now = getTimestamp()

        oldInfo = self.info.get(moduleName, None)
        if oldInfo:
            if oldInfo.get('pub', None) != params.get('pub', None):
                self.announceDisconnect(moduleName)
                self.announceConnect(moduleName, params)
        else:
            self.announceConnect(moduleName, params)

        self.info[moduleName] = params
        keepalive = params.get('keepalive', DEFAULT_KEEPALIVE_US)
        params['timeout'] = now + keepalive
        return 'ok'

    def handleInfo(self):
        return self.info

    def logException(self, whileClause):
        errClass, errObject, errTB = sys.exc_info()[:3]
        errText = '%s.%s: %s' % (errClass.__module__,
                                 errClass.__name__,
                                 str(errObject))
        logging.warning(''.join(traceback.format_tb(errTB)))
        logging.warning(errText)
        logging.warning('[error while %s at time %s]', whileClause, getTimestamp())

    def handleMessages(self, messages):
        for msg in messages:
            if hasAttachments(msg):
                self.logMessageWithAttachments(msg)
            else:
                self.logMessage(msg)
            if msg.startswith('central.heartbeat.'):
                try:
                    _topic, body = msg.split(':', 1)
                    self.handleHeartbeat(json.loads(body))
                except:  # pylint: disable=W0702
                    self.logException('handling heartbeat')

    def handleRpcCall(self, messages):
        for msg in messages:
            try:
                call = json.loads(msg)
                callId = call['id']
            except:  # pylint: disable=W0702
                self.rpcStream.send(json.dumps({'result': None,
                                                'error': 'malformed request'}))

            try:
                method = call['method']
                _params = call['params']
                if method == 'info':
                    result = self.handleInfo()
                else:
                    raise ValueError('unknown method %s' % method)
                self.rpcStream.send(json.dumps({'result': result,
                                                'error': None,
                                                'id': callId}))
            except:  # pylint: disable=W0702
                self.logException('handling rpc message')
                errClass, errObject = sys.exc_info()[:2]
                errText = '%s.%s: %s' % (errClass.__module__,
                                         errClass.__name__,
                                         str(errObject))
                self.rpcStream.send(json.dumps({'result': None,
                                                'error': errText,
                                                'id': callId}))

    def handleDisconnectTimer(self):
        now = getTimestamp()
        disconnectModules = []
        for moduleName, entry in self.info.iteritems():
            timeout = entry.get('timeout', None)
            if timeout is not None and now > timeout:
                disconnectModules.append(moduleName)
        for moduleName in disconnectModules:
            self.announceDisconnect(moduleName)
            del self.info[moduleName]

    def readyLog(self, pathTemplate, timestamp):
        if '%s' in pathTemplate:
            timeText = timestamp.strftime('%Y-%m-%d-%H-%M-%S')
            logFile = pathTemplate % timeText
        else:
            logFile = pathTemplate
        if not os.path.exists(self.logDir):
            os.makedirs(self.logDir)
        logPath = os.path.join(self.logDir, logFile)
        if '%s' in pathTemplate:
            latestPath = os.path.join(self.logDir, pathTemplate % 'latest')
            if os.path.islink(latestPath):
                os.unlink(latestPath)
        os.symlink(logFile, latestPath)
        return logPath

    def start(self):
        # open log files
        now = datetime.datetime.utcnow()
        self.logDir = os.path.abspath(self.opts.logDir)
        self.messageLogPath = self.readyLog(self.opts.messageLog, now)
        self.messageLog = open(self.messageLogPath, 'a')
        self.consoleLogPath = self.readyLog(self.opts.consoleLog, now)

        rootLogger = logging.getLogger()
        rootLogger.setLevel(logging.DEBUG)
        fmt = logging.Formatter('%(asctime)s - %(levelname)-7s - %(message)s')
        fmt.converter = time.gmtime
        fh = logging.FileHandler(self.consoleLogPath)
        fh.setFormatter(fmt)
        fh.setLevel(logging.DEBUG)
        rootLogger.addHandler(fh)
        if self.opts.foreground:
            ch = logging.StreamHandler()
            ch.setLevel(logging.DEBUG)
            ch.setFormatter(fmt)
            rootLogger.addHandler(ch)

        # daemonize
        if self.opts.foreground:
            logging.info('staying in foreground')
        else:
            logging.info('daemonizing')
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.setsid()
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.chdir('/')
            os.close(1)
            os.close(2)
            nullFd = os.open('/dev/null', os.O_RDWR)
            os.dup2(nullFd, 1)
            os.dup2(nullFd, 2)

        try:
            # set up zmq
            self.context = zmq.Context.instance()
            self.rpcStream = ZMQStream(self.context.socket(zmq.REP))
            self.rpcStream.bind(self.opts.rpcEndpoint)
            self.rpcStream.on_recv(self.handleRpcCall)

            self.forwarder = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
            self.forwarder.setsockopt_in(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_out(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_in(zmq.SUBSCRIBE, '')
            self.forwarder.setsockopt_out(zmq.HWM, self.opts.highWaterMark)
            self.forwarder.bind_in(self.opts.subscribeEndpoint)
            self.forwarder.bind_in(INJECT_ENDPOINT)
            self.forwarder.bind_out(self.opts.publishEndpoint)
            self.forwarder.bind_out(MONITOR_ENDPOINT)
            for entry in self.opts.subscribeTo:
                try:
                    moduleName, endpoint = entry.split('@')
                    endpoint = parseEndpoint(endpoint)
                except ValueError:
                    raise ValueError('--subscribeTo argument "%s" is not in the format "<moduleName>@<endpoint>"' % entry)
                self.forwarder.connect_in(endpoint)
                self.info[moduleName] = {'module': moduleName,
                                         'pub': endpoint}
            self.forwarder.start()
            time.sleep(0.1)  # wait for forwarder to bind sockets

            self.monStream = ZMQStream(self.context.socket(zmq.SUB))
            self.monStream.setsockopt(zmq.SUBSCRIBE, '')
            self.monStream.connect(MONITOR_ENDPOINT)
            self.monStream.on_recv(self.handleMessages)

            self.injectStream = ZMQStream(self.context.socket(zmq.PUB))
            self.injectStream.connect(INJECT_ENDPOINT)

            self.disconnectTimer = ioloop.PeriodicCallback(self.handleDisconnectTimer, 5000)
            self.disconnectTimer.start()

        except:  # pylint: disable=W0702
            errClass, errObject, errTB = sys.exc_info()[:3]
            errText = '%s.%s: %s' % (errClass.__module__,
                                     errClass.__name__,
                                     str(errObject))
            logging.error(''.join(traceback.format_tb(errTB)))
            logging.error(errText)
            logging.error('[error during startup -- exiting]')
            sys.exit(1)

    def shutdown(self):
        self.messageLog.flush()
示例#22
0
    def run(self, *args, **kwargs):

        frontend_address = kwargs["frontend-address"]
        assert frontend_address
        backend_address = kwargs["backend-address"]
        assert backend_address

        device = ThreadDevice(zmq.QUEUE, zmq.ROUTER, zmq.DEALER)
        device.bind_in(frontend_address)
        device.setsockopt_in(zmq.IDENTITY, "ROUTER")
        device.bind_out(backend_address)
        device.setsockopt_out(zmq.IDENTITY, "DEALER")
        device.start()
示例#23
0
class MessageProxy(object):
    """
    The MessageProxy wraps a ZeroMQ proxy server (XSUB socket) which forwards 'publish' messages from Python Analysis
    objects to subscribers (Scala Analysis objects) in the streamer process.
    """

    DEFAULT_NUM_THREADS = 1

    # In-process communication settings
    INPROC_PUB_ID = "msg_proxy_pub"
    INPROC_SUB_ID = "msg_proxy_sub"

    # Remote communication settings
    PUB_PORT = settings.PUB_PORT
    SUB_PORT = settings.SUB_PORT

    # TODO: This will only work if the Python front-end and the streamer process are running on the same machine
    DEFAULT_BIND_HOST = "localhost"

    def __init__(self,
                 host=DEFAULT_BIND_HOST,
                 num_threads=DEFAULT_NUM_THREADS):
        self.initialized = False
        self.context = zmq.Context(num_threads)
        self.host = host

        self.device_thread = None

    def start(self):
        # Launch two proxy sockets, one for remote communication, and one for in-process communication.
        self.device_thread = ThreadDevice(zmq.FORWARDER, zmq.XSUB, zmq.XPUB)
        self.device_thread.bind_in("tcp://*:" + str(MessageProxy.SUB_PORT))
        self.device_thread.bind_out("tcp://*:" + str(MessageProxy.PUB_PORT))
        self.device_thread.start()

        # TODO Finish in-process communication
        """
        self.local_frontend = self.context.socket(zmq.XSUB)
        self.local_frontend.bind("inproc://" + MessageProxy.INPROC_SUB_ID)
        self.local_backend = self.context.socket(zmq.XPUB)
        self.local_backend.bind("inproc://" + MessageProxy.INPROC_PUB_ID)
        zmq.device(zmq.FORWARDER, self.local_frontend, self.local_backend)
        """

        self.initialized = True

    def _get_pub_addr(self, remote=True):
        if remote:
            return "tcp://" + self.host + ":" + str(MessageProxy.SUB_PORT)
        else:
            return "inproc://" + MessageProxy.INPROC_SUB_ID

    def _get_sub_addr(self, remote=True):
        if remote:
            return "tcp://" + self.host + ":" + str(MessageProxy.PUB_PORT)
        else:
            return "inproc://" + MessageProxy.INPROC_PUB_ID

    def get_publisher(self, remote=True):
        """
        :param tag: The tag the client will use to publish
        :param remote: True if the client wishes to publish to remote services, False otherwise.
        :return: A Publisher object constructed using this MessageProxy's ZMQ Context
        """
        return Publisher.get_publisher(self.context,
                                       self._get_pub_addr(remote))

    def get_subscriber(self, tag=None, remote=True):
        """
        :param tag: The tag the client will use to subscribe
        :param remote: True if the client wishes to subscribe to remote services, False otherwise.
        :return: A Subscriber object constructed using this MessageProxy's ZMQ Context
        """
        return Subscriber.get_subscriber(self.context,
                                         self._get_sub_addr(remote), tag)
示例#24
0
 def __init__(self, input_addr, output_addr):
     self.device = ThreadDevice(zmq.FORWARDER, in_type=zmq.SUB, out_type=zmq.PUB)
     self.device.bind_in(input_addr)
     self.device.bind_out(output_addr)
     self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
示例#25
0
 def __init__(self, ctx, device_type=zmq.QUEUE, in_type=None, out_type=None):
     self.ctx = ctx
     # XXX: super(ThreadDevice, self).__init__(device_type, in_type, out_type)
     #      but ZMQThreadDevice is an old-style class (yay!).
     ZMQThreadDevice.__init__(self, device_type, in_type, out_type)
示例#26
0
 def __init__(self, ctx, device_type=zmq.QUEUE, in_type=None, out_type=None):
     self.ctx = ctx
     ZMQThreadDevice.__init__(self, device_type, in_type, out_type)
示例#27
0
class MessageProxy(object):
    """
    The MessageProxy wraps a ZeroMQ proxy server (XSUB socket) which forwards 'publish' messages from Python Analysis
    objects to subscribers (Scala Analysis objects) in the streamer process.
    """

    DEFAULT_NUM_THREADS = 1

    # In-process communication settings
    INPROC_PUB_ID = "msg_proxy_pub"
    INPROC_SUB_ID = "msg_proxy_sub"

    # Remote communication settings
    PUB_PORT = settings.PUB_PORT
    SUB_PORT = settings.SUB_PORT

    # TODO: This will only work if the Python front-end and the streamer process are running on the same machine
    DEFAULT_BIND_HOST = "localhost"

    def __init__(self, host=DEFAULT_BIND_HOST, num_threads=DEFAULT_NUM_THREADS):
        self.initialized = False
        self.context = zmq.Context(num_threads)
        self.host = host

        self.device_thread = None

    def start(self):
        # Launch two proxy sockets, one for remote communication, and one for in-process communication.
        self.device_thread = ThreadDevice(zmq.FORWARDER, zmq.XSUB, zmq.XPUB)
        self.device_thread.bind_in("tcp://*:" + str(MessageProxy.SUB_PORT))
        self.device_thread.bind_out("tcp://*:" + str(MessageProxy.PUB_PORT))
        self.device_thread.start()

        # TODO Finish in-process communication
        """
        self.local_frontend = self.context.socket(zmq.XSUB)
        self.local_frontend.bind("inproc://" + MessageProxy.INPROC_SUB_ID)
        self.local_backend = self.context.socket(zmq.XPUB)
        self.local_backend.bind("inproc://" + MessageProxy.INPROC_PUB_ID)
        zmq.device(zmq.FORWARDER, self.local_frontend, self.local_backend)
        """

        self.initialized = True

    def _get_pub_addr(self, remote=True):
        if remote:
            return "tcp://" + self.host + ":" + str(MessageProxy.SUB_PORT)
        else:
            return "inproc://" + MessageProxy.INPROC_SUB_ID

    def _get_sub_addr(self, remote=True):
        if remote:
            return "tcp://" + self.host + ":" + str(MessageProxy.PUB_PORT)
        else:
            return "inproc://" + MessageProxy.INPROC_PUB_ID

    def get_publisher(self, remote=True):
        """
        :param tag: The tag the client will use to publish
        :param remote: True if the client wishes to publish to remote services, False otherwise.
        :return: A Publisher object constructed using this MessageProxy's ZMQ Context
        """
        return Publisher.get_publisher(self.context, self._get_pub_addr(remote))

    def get_subscriber(self, tag=None, remote=True):
        """
        :param tag: The tag the client will use to subscribe
        :param remote: True if the client wishes to subscribe to remote services, False otherwise.
        :return: A Subscriber object constructed using this MessageProxy's ZMQ Context
        """
        return Subscriber.get_subscriber(self.context, self._get_sub_addr(remote), tag)
示例#28
0
class ZmqCentral(object):
    def __init__(self, opts):
        self.opts = opts
        self.info = {}

    def announceConnect(self, moduleName, params):
        logging.info('module %s connected', moduleName)
        self.injectStream.send('central.connect.%s:%s' %
                               (moduleName, json.dumps(params)))

    def announceDisconnect(self, moduleName):
        logging.info('module %s disconnected', moduleName)
        self.injectStream.send(
            'central.disconnect.%s:%s' %
            (moduleName, json.dumps({'timestamp': str(getTimestamp())})))

    def logMessage(self, msg, posixTime=None, attachmentDir='-'):
        mlog = self.messageLog
        mlog.write('@@@ %d %d %s ' %
                   (getTimestamp(posixTime), len(msg), attachmentDir))
        mlog.write(msg)
        mlog.write('\n')

    def logMessageWithAttachments0(self, msg):
        parsed = parseMessage(msg)
        posixTime = time.time()

        # construct attachment directory
        dt = datetime.datetime.utcfromtimestamp(posixTime)
        dateText = dt.strftime('%Y-%m-%d')
        timeText = dt.strftime('%H-%M-%S') + '.%06d' % dt.microsecond
        uniq = '%08x' % random.getrandbits(32)
        attachmentSuffix = os.path.join('attachments', dateText, timeText,
                                        parsed['topic'], uniq)
        attachmentPath = os.path.join(self.logDir, attachmentSuffix)
        os.makedirs(attachmentPath)

        # write attachments to attachment directory
        for attachment in parsed['attachments']:
            fullName = os.path.join(attachmentPath, attachment.get_filename())
            open(fullName, 'wb').write(attachment.get_payload())

        # log message with a pointer to the attachment directory
        self.logMessage(':'.join((parsed['topic'], parsed['json'])), posixTime,
                        attachmentSuffix)

    def logMessageWithAttachments(self, msg):
        try:
            return self.logMessageWithAttachments0(msg)
        except:  # pylint: disable=W0702
            self.logException('logging message with attachments')

    def handleHeartbeat(self, params):
        moduleName = params['module'].encode('utf-8')
        now = getTimestamp()

        oldInfo = self.info.get(moduleName, None)
        if oldInfo:
            if oldInfo.get('pub', None) != params.get('pub', None):
                self.announceDisconnect(moduleName)
                self.announceConnect(moduleName, params)
        else:
            self.announceConnect(moduleName, params)

        self.info[moduleName] = params
        keepalive = params.get('keepalive', DEFAULT_KEEPALIVE_US)
        params['timeout'] = now + keepalive
        return 'ok'

    def handleInfo(self):
        return self.info

    def logException(self, whileClause):
        errClass, errObject, errTB = sys.exc_info()[:3]
        errText = '%s.%s: %s' % (errClass.__module__, errClass.__name__,
                                 str(errObject))
        logging.warning(''.join(traceback.format_tb(errTB)))
        logging.warning(errText)
        logging.warning('[error while %s at time %s]', whileClause,
                        getTimestamp())

    def handleMessages(self, messages):
        for msg in messages:
            if hasAttachments(msg):
                self.logMessageWithAttachments(msg)
            else:
                self.logMessage(msg)
            if msg.startswith('central.heartbeat.'):
                try:
                    _topic, body = msg.split(':', 1)
                    self.handleHeartbeat(json.loads(body))
                except:  # pylint: disable=W0702
                    self.logException('handling heartbeat')

    def handleRpcCall(self, messages):
        for msg in messages:
            try:
                call = json.loads(msg)
                callId = call['id']
            except:  # pylint: disable=W0702
                self.rpcStream.send(
                    json.dumps({
                        'result': None,
                        'error': 'malformed request'
                    }))

            try:
                method = call['method']
                _params = call['params']
                if method == 'info':
                    result = self.handleInfo()
                else:
                    raise ValueError('unknown method %s' % method)
                self.rpcStream.send(
                    json.dumps({
                        'result': result,
                        'error': None,
                        'id': callId
                    }))
            except:  # pylint: disable=W0702
                self.logException('handling rpc message')
                errClass, errObject = sys.exc_info()[:2]
                errText = '%s.%s: %s' % (errClass.__module__,
                                         errClass.__name__, str(errObject))
                self.rpcStream.send(
                    json.dumps({
                        'result': None,
                        'error': errText,
                        'id': callId
                    }))

    def handleDisconnectTimer(self):
        now = getTimestamp()
        disconnectModules = []
        for moduleName, entry in self.info.iteritems():
            timeout = entry.get('timeout', None)
            if timeout is not None and now > timeout:
                disconnectModules.append(moduleName)
        for moduleName in disconnectModules:
            self.announceDisconnect(moduleName)
            del self.info[moduleName]

    def readyLog(self, pathTemplate, timestamp):
        if '%s' in pathTemplate:
            timeText = timestamp.strftime('%Y-%m-%d-%H-%M-%S')
            logFile = pathTemplate % timeText
        else:
            logFile = pathTemplate
        if not os.path.exists(self.logDir):
            os.makedirs(self.logDir)
        logPath = os.path.join(self.logDir, logFile)
        if '%s' in pathTemplate:
            latestPath = os.path.join(self.logDir, pathTemplate % 'latest')
            if os.path.islink(latestPath):
                os.unlink(latestPath)
        os.symlink(logFile, latestPath)
        return logPath

    def start(self):
        # open log files
        now = datetime.datetime.utcnow()
        self.logDir = os.path.abspath(self.opts.logDir)
        self.messageLogPath = self.readyLog(self.opts.messageLog, now)
        self.messageLog = open(self.messageLogPath, 'a')
        self.consoleLogPath = self.readyLog(self.opts.consoleLog, now)

        rootLogger = logging.getLogger()
        rootLogger.setLevel(logging.DEBUG)
        fmt = logging.Formatter('%(asctime)s - %(levelname)-7s - %(message)s')
        fmt.converter = time.gmtime
        fh = logging.FileHandler(self.consoleLogPath)
        fh.setFormatter(fmt)
        fh.setLevel(logging.DEBUG)
        rootLogger.addHandler(fh)
        if self.opts.foreground:
            ch = logging.StreamHandler()
            ch.setLevel(logging.DEBUG)
            ch.setFormatter(fmt)
            rootLogger.addHandler(ch)

        # daemonize
        if self.opts.foreground:
            logging.info('staying in foreground')
        else:
            logging.info('daemonizing')
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.setsid()
            pid = os.fork()
            if pid != 0:
                os._exit(0)
            os.chdir('/')
            os.close(1)
            os.close(2)
            nullFd = os.open('/dev/null', os.O_RDWR)
            os.dup2(nullFd, 1)
            os.dup2(nullFd, 2)

        try:
            # set up zmq
            self.context = zmq.Context.instance()
            self.rpcStream = ZMQStream(self.context.socket(zmq.REP))
            self.rpcStream.bind(self.opts.rpcEndpoint)
            self.rpcStream.on_recv(self.handleRpcCall)

            self.forwarder = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
            self.forwarder.setsockopt_in(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_out(zmq.IDENTITY, THIS_MODULE)
            self.forwarder.setsockopt_in(zmq.SUBSCRIBE, '')
            self.forwarder.setsockopt_out(zmq.HWM, self.opts.highWaterMark)
            self.forwarder.bind_in(self.opts.subscribeEndpoint)
            self.forwarder.bind_in(INJECT_ENDPOINT)
            self.forwarder.bind_out(self.opts.publishEndpoint)
            self.forwarder.bind_out(MONITOR_ENDPOINT)
            for entry in self.opts.subscribeTo:
                try:
                    moduleName, endpoint = entry.split('@')
                    endpoint = parseEndpoint(endpoint)
                except ValueError:
                    raise ValueError(
                        '--subscribeTo argument "%s" is not in the format "<moduleName>@<endpoint>"'
                        % entry)
                self.forwarder.connect_in(endpoint)
                self.info[moduleName] = {'module': moduleName, 'pub': endpoint}
            self.forwarder.start()
            time.sleep(0.1)  # wait for forwarder to bind sockets

            self.monStream = ZMQStream(self.context.socket(zmq.SUB))
            self.monStream.setsockopt(zmq.SUBSCRIBE, '')
            self.monStream.connect(MONITOR_ENDPOINT)
            self.monStream.on_recv(self.handleMessages)

            self.injectStream = ZMQStream(self.context.socket(zmq.PUB))
            self.injectStream.connect(INJECT_ENDPOINT)

            self.disconnectTimer = ioloop.PeriodicCallback(
                self.handleDisconnectTimer, 5000)
            self.disconnectTimer.start()

        except:  # pylint: disable=W0702
            errClass, errObject, errTB = sys.exc_info()[:3]
            errText = '%s.%s: %s' % (errClass.__module__, errClass.__name__,
                                     str(errObject))
            logging.error(''.join(traceback.format_tb(errTB)))
            logging.error(errText)
            logging.error('[error during startup -- exiting]')
            sys.exit(1)

    def shutdown(self):
        self.messageLog.flush()