Пример #1
0
    def __init__(self, controller_identity=None, logger=None, *args, **kwargs):
        RegistrationService.__init__(self, *args, **kwargs)

        self.controller_identity = controller_identity or kwargs.get(
            'controller_identity', None)

        self.subscriber_socket = self.context.socket(zmq.SUB)
        self.subscriber_socket.setsockopt(zmq.SUBSCRIBE,
                                          self.manager_subscriber_scope)
        self.subscriber_socket.connect("tcp://localhost:{0}".format(
            self.registration_publisher_port))

        self.verified_brokers = {}
        self.unverified_brokers = {}
        self.inbound_sockets = []

        self.logger = logger

        self.inbound_poller = zmq.Poller()

        self.heartbeat_manager = HeartbeatManager(*args, **kwargs)
        self.verification_manager = HeartbeatManager(*args, **kwargs)

        self.poller = zmq.Poller()
        self.poller.register(self.subscriber_socket, zmq.POLLIN)

        gevent.spawn(self.listen_for_updates)
Пример #2
0
    def client(recv_port, graph_id):
        context = zmq.Context()
        socket_pull = context.socket(zmq.PULL)
        socket_pull.connect("tcp://localhost:%s" % recv_port)
        print("Connected to server with port %s" % recv_port)
        # Initialize poll set
        poller = zmq.Poller()
        poller.register(socket_pull, zmq.POLLIN)

        # even though InMemoryGraphDispatcher can handle multiple graphs, there
        # will be one, because we want one subprocess per graph.

        def responder(msg):
            msg.sendto(socket)

        dispatcher = InMemoryGraphDispatcher(responder=responder)
        dispatcher._new_graph(graph_id)

        should_continue = True
        while should_continue:
            socks = dict(poller.poll())
            if socket_pull in socks and socks[socket_pull] == zmq.POLLIN:
                msg = Message.from_frames(*socket_pull.recv_multipart())
                try:
                    dispatcher.recv_message(msg)
                except FlowError:
                    # FIXME: should we use REQ/REP so that we can confirm that each message
                    # is successfully applied?
                    raise
Пример #3
0
 def monitor_worker(self, monitor_socket, log_name):
     monitor_socket.linger = 0
     poller = zmq.Poller()
     poller.register(monitor_socket, zmq.POLLIN)
     while True:
         socks = poller.poll(1)
         gevent.sleep(0.1)
         if len(socks) > 0:
             data = recv_monitor_message(monitor_socket)
             event = data['event']
             value = data['value']
             if event == zmq.EVENT_CONNECTED:
                 logger.info('Connected to {0}'.format(log_name))
                 if 'outgoing' in log_name:
                     send_zmq_push('inproc://serverRelay',
                                   '{0}'.format(Messages.PING))
                     own_ip = gevent.socket.gethostbyname(
                         socket.gethostname())
                     send_zmq_push('inproc://serverRelay',
                                   '{0} {1}'.format(Messages.IP, own_ip))
                     send_zmq_push('inproc://serverRelay',
                                   '{0}'.format(Messages.DRONE_CONFIG))
                 elif 'incomming':
                     pass
                 else:
                     assert False
             elif event == zmq.EVENT_DISCONNECTED:
                 logger.warning(
                     'Disconnected from {0}, will reconnect in {1} seconds.'
                     .format(log_name, 5))
         gevent.sleep()
Пример #4
0
 def _loop(ns, path, *addr):
     _in = ns.V(f"{path}/in").value
     _name = ns.V(f"{path}/name").value
     _cb = ns.V(f"{path}/callbacks").value
     ctx = zmq.Context()
     socket = ctx.socket(zmq.SUB)
     for a in addr:
         socket.connect(a)
     poller = zmq.Poller()
     poller.register(socket, zmq.POLLIN)
     socket.subscribe(bytes(_name, "utf-8"))
     while ns.V(f"{path}/continue").value:
         while True:
             gevent.time.sleep(0)
             s = dict(poller.poll(1.0))
             if socket in s and s[socket] == zmq.POLLIN:
                 data = socket.recv_multipart()
                 for fun_name in _cb:
                     gevent.time.sleep(0)
                     data = _cb[fun_name](ns, path, data)
                 _in.put(data)
             else:
                 break
         gevent.time.sleep(0.5)
     ctx.term()
Пример #5
0
    def run(self):
        """Start the Authentication Agent thread task"""
        self.authenticator.start()
        zap = self.authenticator.zap_socket
        poller = zmq.Poller()
        poller.register(self.pipe, zmq.POLLIN)
        poller.register(zap, zmq.POLLIN)
        while True:
            gevent.sleep(0.1)
            try:
                socks = dict(poller.poll(1))
            except zmq.ZMQError:
                break  # interrupted
            if self.pipe in socks and socks[self.pipe] == zmq.POLLIN:
                terminate = self._handle_pipe()
                if terminate:
                    break
                if not (self.allow and self.curve):
                    continue

            if zap in socks and socks[zap] == zmq.POLLIN:
                self._handle_zap()

        self.pipe.close()
        self.authenticator.stop()
Пример #6
0
    def kill_workers(self, timeout=5):
        """
        Send a suicide message to all workers, with some kind of timeout.
        """
        logging.info('Killing workers, taking up to %d seconds.', int(timeout))
        poller = zmq.Poller()
        poller.register(self.results_pull, zmq.POLLIN)

        while True:
            # Seems to get stuck gevent-blocking in the work_push.send() after
            # all the workers have died.  Also, gevent.Timeout() doesn't seem
            # to work here?!
            signal.alarm(int(timeout))
            self.work_push.send(msgpack.dumps([{'type': 'PING'}]))
            socks = dict(poller.poll(timeout * 1500))
            if self.results_pull in socks \
                    and socks[self.results_pull] == zmq.POLLIN:
                result_packed = self.results_pull.recv()
                result = msgpack.loads(result_packed, use_list=False)
                logging.info('Heard from worker id=%d; sending SUICIDE',
                             result[0]['worker_id'])
                self.work_push.send(msgpack.dumps([{'type': 'SUICIDE'}]))
                gevent.sleep(0.1)
            else:
                break
            signal.alarm(0)
Пример #7
0
def asterisk_action(action):
    """
    :param action: {'Action': 'Name', ...}
    :return: reply from Asterisk
    """
    logger.debug('Sending action %s to %s' %
                 (action.get('Action'), ASTERISK_CMD_URL))
    # Asterisk CMD socket
    context = zmq.Context.instance()
    sock = context.socket(zmq.REQ)
    sock.setsockopt(zmq.LINGER, 0)
    sock.connect(ASTERISK_CMD_URL)
    poll = zmq.Poller()
    try:
        poll.register(sock, zmq.POLLIN)
        sock.send(json.dumps(action))
        socks = dict(poll.poll(1000))  # 1 seconds to reply!
        if socks.get(sock) == zmq.POLLIN:
            reply = sock.recv_json()
            logger.debug('Asterisk reply: %s' %
                         json.dumps(reply, indent=2, sort_keys=True))
            return reply
        else:
            logger.error('Asterisk did not reply! Action: %s' %
                         json.dumps(action, indent=2, sort_keys=True))

    except zmq.ZMQError, e:
        logger.error('Asterisk command ZMQError: %s' % e)
Пример #8
0
def websocket_zeromq(*args, **kwargs):

    queue_host = os.environ.get("FRB_QUEUE_HOST", "localhost")
    monitor_port = os.environ.get("FRB_MONITOR_PORT", "5570")

    report_printer = zmq_context.socket(zmq.SUB)
    report_printer.setsockopt(zmq.SUBSCRIBE, b'')
    report_printer.connect('tcp://{}:{}'.format(queue_host, monitor_port))

    poller = zmq.Poller()
    poller.register(report_printer, zmq.POLLIN)

    logging.getLogger("requests").setLevel(logging.WARNING)
    logger.info('listening to zeromq on 0.0.0.0:8888')
    running = True
    while running:
        socks = dict(poller.poll(0.5))
        if report_printer in socks and socks[report_printer] == zmq.POLLIN:
            raw = report_printer.recv_multipart()
            try:
                data = parse_monitor(raw)
                emit('zeromq', data)
            except TypeError:
                data = None
                logger.exception("could not json decode %s", repr(raw))

            gevent.sleep(0)
Пример #9
0
 def monitor_worker(self, monitor_socket, log_name):
     monitor_socket.linger = 0
     poller = zmq.Poller()
     poller.register(monitor_socket, zmq.POLLIN)
     while True:
         socks = poller.poll(1)
         gevent.sleep(0.1)
         if len(socks) > 0:
             data = recv_monitor_message(monitor_socket)
             event = data['event']
             if event == zmq.EVENT_CONNECTED:
                 logger.info('Connected to {0}'.format(log_name))
                 # always ask for config to avoid race condition.
                 send_zmq_push(SocketNames.SERVER_RELAY.value, '{0}'.format(Messages.DRONE_WANT_CONFIG.value))
                 if 'outgoing' in log_name:
                     send_zmq_push(SocketNames.SERVER_RELAY.value, '{0}'.format(Messages.PING.value))
                     own_ip = get_most_likely_ip()
                     send_zmq_push(SocketNames.SERVER_RELAY.value, '{0} {1}'.format(Messages.IP.value, own_ip))
                 elif 'incomming':
                     pass
                 else:
                     assert False
             elif event == zmq.EVENT_DISCONNECTED:
                 logger.warning('Disconnected from {0}, will reconnect in {1} seconds.'.format(log_name, 5))
         gevent.sleep()
Пример #10
0
    def _run(self):
        context = beeswarm.shared.zmq_context
        processed_sessions_socket = context.socket(zmq.SUB)
        processed_sessions_socket.connect(SocketNames.PROCESSED_SESSIONS.value)
        processed_sessions_socket.setsockopt(zmq.SUBSCRIBE, '')

        live_sessions_socket = context.socket(zmq.SUB)
        live_sessions_socket.connect(SocketNames.DRONE_DATA.value)
        # this auto wildcards to SESSION_PART*
        live_sessions_socket.setsockopt(zmq.SUBSCRIBE, 'SESSION_PART')

        poller = zmq.Poller()
        poller.register(processed_sessions_socket, zmq.POLLIN)
        poller.register(live_sessions_socket, zmq.POLLIN)

        while self.enabled:
            socks = dict(poller.poll(1000))
            if processed_sessions_socket in socks and socks[processed_sessions_socket] == zmq.POLLIN:
                topic, data = processed_sessions_socket.recv().split(' ', 1)
                self.handle_processed_session(topic, data)
            elif live_sessions_socket in socks and socks[live_sessions_socket] == zmq.POLLIN:
                topic, data = live_sessions_socket.recv().split(' ', 1)
                self.handle_live_session_part(topic, data)

        live_sessions_socket.close()
        processed_sessions_socket.close()
Пример #11
0
    def _start_zmq_puller(self):
        context = zmq.Context()

        # Authenticator runs in different greenlet.
        auth = GreenThreadAuthenticator(context)
        auth.start()
        auth.allow('127.0.0.1')
        auth.configure_curve(domain='*',
                             location='heralding/tests/zmq_public_keys')

        # Bind our mock zmq pull server
        socket = context.socket(zmq.PULL)
        socket.curve_secretkey = "}vxNPm8lOJT1yvqu7-A<m<w>7OZ1ok<d?Qbq+a?5"
        socket.curve_server = True
        self.zmq_tcp_port = socket.bind_to_random_port('tcp://*',
                                                       min_port=40000,
                                                       max_port=50000,
                                                       max_tries=10)

        # Poll and wait for data from test client
        poller = zmq.Poller()
        poller.register(socket, zmq.POLLIN)

        # Need to notify test client that the server is ready
        self.zmq_server_listning_event.set()

        while self.test_running:
            socks = dict(poller.poll())
            if socket in socks and socks[socket] == zmq.POLLIN:
                data = socket.recv()
                self.testing_queue.put(data)
        socket.close()
Пример #12
0
 def __init__(self, ctx, pipe, outbox, *args, **kwargs):
     self._ctx = ctx                             #... until we use zbeacon actor
     self._pipe = pipe                           # We send command replies and signals to the pipe
                                                 # Pipe back to application
     self.outbox = outbox                        # Outbox back to application
     self._terminated = False                    # API shut us down
     self._verbose = False                       # Log all traffic (logging module?)
     self.beacon_interface = ''                  # Beacon interface
     self.beacon_port = ZRE_DISCOVERY_PORT       # Beacon port number
     self.interval = 0                           # Beacon interval 0=default
     self.beacon = None                          # Beacon actor
     self.beacon_socket = None                   # Beacon socket for polling
     self.poller = zmq.Poller()                  # Socket poller
     self.identity = uuid.uuid4()                # Our UUID as object
     self.bound = False
     self.inbox = ctx.socket(zmq.ROUTER)         # Our inbox socket (ROUTER)
     try:
         self.inbox.setsockopt(zmq.ROUTER_HANDOVER, 1)
     except AttributeError as e:
         logging.warning("can't set ROUTER_HANDOVER, needs zmq version >=4.1 but installed is {0}".format(zmq.zmq_version()))
     self.poller.register(self._pipe, zmq.POLLIN)
     self.name = str(self.identity)[:6]          # Our public name (default=first 6 uuid chars)
     self.endpoint = ""                          # Our public endpoint
     self.port = 0                               # Our inbox port, if any
     self.status = 0                             # Our own change counter
     self.peers = {}                             # Hash of known peers, fast lookup
     self.peer_groups = {}                       # Groups that our peers are in
     self.own_groups = {}                        # Groups that we are in
     self.headers = {}                           # Our header values
     # TODO: gossip stuff
     #self.start()
     self.run()
Пример #13
0
    def cmdGreenlet(self):
        # Nonblocking
        self.cmdcontext = zmq.Context()

        frontend = self.cmdcontext.socket(zmq.ROUTER)
        backend = self.cmdcontext.socket(zmq.DEALER)

        frontend.bind("tcp://*:%s" % self.port)
        backend.bind("inproc://cmdworkers")

        # Initialize poll set
        poller = zmq.Poller()
        poller.register(frontend, zmq.POLLIN)
        poller.register(backend, zmq.POLLIN)

        workers = []

        for i in range(self.nrCmdGreenlets):
            workers.append(gevent.spawn(self.repCmdServer))

        while True:
            socks = dict(poller.poll())
            if socks.get(frontend) == zmq.POLLIN:
                parts = frontend.recv_multipart()
                parts.append(parts[0])  # add session id at end
                backend.send_multipart([parts[0]] + parts)

            if socks.get(backend) == zmq.POLLIN:
                parts = backend.recv_multipart()
                frontend.send_multipart(
                    parts[1:]
                )  # @todo dont understand why I need to remove first part of parts?
Пример #14
0
    def __init__(self,
                 name,
                 listen_port=None,
                 publish_port=None,
                 *args,
                 **kwargs):

        Actor.__init__(self, name, *args, **kwargs)
        RegistrationService.__init__(self, *args, **kwargs)
        self.brokers = {}

        if listen_port is not None:
            self.registration_service_port = listen_port

        if publish_port is not None:
            self.registration_publisher_port = publish_port

        self.receiver_socket = self.context.socket(zmq.ROUTER)
        self.receiver_socket.bind("tcp://*:{0}".format(
            self.registration_service_port))

        self.client_publisher_socket = self.context.socket(zmq.PUB)
        self.client_publisher_socket.bind("tcp://*:{0}".format(
            self.registration_publisher_port))

        gevent.sleep(
            0.1
        )  # Make sure publisher has time to fully connect. This is a zmq nuance

        #self.heartbeat_manager = HeartbeatManager(heartbeat_interval=self.timeout)
        self.poller = zmq.Poller()
        self.poller.register(self.receiver_socket, zmq.POLLIN)
Пример #15
0
    def __init__(self,
                 service,
                 logger=None,
                 interval=None,
                 expiry=None,
                 burn_in_pings=None):
        self.pool = Pool()
        self.udp = UDP(PING_PORT_NUMBER)
        self.uuid = uuid.uuid4().hex.encode('utf8')
        self.poller = zmq.Poller()
        self.poller.register(self.udp.handle, zmq.POLLIN)
        self.service = service
        self.logger = logger

        self.expiry = expiry or PEER_EXPIRY
        self.interval = interval or PING_INTERVAL
        self.burn_in_pings = burn_in_pings or BURN_IN_PINGS

        self.__master_block = event.Event()
        self.__master_block.clear()

        self.__block = event.Event()
        self.__block.clear()

        self.__burned_in = False

        self.__loop = event.Event()
        self.__loop = False
        self.__set_slave()
        self.peers = {}
Пример #16
0
    def process(self, request):
        zmq_socket = self.zmq_socket

        poller = zmq.Poller()
        poller.register(zmq_socket, zmq.POLLIN)

        zmq_socket.send_json({
            "command": request["command"],
            "arguments": request["arguments"]
        })
        sockets = dict(poller.poll(10 * 1000))

        if zmq_socket not in sockets:
            self.init_socket()
            result = request["result"]
            result.set_exception(TimeoutError("The request to the backend timed out."))
            return
 
        received = zmq_socket.recv_json()

        result = request["result"]
        if received["success"]:
            result.set(received["result"])
        else:
            result.set_exception(BackendError(received["result"]))
Пример #17
0
    def run(self):
        # Signal actor successfully initialized
        self.pipe.signal()

        self.poller = zmq.Poller()
        self.poller.register(self.pipe, zmq.POLLIN)
        self.poller.register(self.udpsock, zmq.POLLIN)

        while not self.terminated:
            timeout = 1
            if self.transmit:
                timeout = self.ping_at - time.time()
                if timeout < 0:
                    timeout = 0
            # Poll on API pipe and on UDP socket
            items = dict(self.poller.poll(timeout * 1000))
            if self.pipe in items and items[self.pipe] == zmq.POLLIN:
                self.handle_pipe()
            if self.udpsock.fileno() in items and items[
                    self.udpsock.fileno()] == zmq.POLLIN:
                self.handle_udp()

            if self.transmit and time.time() >= self.ping_at:
                self.send_beacon()
                self.ping_at = time.time() + self.interval
Пример #18
0
 def __init__(self):
     self.context = zmq.Context()
     self.data_socket = self.context.socket(zmq.PUB)
     self.data_socket.connect ("tcp://localhost:5556")
     self.control_socket = self.context.socket(zmq.REQ)
     self.control_socket.connect("tcp://localhost:5557")
     self.control_poller = zmq.Poller()
     self.control_poller.register(self.control_socket, zmq.POLLOUT)
Пример #19
0
    def __init__(self, sockets, time):
        self.sockets = sockets
        self.poller = zmq.Poller()

        for socket in self.sockets.values():
            self.poller.register(socket["socket"], zmq.POLLIN | zmq.POLLOUT)

        self.time = time
Пример #20
0
def agent_exchange(in_addr, out_addr, logger_name=None):
    '''Agent message publish/subscribe exchange loop

    Accept multi-part messages from sockets connected to in_addr, which
    is a PULL socket, and forward them to sockets connected to out_addr,
    which is a XPUB socket. When subscriptions are added or removed, a
    message of the form 'subscriptions/<OP>/<TOPIC>' is broadcast to the
    PUB socket where <OP> is either 'add' or 'remove' and <TOPIC> is the
    topic being subscribed or unsubscribed. When a message is received
    of the form 'subscriptions/list/<PREFIX>', a multipart message will
    be broadcast with the first two received frames (topic and headers)
    sent unchanged and with the remainder of the message containing
    currently subscribed topics which start with <PREFIX>, each frame
    containing exactly one topic.

    If logger_name is given, a new logger will be created with the given
    name. Otherwise, the module logger will be used.
    '''
    log = _log if logger_name is None else logging.getLogger(logger_name)
    ctx = zmq.Context.instance()
    with closing(ctx.socket(zmq.PULL)) as in_sock, \
            closing(ctx.socket(zmq.XPUB)) as out_sock:
        in_sock.bind(in_addr)
        out_sock.bind(out_addr)
        poller = zmq.Poller()
        poller.register(in_sock, zmq.POLLIN)
        poller.register(out_sock, zmq.POLLIN)
        subscriptions = set()
        while True:
            for sock, event in poller.poll():
                if sock is in_sock:
                    message = in_sock.recv_multipart()
                    log.debug('incoming message: {!r}'.format(message))
                    topic = message[0]
                    if (topic.startswith('subscriptions/list')
                            and topic[18:19] in ['/', '']):
                        if len(message) > 2:
                            del message[2:]
                        elif len(message) == 1:
                            message.append('')
                        prefix = topic[19:]
                        message.extend(
                            [t for t in subscriptions if t.startswith(prefix)])
                    out_sock.send_multipart(message)
                elif sock is out_sock:
                    message = out_sock.recv()
                    if message:
                        add = bool(ord(message[0]))
                        topic = message[1:]
                        if add:
                            subscriptions.add(topic)
                        else:
                            subscriptions.discard(topic)
                        log.debug('incoming subscription: {} {!r}'.format(
                            ('add' if add else 'remove'), topic))
                        out_sock.send('subscriptions/{}{}{}'.format(
                            ('add' if add else 'remove'),
                            ('' if topic[:1] == '/' else '/'), topic))
Пример #21
0
 def __init__(self, context, continuum_ip, continuum_port):
     threading.Thread.__init__(self)
     self.context = context
     self.continuum_ip = continuum_ip
     self.continuum_port = continuum_port
     self.socket_state = STATE_DESTROYED
     self.last_active_time = datetime.now()
     self.connected = False
     self.poller = zmq.Poller()
Пример #22
0
 def __init__(self, ws=None):
     print '__init__'
     super(ZmqApplication, self).__init__(ws)
     self.push_socket = context.socket(zmq.PUSH)
     self.push_socket.connect('tcp://127.0.0.1:5557')
     self.rep_socket = context.socket(zmq.REP)
     self.rep_socket.connect('tcp://127.0.0.1:5558')
     self.poller = zmq.Poller()
     self.poller.register(self.rep_socket, zmq.POLLIN)
Пример #23
0
def push():
    req_socket = context.socket(zmq.REQ)
    req_socket.bind('tcp://127.0.0.1:5558')
    pollout = zmq.Poller()
    pollout.register(req_socket, zmq.POLLOUT)
    pollin = zmq.Poller()
    pollin.register(req_socket, zmq.POLLIN)

    i = 1
    while True:
        if pollout.poll():
            print 'send message %d' % i,
            req_socket.send('new_message: %d' % i)
            if pollin.poll():
                print 'received %r' % req_socket.recv()
            else:
                print ''
            i += 1
        gevent.sleep(random.random())
Пример #24
0
    def _run(self):
        self.config_commands.bind(SocketNames.CONFIG_COMMANDS.value)

        poller = zmq.Poller()
        poller.register(self.config_commands, zmq.POLLIN)
        while self.enabled:
            socks = dict(poller.poll(500))
            if self.config_commands in socks and socks[
                    self.config_commands] == zmq.POLLIN:
                self._handle_commands()
Пример #25
0
    def _setup_proxy(self):
        self.frontend = self.context.socket(zmq.XSUB)
        self.frontend.bind(self.XSUB_URL)

        self.backend = self.context.socket(zmq.XPUB)
        self.backend.bind(self.XPUB_URL)

        self.poller = zmq.Poller()
        self.poller.register(self.frontend, zmq.POLLIN)
        self.poller.register(self.backend, zmq.POLLIN)
Пример #26
0
    def __init__(self, config, on_message_callback):
        self.config = config
        self.on_message_callback = on_message_callback
        self.address = config.address
        self.poll_interval = config.poll_interval
        self.pool_strategy = config.pool_strategy
        self.service_source = config.service_source
        self.keep_running = True
        self.tcp_port = int(self.address.split(':')[-1])

        # A hundred years in seconds, used when creating internal workers
        self.y100 = 60 * 60 * 24 * 365 * 100

        # So they do not have to be looked up on each request or event
        self.has_info = logger.isEnabledFor(logging.INFO)
        self.has_debug = logger.isEnabledFor(logging.DEBUG)
        self.has_pool_strategy_simple = self.pool_strategy == ZMQ.POOL_STRATEGY_NAME.SINGLE
        self.has_service_source_zato = self.service_source == ZMQ.SERVICE_SOURCE_NAME.ZATO
        self.zato_service_name = config.service_name
        self.zato_channel = CHANNEL.ZMQ

        if self.has_pool_strategy_simple:
            self.workers_pool_initial = 1
            self.workers_pool_mult = 0
            self.workers_pool_max = 1
        else:
            self.workers_pool_initial = config.workers_pool_initial
            self.workers_pool_mult = config.workers_pool_mult
            self.workers_pool_max = config.workers_pool_max

        # Maps service names to workers registered to handle requests to that service
        self.services = {}

        # Details about each worker, mapped by worker_id:Worker object
        self.workers = {}

        # Held upon most operations on sockets
        self.lock = RLock()

        # How often, in seconds, to send a heartbeat to workers
        self.heartbeat = config.heartbeat

        self.ctx = zmq.Context()
        self.socket = self.ctx.socket(zmq.ROUTER)
        self.socket.linger = config.linger
        self.poller = zmq.Poller()
        self.poller.register(self.socket, zmq.POLLIN)

        # Maps event IDs to methods that handle a given one
        self.handle_event_map = {
            const.v01.ready: self.on_event_ready,
            const.v01.reply_from_worker: self.on_event_reply,
            const.v01.heartbeat: self.on_event_heartbeat,
            const.v01.disconnect: self.on_event_disconnect,
        }
Пример #27
0
    def _run(self):
        self.config_commands.bind('inproc://configCommands')
        if not self.commands_only:
            self.drone_command_receiver.connect('inproc://droneCommandReceiver')

        poller = zmq.Poller()
        poller.register(self.config_commands, zmq.POLLIN)
        while self.enabled:
            socks = dict(poller.poll(500))
            if self.config_commands in socks and socks[self.config_commands] == zmq.POLLIN:
                self._handle_commands()
Пример #28
0
    def collect(self):
        poller = zmq.Poller()
        poller.register(self.pull_socket, zmq.POLLIN)
        poller.register(self.ctrl_socket, zmq.POLLIN)

        pull_socket = self.pull_socket
        ctrl_socket = self.ctrl_socket

        while True:

            if self.have_bytecode:

                try:
                    events = dict(poller.poll())
                except zmq.ZMQError:
                    self._kill()
                    break

                if events.get(pull_socket) == zmq.POLLIN:

                    command = self.pull_socket.recv(flags=zmq.SNDMORE)
                    key = self.pull_socket.recv(flags=zmq.SNDMORE)
                    data = self.pull_socket.recv(copy=False)

                    if command == MAP:
                        self.call_mapfn(key, data)
                    elif command == REDUCE:
                        self.call_reducefn(key, data)

                if events.get(ctrl_socket) == zmq.POLLIN:
                    worker_id, command = self.ctrl_socket.recv_multipart()
                    if command == HEARTBEAT:
                        self.ctrl_socket.send(PONG)
                        print 'pong'
                    if command == DONE:
                        self.kill()
                        break

            else:
                self.logging.info('Waiting for server')

                # Associate with the server
                self.push_socket.send_multipart([CONNECT, self.worker_id])

                # Wait for the server to route us the bytecode,
                # then start the work cycle
                worker_id, payload = self.ctrl_socket.recv_multipart()
                command, (mapbc, reducebc) = srl.loads(payload)

                assert command == BYTECODE
                self.set_bytecode(mapbc, reducebc)
                self.logging.info('Received Bytecode')
Пример #29
0
def send_data(ws):
    logger.info('Got a websocket connection, sending up data from zmq')
    socket = context.socket(zmq.SUB)
    socket.connect('tcp://localhost:{PORT}'.format(PORT=ZMQ_LISTENING_PORT))
    socket.setsockopt(zmq.SUBSCRIBE, "")
    poller = zmq.Poller()
    poller.register(socket, zmq.POLLIN)
    gevent.sleep()
    while True:
        # socks = dict(poller.poll())
        # if socket in socks and socks[socket] == zmq.POLLIN:
        data = socket.recv_json()
        logger.info(data)
        ws.send(json.dumps(data))
        gevent.sleep()
Пример #30
0
    def __init__(self, broker, cls, service_suffix=''):
        self._broker = broker
        self._wrapped_cls = cls
        self._service = bytes('{}{}'.format(cls.__name__, service_suffix),
                              'utf8')
        self._ignore_service = False

        self._retries = config['client_retries']
        self._timeout = config['client_timeout']

        self._context = zmq.Context()
        self._poller = zmq.Poller()
        self._sock = None

        self._connect_to_broker()