def __init__(self, controller_identity=None, logger=None, *args, **kwargs): RegistrationService.__init__(self, *args, **kwargs) self.controller_identity = controller_identity or kwargs.get( 'controller_identity', None) self.subscriber_socket = self.context.socket(zmq.SUB) self.subscriber_socket.setsockopt(zmq.SUBSCRIBE, self.manager_subscriber_scope) self.subscriber_socket.connect("tcp://localhost:{0}".format( self.registration_publisher_port)) self.verified_brokers = {} self.unverified_brokers = {} self.inbound_sockets = [] self.logger = logger self.inbound_poller = zmq.Poller() self.heartbeat_manager = HeartbeatManager(*args, **kwargs) self.verification_manager = HeartbeatManager(*args, **kwargs) self.poller = zmq.Poller() self.poller.register(self.subscriber_socket, zmq.POLLIN) gevent.spawn(self.listen_for_updates)
def client(recv_port, graph_id): context = zmq.Context() socket_pull = context.socket(zmq.PULL) socket_pull.connect("tcp://localhost:%s" % recv_port) print("Connected to server with port %s" % recv_port) # Initialize poll set poller = zmq.Poller() poller.register(socket_pull, zmq.POLLIN) # even though InMemoryGraphDispatcher can handle multiple graphs, there # will be one, because we want one subprocess per graph. def responder(msg): msg.sendto(socket) dispatcher = InMemoryGraphDispatcher(responder=responder) dispatcher._new_graph(graph_id) should_continue = True while should_continue: socks = dict(poller.poll()) if socket_pull in socks and socks[socket_pull] == zmq.POLLIN: msg = Message.from_frames(*socket_pull.recv_multipart()) try: dispatcher.recv_message(msg) except FlowError: # FIXME: should we use REQ/REP so that we can confirm that each message # is successfully applied? raise
def monitor_worker(self, monitor_socket, log_name): monitor_socket.linger = 0 poller = zmq.Poller() poller.register(monitor_socket, zmq.POLLIN) while True: socks = poller.poll(1) gevent.sleep(0.1) if len(socks) > 0: data = recv_monitor_message(monitor_socket) event = data['event'] value = data['value'] if event == zmq.EVENT_CONNECTED: logger.info('Connected to {0}'.format(log_name)) if 'outgoing' in log_name: send_zmq_push('inproc://serverRelay', '{0}'.format(Messages.PING)) own_ip = gevent.socket.gethostbyname( socket.gethostname()) send_zmq_push('inproc://serverRelay', '{0} {1}'.format(Messages.IP, own_ip)) send_zmq_push('inproc://serverRelay', '{0}'.format(Messages.DRONE_CONFIG)) elif 'incomming': pass else: assert False elif event == zmq.EVENT_DISCONNECTED: logger.warning( 'Disconnected from {0}, will reconnect in {1} seconds.' .format(log_name, 5)) gevent.sleep()
def _loop(ns, path, *addr): _in = ns.V(f"{path}/in").value _name = ns.V(f"{path}/name").value _cb = ns.V(f"{path}/callbacks").value ctx = zmq.Context() socket = ctx.socket(zmq.SUB) for a in addr: socket.connect(a) poller = zmq.Poller() poller.register(socket, zmq.POLLIN) socket.subscribe(bytes(_name, "utf-8")) while ns.V(f"{path}/continue").value: while True: gevent.time.sleep(0) s = dict(poller.poll(1.0)) if socket in s and s[socket] == zmq.POLLIN: data = socket.recv_multipart() for fun_name in _cb: gevent.time.sleep(0) data = _cb[fun_name](ns, path, data) _in.put(data) else: break gevent.time.sleep(0.5) ctx.term()
def run(self): """Start the Authentication Agent thread task""" self.authenticator.start() zap = self.authenticator.zap_socket poller = zmq.Poller() poller.register(self.pipe, zmq.POLLIN) poller.register(zap, zmq.POLLIN) while True: gevent.sleep(0.1) try: socks = dict(poller.poll(1)) except zmq.ZMQError: break # interrupted if self.pipe in socks and socks[self.pipe] == zmq.POLLIN: terminate = self._handle_pipe() if terminate: break if not (self.allow and self.curve): continue if zap in socks and socks[zap] == zmq.POLLIN: self._handle_zap() self.pipe.close() self.authenticator.stop()
def kill_workers(self, timeout=5): """ Send a suicide message to all workers, with some kind of timeout. """ logging.info('Killing workers, taking up to %d seconds.', int(timeout)) poller = zmq.Poller() poller.register(self.results_pull, zmq.POLLIN) while True: # Seems to get stuck gevent-blocking in the work_push.send() after # all the workers have died. Also, gevent.Timeout() doesn't seem # to work here?! signal.alarm(int(timeout)) self.work_push.send(msgpack.dumps([{'type': 'PING'}])) socks = dict(poller.poll(timeout * 1500)) if self.results_pull in socks \ and socks[self.results_pull] == zmq.POLLIN: result_packed = self.results_pull.recv() result = msgpack.loads(result_packed, use_list=False) logging.info('Heard from worker id=%d; sending SUICIDE', result[0]['worker_id']) self.work_push.send(msgpack.dumps([{'type': 'SUICIDE'}])) gevent.sleep(0.1) else: break signal.alarm(0)
def asterisk_action(action): """ :param action: {'Action': 'Name', ...} :return: reply from Asterisk """ logger.debug('Sending action %s to %s' % (action.get('Action'), ASTERISK_CMD_URL)) # Asterisk CMD socket context = zmq.Context.instance() sock = context.socket(zmq.REQ) sock.setsockopt(zmq.LINGER, 0) sock.connect(ASTERISK_CMD_URL) poll = zmq.Poller() try: poll.register(sock, zmq.POLLIN) sock.send(json.dumps(action)) socks = dict(poll.poll(1000)) # 1 seconds to reply! if socks.get(sock) == zmq.POLLIN: reply = sock.recv_json() logger.debug('Asterisk reply: %s' % json.dumps(reply, indent=2, sort_keys=True)) return reply else: logger.error('Asterisk did not reply! Action: %s' % json.dumps(action, indent=2, sort_keys=True)) except zmq.ZMQError, e: logger.error('Asterisk command ZMQError: %s' % e)
def websocket_zeromq(*args, **kwargs): queue_host = os.environ.get("FRB_QUEUE_HOST", "localhost") monitor_port = os.environ.get("FRB_MONITOR_PORT", "5570") report_printer = zmq_context.socket(zmq.SUB) report_printer.setsockopt(zmq.SUBSCRIBE, b'') report_printer.connect('tcp://{}:{}'.format(queue_host, monitor_port)) poller = zmq.Poller() poller.register(report_printer, zmq.POLLIN) logging.getLogger("requests").setLevel(logging.WARNING) logger.info('listening to zeromq on 0.0.0.0:8888') running = True while running: socks = dict(poller.poll(0.5)) if report_printer in socks and socks[report_printer] == zmq.POLLIN: raw = report_printer.recv_multipart() try: data = parse_monitor(raw) emit('zeromq', data) except TypeError: data = None logger.exception("could not json decode %s", repr(raw)) gevent.sleep(0)
def monitor_worker(self, monitor_socket, log_name): monitor_socket.linger = 0 poller = zmq.Poller() poller.register(monitor_socket, zmq.POLLIN) while True: socks = poller.poll(1) gevent.sleep(0.1) if len(socks) > 0: data = recv_monitor_message(monitor_socket) event = data['event'] if event == zmq.EVENT_CONNECTED: logger.info('Connected to {0}'.format(log_name)) # always ask for config to avoid race condition. send_zmq_push(SocketNames.SERVER_RELAY.value, '{0}'.format(Messages.DRONE_WANT_CONFIG.value)) if 'outgoing' in log_name: send_zmq_push(SocketNames.SERVER_RELAY.value, '{0}'.format(Messages.PING.value)) own_ip = get_most_likely_ip() send_zmq_push(SocketNames.SERVER_RELAY.value, '{0} {1}'.format(Messages.IP.value, own_ip)) elif 'incomming': pass else: assert False elif event == zmq.EVENT_DISCONNECTED: logger.warning('Disconnected from {0}, will reconnect in {1} seconds.'.format(log_name, 5)) gevent.sleep()
def _run(self): context = beeswarm.shared.zmq_context processed_sessions_socket = context.socket(zmq.SUB) processed_sessions_socket.connect(SocketNames.PROCESSED_SESSIONS.value) processed_sessions_socket.setsockopt(zmq.SUBSCRIBE, '') live_sessions_socket = context.socket(zmq.SUB) live_sessions_socket.connect(SocketNames.DRONE_DATA.value) # this auto wildcards to SESSION_PART* live_sessions_socket.setsockopt(zmq.SUBSCRIBE, 'SESSION_PART') poller = zmq.Poller() poller.register(processed_sessions_socket, zmq.POLLIN) poller.register(live_sessions_socket, zmq.POLLIN) while self.enabled: socks = dict(poller.poll(1000)) if processed_sessions_socket in socks and socks[processed_sessions_socket] == zmq.POLLIN: topic, data = processed_sessions_socket.recv().split(' ', 1) self.handle_processed_session(topic, data) elif live_sessions_socket in socks and socks[live_sessions_socket] == zmq.POLLIN: topic, data = live_sessions_socket.recv().split(' ', 1) self.handle_live_session_part(topic, data) live_sessions_socket.close() processed_sessions_socket.close()
def _start_zmq_puller(self): context = zmq.Context() # Authenticator runs in different greenlet. auth = GreenThreadAuthenticator(context) auth.start() auth.allow('127.0.0.1') auth.configure_curve(domain='*', location='heralding/tests/zmq_public_keys') # Bind our mock zmq pull server socket = context.socket(zmq.PULL) socket.curve_secretkey = "}vxNPm8lOJT1yvqu7-A<m<w>7OZ1ok<d?Qbq+a?5" socket.curve_server = True self.zmq_tcp_port = socket.bind_to_random_port('tcp://*', min_port=40000, max_port=50000, max_tries=10) # Poll and wait for data from test client poller = zmq.Poller() poller.register(socket, zmq.POLLIN) # Need to notify test client that the server is ready self.zmq_server_listning_event.set() while self.test_running: socks = dict(poller.poll()) if socket in socks and socks[socket] == zmq.POLLIN: data = socket.recv() self.testing_queue.put(data) socket.close()
def __init__(self, ctx, pipe, outbox, *args, **kwargs): self._ctx = ctx #... until we use zbeacon actor self._pipe = pipe # We send command replies and signals to the pipe # Pipe back to application self.outbox = outbox # Outbox back to application self._terminated = False # API shut us down self._verbose = False # Log all traffic (logging module?) self.beacon_interface = '' # Beacon interface self.beacon_port = ZRE_DISCOVERY_PORT # Beacon port number self.interval = 0 # Beacon interval 0=default self.beacon = None # Beacon actor self.beacon_socket = None # Beacon socket for polling self.poller = zmq.Poller() # Socket poller self.identity = uuid.uuid4() # Our UUID as object self.bound = False self.inbox = ctx.socket(zmq.ROUTER) # Our inbox socket (ROUTER) try: self.inbox.setsockopt(zmq.ROUTER_HANDOVER, 1) except AttributeError as e: logging.warning("can't set ROUTER_HANDOVER, needs zmq version >=4.1 but installed is {0}".format(zmq.zmq_version())) self.poller.register(self._pipe, zmq.POLLIN) self.name = str(self.identity)[:6] # Our public name (default=first 6 uuid chars) self.endpoint = "" # Our public endpoint self.port = 0 # Our inbox port, if any self.status = 0 # Our own change counter self.peers = {} # Hash of known peers, fast lookup self.peer_groups = {} # Groups that our peers are in self.own_groups = {} # Groups that we are in self.headers = {} # Our header values # TODO: gossip stuff #self.start() self.run()
def cmdGreenlet(self): # Nonblocking self.cmdcontext = zmq.Context() frontend = self.cmdcontext.socket(zmq.ROUTER) backend = self.cmdcontext.socket(zmq.DEALER) frontend.bind("tcp://*:%s" % self.port) backend.bind("inproc://cmdworkers") # Initialize poll set poller = zmq.Poller() poller.register(frontend, zmq.POLLIN) poller.register(backend, zmq.POLLIN) workers = [] for i in range(self.nrCmdGreenlets): workers.append(gevent.spawn(self.repCmdServer)) while True: socks = dict(poller.poll()) if socks.get(frontend) == zmq.POLLIN: parts = frontend.recv_multipart() parts.append(parts[0]) # add session id at end backend.send_multipart([parts[0]] + parts) if socks.get(backend) == zmq.POLLIN: parts = backend.recv_multipart() frontend.send_multipart( parts[1:] ) # @todo dont understand why I need to remove first part of parts?
def __init__(self, name, listen_port=None, publish_port=None, *args, **kwargs): Actor.__init__(self, name, *args, **kwargs) RegistrationService.__init__(self, *args, **kwargs) self.brokers = {} if listen_port is not None: self.registration_service_port = listen_port if publish_port is not None: self.registration_publisher_port = publish_port self.receiver_socket = self.context.socket(zmq.ROUTER) self.receiver_socket.bind("tcp://*:{0}".format( self.registration_service_port)) self.client_publisher_socket = self.context.socket(zmq.PUB) self.client_publisher_socket.bind("tcp://*:{0}".format( self.registration_publisher_port)) gevent.sleep( 0.1 ) # Make sure publisher has time to fully connect. This is a zmq nuance #self.heartbeat_manager = HeartbeatManager(heartbeat_interval=self.timeout) self.poller = zmq.Poller() self.poller.register(self.receiver_socket, zmq.POLLIN)
def __init__(self, service, logger=None, interval=None, expiry=None, burn_in_pings=None): self.pool = Pool() self.udp = UDP(PING_PORT_NUMBER) self.uuid = uuid.uuid4().hex.encode('utf8') self.poller = zmq.Poller() self.poller.register(self.udp.handle, zmq.POLLIN) self.service = service self.logger = logger self.expiry = expiry or PEER_EXPIRY self.interval = interval or PING_INTERVAL self.burn_in_pings = burn_in_pings or BURN_IN_PINGS self.__master_block = event.Event() self.__master_block.clear() self.__block = event.Event() self.__block.clear() self.__burned_in = False self.__loop = event.Event() self.__loop = False self.__set_slave() self.peers = {}
def process(self, request): zmq_socket = self.zmq_socket poller = zmq.Poller() poller.register(zmq_socket, zmq.POLLIN) zmq_socket.send_json({ "command": request["command"], "arguments": request["arguments"] }) sockets = dict(poller.poll(10 * 1000)) if zmq_socket not in sockets: self.init_socket() result = request["result"] result.set_exception(TimeoutError("The request to the backend timed out.")) return received = zmq_socket.recv_json() result = request["result"] if received["success"]: result.set(received["result"]) else: result.set_exception(BackendError(received["result"]))
def run(self): # Signal actor successfully initialized self.pipe.signal() self.poller = zmq.Poller() self.poller.register(self.pipe, zmq.POLLIN) self.poller.register(self.udpsock, zmq.POLLIN) while not self.terminated: timeout = 1 if self.transmit: timeout = self.ping_at - time.time() if timeout < 0: timeout = 0 # Poll on API pipe and on UDP socket items = dict(self.poller.poll(timeout * 1000)) if self.pipe in items and items[self.pipe] == zmq.POLLIN: self.handle_pipe() if self.udpsock.fileno() in items and items[ self.udpsock.fileno()] == zmq.POLLIN: self.handle_udp() if self.transmit and time.time() >= self.ping_at: self.send_beacon() self.ping_at = time.time() + self.interval
def __init__(self): self.context = zmq.Context() self.data_socket = self.context.socket(zmq.PUB) self.data_socket.connect ("tcp://localhost:5556") self.control_socket = self.context.socket(zmq.REQ) self.control_socket.connect("tcp://localhost:5557") self.control_poller = zmq.Poller() self.control_poller.register(self.control_socket, zmq.POLLOUT)
def __init__(self, sockets, time): self.sockets = sockets self.poller = zmq.Poller() for socket in self.sockets.values(): self.poller.register(socket["socket"], zmq.POLLIN | zmq.POLLOUT) self.time = time
def agent_exchange(in_addr, out_addr, logger_name=None): '''Agent message publish/subscribe exchange loop Accept multi-part messages from sockets connected to in_addr, which is a PULL socket, and forward them to sockets connected to out_addr, which is a XPUB socket. When subscriptions are added or removed, a message of the form 'subscriptions/<OP>/<TOPIC>' is broadcast to the PUB socket where <OP> is either 'add' or 'remove' and <TOPIC> is the topic being subscribed or unsubscribed. When a message is received of the form 'subscriptions/list/<PREFIX>', a multipart message will be broadcast with the first two received frames (topic and headers) sent unchanged and with the remainder of the message containing currently subscribed topics which start with <PREFIX>, each frame containing exactly one topic. If logger_name is given, a new logger will be created with the given name. Otherwise, the module logger will be used. ''' log = _log if logger_name is None else logging.getLogger(logger_name) ctx = zmq.Context.instance() with closing(ctx.socket(zmq.PULL)) as in_sock, \ closing(ctx.socket(zmq.XPUB)) as out_sock: in_sock.bind(in_addr) out_sock.bind(out_addr) poller = zmq.Poller() poller.register(in_sock, zmq.POLLIN) poller.register(out_sock, zmq.POLLIN) subscriptions = set() while True: for sock, event in poller.poll(): if sock is in_sock: message = in_sock.recv_multipart() log.debug('incoming message: {!r}'.format(message)) topic = message[0] if (topic.startswith('subscriptions/list') and topic[18:19] in ['/', '']): if len(message) > 2: del message[2:] elif len(message) == 1: message.append('') prefix = topic[19:] message.extend( [t for t in subscriptions if t.startswith(prefix)]) out_sock.send_multipart(message) elif sock is out_sock: message = out_sock.recv() if message: add = bool(ord(message[0])) topic = message[1:] if add: subscriptions.add(topic) else: subscriptions.discard(topic) log.debug('incoming subscription: {} {!r}'.format( ('add' if add else 'remove'), topic)) out_sock.send('subscriptions/{}{}{}'.format( ('add' if add else 'remove'), ('' if topic[:1] == '/' else '/'), topic))
def __init__(self, context, continuum_ip, continuum_port): threading.Thread.__init__(self) self.context = context self.continuum_ip = continuum_ip self.continuum_port = continuum_port self.socket_state = STATE_DESTROYED self.last_active_time = datetime.now() self.connected = False self.poller = zmq.Poller()
def __init__(self, ws=None): print '__init__' super(ZmqApplication, self).__init__(ws) self.push_socket = context.socket(zmq.PUSH) self.push_socket.connect('tcp://127.0.0.1:5557') self.rep_socket = context.socket(zmq.REP) self.rep_socket.connect('tcp://127.0.0.1:5558') self.poller = zmq.Poller() self.poller.register(self.rep_socket, zmq.POLLIN)
def push(): req_socket = context.socket(zmq.REQ) req_socket.bind('tcp://127.0.0.1:5558') pollout = zmq.Poller() pollout.register(req_socket, zmq.POLLOUT) pollin = zmq.Poller() pollin.register(req_socket, zmq.POLLIN) i = 1 while True: if pollout.poll(): print 'send message %d' % i, req_socket.send('new_message: %d' % i) if pollin.poll(): print 'received %r' % req_socket.recv() else: print '' i += 1 gevent.sleep(random.random())
def _run(self): self.config_commands.bind(SocketNames.CONFIG_COMMANDS.value) poller = zmq.Poller() poller.register(self.config_commands, zmq.POLLIN) while self.enabled: socks = dict(poller.poll(500)) if self.config_commands in socks and socks[ self.config_commands] == zmq.POLLIN: self._handle_commands()
def _setup_proxy(self): self.frontend = self.context.socket(zmq.XSUB) self.frontend.bind(self.XSUB_URL) self.backend = self.context.socket(zmq.XPUB) self.backend.bind(self.XPUB_URL) self.poller = zmq.Poller() self.poller.register(self.frontend, zmq.POLLIN) self.poller.register(self.backend, zmq.POLLIN)
def __init__(self, config, on_message_callback): self.config = config self.on_message_callback = on_message_callback self.address = config.address self.poll_interval = config.poll_interval self.pool_strategy = config.pool_strategy self.service_source = config.service_source self.keep_running = True self.tcp_port = int(self.address.split(':')[-1]) # A hundred years in seconds, used when creating internal workers self.y100 = 60 * 60 * 24 * 365 * 100 # So they do not have to be looked up on each request or event self.has_info = logger.isEnabledFor(logging.INFO) self.has_debug = logger.isEnabledFor(logging.DEBUG) self.has_pool_strategy_simple = self.pool_strategy == ZMQ.POOL_STRATEGY_NAME.SINGLE self.has_service_source_zato = self.service_source == ZMQ.SERVICE_SOURCE_NAME.ZATO self.zato_service_name = config.service_name self.zato_channel = CHANNEL.ZMQ if self.has_pool_strategy_simple: self.workers_pool_initial = 1 self.workers_pool_mult = 0 self.workers_pool_max = 1 else: self.workers_pool_initial = config.workers_pool_initial self.workers_pool_mult = config.workers_pool_mult self.workers_pool_max = config.workers_pool_max # Maps service names to workers registered to handle requests to that service self.services = {} # Details about each worker, mapped by worker_id:Worker object self.workers = {} # Held upon most operations on sockets self.lock = RLock() # How often, in seconds, to send a heartbeat to workers self.heartbeat = config.heartbeat self.ctx = zmq.Context() self.socket = self.ctx.socket(zmq.ROUTER) self.socket.linger = config.linger self.poller = zmq.Poller() self.poller.register(self.socket, zmq.POLLIN) # Maps event IDs to methods that handle a given one self.handle_event_map = { const.v01.ready: self.on_event_ready, const.v01.reply_from_worker: self.on_event_reply, const.v01.heartbeat: self.on_event_heartbeat, const.v01.disconnect: self.on_event_disconnect, }
def _run(self): self.config_commands.bind('inproc://configCommands') if not self.commands_only: self.drone_command_receiver.connect('inproc://droneCommandReceiver') poller = zmq.Poller() poller.register(self.config_commands, zmq.POLLIN) while self.enabled: socks = dict(poller.poll(500)) if self.config_commands in socks and socks[self.config_commands] == zmq.POLLIN: self._handle_commands()
def collect(self): poller = zmq.Poller() poller.register(self.pull_socket, zmq.POLLIN) poller.register(self.ctrl_socket, zmq.POLLIN) pull_socket = self.pull_socket ctrl_socket = self.ctrl_socket while True: if self.have_bytecode: try: events = dict(poller.poll()) except zmq.ZMQError: self._kill() break if events.get(pull_socket) == zmq.POLLIN: command = self.pull_socket.recv(flags=zmq.SNDMORE) key = self.pull_socket.recv(flags=zmq.SNDMORE) data = self.pull_socket.recv(copy=False) if command == MAP: self.call_mapfn(key, data) elif command == REDUCE: self.call_reducefn(key, data) if events.get(ctrl_socket) == zmq.POLLIN: worker_id, command = self.ctrl_socket.recv_multipart() if command == HEARTBEAT: self.ctrl_socket.send(PONG) print 'pong' if command == DONE: self.kill() break else: self.logging.info('Waiting for server') # Associate with the server self.push_socket.send_multipart([CONNECT, self.worker_id]) # Wait for the server to route us the bytecode, # then start the work cycle worker_id, payload = self.ctrl_socket.recv_multipart() command, (mapbc, reducebc) = srl.loads(payload) assert command == BYTECODE self.set_bytecode(mapbc, reducebc) self.logging.info('Received Bytecode')
def send_data(ws): logger.info('Got a websocket connection, sending up data from zmq') socket = context.socket(zmq.SUB) socket.connect('tcp://localhost:{PORT}'.format(PORT=ZMQ_LISTENING_PORT)) socket.setsockopt(zmq.SUBSCRIBE, "") poller = zmq.Poller() poller.register(socket, zmq.POLLIN) gevent.sleep() while True: # socks = dict(poller.poll()) # if socket in socks and socks[socket] == zmq.POLLIN: data = socket.recv_json() logger.info(data) ws.send(json.dumps(data)) gevent.sleep()
def __init__(self, broker, cls, service_suffix=''): self._broker = broker self._wrapped_cls = cls self._service = bytes('{}{}'.format(cls.__name__, service_suffix), 'utf8') self._ignore_service = False self._retries = config['client_retries'] self._timeout = config['client_timeout'] self._context = zmq.Context() self._poller = zmq.Poller() self._sock = None self._connect_to_broker()