class Leatherneck(Thread): def __init__(self): super(Leatherneck, self).__init__(name="Leatherneck") self.context = Context() self.pull = self.context.socket(PULL) self.pull.connect("tcp://localhost:7000") self.push = self.context.socket(PUSH) self.push.connect("tcp://localhost:7001") self.poller = Poller() self.poller.register(self.pull, POLLIN) self._shutdown = False def cleanup(self): self.push.close() self.pull.close() self.context.term() def run(self): while True: socks = dict(self.poller.poll(timeout=1)) if socks.get(self.pull) == POLLIN: msg = self.pull.recv() msg += " WORK COMPLETE, " + str(time()) self.push.send(msg) if self._shutdown: break self.cleanup()
class Listener(Thread): def __init__(self): super(Listener, self).__init__(name="Listener") self._shutdown = False self.context = Context() self.sub = self.context.socket(SUB) self.sub.bind('tcp://*:7000') self.sub.setsockopt(SUBSCRIBE, "") self.poller = Poller() self.poller.register(self.sub, POLLIN) def cleanup(self): self.sub.close() self.context.term() def run(self): while True: socks = dict(self.poller.poll(timeout=1)) if socks.get(self.sub) == POLLIN: msg = self.sub.recv(flags=NOBLOCK) print msg if self._shutdown: break self.cleanup()
class Stethoscope(Thread): def __init__(self, context, *args, **kw): self.context = context self.recv = self.context.socket(PAIR) self.recv.connect("inproc://#1") self.pub = self.context.socket(PUB) self.pub.connect('tcp://localhost:7003') self.pub.setsockopt(HWM, 1000) self.poller = Poller() self.poller.register(self.recv, POLLIN) super(Stethoscope, self).__init__(*args, **kw) def cleanup(self): self.recv.close() self.pub.close() def run(self): try: while not shutdown.is_set(): socks = dict(self.poller.poll()) if socks.get(self.recv) == POLLIN: msg = self.recv.recv() self.pub.send(msg, flags=NOBLOCK) if msg == "DIE": raise KillThread except KillThread: print "%s exiting..." % self.name finally: self.cleanup()
def run(self, *args): """Run the listener and answer to requests. """ del args arec = AddressReceiver(max_age=self._max_age) arec.start() port = PORT try: self.listener = context.socket(REP) self.listener.bind("tcp://*:" + str(port)) poller = Poller() poller.register(self.listener, POLLIN) while self.loop: socks = dict(poller.poll(1000)) if socks: if socks.get(self.listener) == POLLIN: msg = self.listener.recv() else: continue logger.debug("Replying to request: " + str(msg)) msg = Message.decode(msg) self.listener.send_unicode(str(get_active_address( msg.data["service"], arec))) except KeyboardInterrupt: # Needed to stop the nameserver. pass finally: arec.stop() self.listener.close()
def get_pub_address(name, timeout=10, nameserver="localhost"): """Get the address of the publisher for a given publisher *name* from the nameserver on *nameserver* (localhost by default). """ # Socket to talk to server socket = context.socket(REQ) try: socket.setsockopt(LINGER, timeout * 1000) socket.connect("tcp://" + nameserver + ":" + str(PORT)) poller = Poller() poller.register(socket, POLLIN) message = Message("/oper/ns", "request", {"service": name}) socket.send(str(message)) # Get the reply. sock = poller.poll(timeout=timeout * 1000) if sock: if sock[0][0] == socket: message = Message.decode(socket.recv(NOBLOCK)) return message.data else: raise TimeoutError("Didn't get an address after %d seconds." % timeout) finally: socket.close()
def get_pub_address(name, timeout=10, nameserver="localhost"): """Get the address of the publisher for a given publisher *name* from the nameserver on *nameserver* (localhost by default). """ # Socket to talk to server socket = get_context().socket(REQ) try: socket.setsockopt(LINGER, timeout * 1000) socket.connect("tcp://" + nameserver + ":" + str(PORT)) logger.debug('Connecting to %s', "tcp://" + nameserver + ":" + str(PORT)) poller = Poller() poller.register(socket, POLLIN) message = Message("/oper/ns", "request", {"service": name}) socket.send_string(six.text_type(message)) # Get the reply. sock = poller.poll(timeout=timeout * 1000) if sock: if sock[0][0] == socket: message = Message.decode(socket.recv_string(NOBLOCK)) return message.data else: raise TimeoutError("Didn't get an address after %d seconds." % timeout) finally: socket.close()
class Leatherneck(Thread): def __init__(self): super(Leatherneck, self).__init__(name="Leatherneck") self.context = Context() self.pull = self.context.socket(PULL) self.pull.connect("tcp://localhost:7000") self.push = self.context.socket(PUSH) self.push.connect("tcp://localhost:7001") self.poller = Poller() self.poller.register(self.pull, POLLIN) self._shutdown = False for th in t_enum(): if th.name == "MainThread": self.mainthread = th def cleanup(self): print "Workers exiting..." self.push.close() self.pull.close() self.context.term() def run(self): while True: if not self.mainthread.is_alive(): self._shutdown = True break socks = dict(self.poller.poll(timeout=1)) if socks.get(self.pull) == POLLIN: msg = self.pull.recv(flags=NOBLOCK) msg += " WORK COMPLETE, " + str(time()) self.push.send(msg, flags=NOBLOCK) if self._shutdown: break self.cleanup()
class HomeBase(Thread): def __init__(self): super(HomeBase, self).__init__(name="HomeBase") self.context = Context() self.pull = self.context.socket(PULL) self.pull.bind("tcp://*:7001") self._shutdown = False self.poller = Poller() self.poller.register(self.pull, POLLIN) for th in t_enum(): if th.name == "MainThread": self.mainthread = th def cleanup(self): print "Home exiting..." self.pull.close() self.context.term() def run(self): while True: if not self.mainthread.is_alive(): self._shutdown = True break socks = dict(self.poller.poll(timeout=1)) if socks.get(self.pull) == POLLIN: msg = self.pull.recv(flags=NOBLOCK) msg += ", WORK RECEIVED " print msg if self._shutdown: break self.cleanup()
class ControllerServer: def __init__(self): self.pairSockets = [] self.poller = Poller() def connect(self, address): socket = zmq.Context().socket(zmq.PAIR) socket.connect(address) self.poller.register(socket, zmq.POLLIN) self.pairSockets.append(socket) return socket def send(self, message, socket=None): message = IO.wrap(message) if socket: socket.send(message) else: for socket in self.pairSockets: socket.send(message) def sendAction(self, action, socket=None): message = {'action': action} self.send(message, socket) def receive(self, receiveCb, timeout=-1): socks = dict(self.poller.poll(timeout)) if socks: for si, socket in enumerate(self.pairSockets): if self.pairSockets[si] in socks and socks[ self.pairSockets[si]] == zmq.POLLIN: msg = socket.recv() msg, _ = IO.unwrap(msg) receiveCb(si, msg)
def _send_request(req_id, action, key, value=None): """ Generate a request ID, push the request to the node and wait for the result, filtering by the ID on the subscription. This should be a request/response call to the node, but since many must be possible at the same time, a PUSH/PULL AND PUB/SUB with an unique ID for each request is used instead. :return: the node response :rtype: [] """ # Create and connect the sockets. context = current_app.config["context"] subscriber = context.socket(SUB) subscriber.setsockopt_string(SUBSCRIBE, req_id) subscriber.connect(SUB_ENDPOINT) request = context.socket(PUSH) request.setsockopt(SNDTIMEO, _TIMEOUT) request.connect(PUSH_ENDPOINT) # Push the request. request.send_json([req_id, action, key, value]) # Wait for the response from the publisher. poller = Poller() poller.register(subscriber, POLLIN) sockets = dict(poller.poll(_TIMEOUT)) if subscriber not in sockets: # no response, time out raise InternalServerError() # Return the response, without the unique request ID return subscriber.recv_multipart()[1:]
class HomeBase(Thread): def __init__(self): super(HomeBase, self).__init__(name="HomeBase") self.context = Context() self.pull = self.context.socket(PULL) self.pull.bind("tcp://*:7001") self._shutdown = False self.poller = Poller() self.poller.register(self.pull, POLLIN) def cleanup(self): self.pull.close() self.context.term() def run(self): while True: socks = dict(self.poller.poll(timeout=1)) if socks.get(self.pull) == POLLIN: msg = self.pull.recv() msg += ", WORK RECEIVED " print msg if self._shutdown: break self.cleanup()
class MirrorWatcher(Thread): """Watches a other server. """ def __init__(self, holder, host, pubport, reqport, sched): Thread.__init__(self) self._holder = holder self._pubaddress = "tcp://" + host + ":" + str(pubport) self._reqaddress = "tcp://" + host + ":" + str(reqport) self._req = SimpleRequester(host, reqport) self._subsocket = context.socket(SUB) self._subsocket.connect(self._pubaddress) self._subsocket.setsockopt(SUBSCRIBE, "pytroll") self._poller = Poller() self._poller.register(self._subsocket, POLLIN) self._lock = Lock() self._loop = True self._sched = sched def run(self): last_hb = datetime.now() minutes = 2 while self._loop: if datetime.now() - last_hb > timedelta(minutes=minutes): logger.error("No heartbeat from " + str(self._pubaddress)) last_hb = datetime.now() minutes = 1440 socks = dict(self._poller.poll(2000)) if (socks and self._subsocket in socks and socks[self._subsocket] == POLLIN): message = Message.decode(self._subsocket.recv()) else: continue if message.type == "have": sat = message.data["satellite"] key = strp_isoformat(message.data["timecode"]) elevation = message.data["elevation"] quality = message.data.get("quality", 100) data = _MirrorGetter(self._req, sat, key) self._holder.add(sat, key, elevation, quality, data) if message.type == "heartbeat": logger.debug("Got heartbeat from " + str(self._pubaddress) + ": " + str(message)) self._sched.mirror_next_pass = message.data["next_pass"] last_hb = datetime.now() minutes = 2 def stop(self): """Stop the watcher """ self._loop = False self._req.stop() self._subsocket.setsockopt(LINGER, 0) self._subsocket.close()
def start_poll(pull, push, sub, hash_checker): global INPUT_DATA seq_no = 0 poller = Poller() poller.register(pull, zmq.POLLIN) poller.register(sub, zmq.POLLIN) while True: try: socks = dict(poller.poll()) except KeyboardInterrupt: return if sub in socks: log.debug('recv -> {}'.format(sub.recv_multipart())) try: log.info('Concatting...') concat_string = process() concat_string_hash = HASHER(concat_string).digest() # Send final hash push.send_multipart([ MessageType.INPUT.value, seq_no.to_bytes(2, byteorder='big'), concat_string_hash, ]) log.debug('send -> {} | {} | {}', MessageType.INPUT.name, seq_no, concat_string_hash.hex()) # Send original concatted string push.send_multipart([ MessageType.COMMIT.value, seq_no.to_bytes(2, byteorder='big'), concat_string, ]) log.debug('send -> {} | {} | {}...', MessageType.COMMIT.name, seq_no, concat_string[:20]) INPUT_DATA.clear() log.info("Reset concat") if seq_no == 0xFFFF: seq_no = 0 else: seq_no += 1 except Exception as e: log.error("Could not concat inputs -> {}".format(e)) if pull in socks: inp = pull.recv() log.debug('recv -> {}'.format(inp.hex())) try: hash_checker.check(inp) except: log.warn('Bad hash received! -> {}', inp) continue INPUT_DATA.append(inp) log.info('Input added | Elements: {}'.format(len(INPUT_DATA)))
class Requester: def __init__(self, timeout=-1, poolSize=4): self.timeout = timeout self.socket = None self.numConnections = 0 self.poller = Poller() def addServer(self, address, bind=False, linger=-1): if self.socket is None: self.socket = zmq.Context().socket(zmq.REQ) # self.socket.setsockopt(zmq.LINGER, linger) self.poller.register(self.socket, zmq.POLLIN) if bind: self.socket.bind(address) else: self.socket.connect(address) self.numConnections += 1 def numConnections(self): return self.numConnections def request(self, message): if message is None or self.socket is None: return None message = IO.wrap(message) self.socket.send(message) socks = dict(self.poller.poll(self.timeout)) if socks: replyMessage = self.socket.recv() reply, _ = IO.unwrap(replyMessage) return reply return None def requestAllServers(self, message): replies = [] for n in range(self.numConnections): reply = self.request(message) replies.append(reply) return replies def requestAllServersCb(self, message, replyCb): for n in range(self.numConnections): reply = self.request(message) replyCb(reply) def requestAllServersPool(self, message, poolSize=0): if poolSize < 1: poolSize = self.numConnections pool = Pool(processes=poolSize) replies = pool.map(self.request, [message for n in range(self.numConnections)]) return replies
def refresh_serum_index(self): try: rawData = [] for i in range(3): socket = self.context.socket(REQ) socket.connect("tcp://serum-server:6900") socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) socket.send(dumps({"endpoint": "list"})) responses = poller.poll(10000) if len(responses) != 0: response = socket.recv() socket.close() rawData = loads(response) break else: socket.close() for market in rawData["markets"]: base, quote = market["name"].split("/", 1) if base not in self.serumIndex: self.serumIndex[base] = [] self.serumIndex[base].append({"id": market["address"], "name": base, "base": base, "quote": quote, "image": None, "program": market["programId"]}) for token in rawData["tokenList"]: symbol = token["symbol"].upper() if symbol not in self.serumIndex: self.serumIndex[symbol] = [] processed = [] for market in self.serumIndex[symbol]: processed.append(market["quote"]) market["name"] = token["name"] market["image"] = token.get("logoURI") for extension, address in token.get("extensions", {}).items(): if extension.startswith("serumV3"): quote = extension.removeprefix("serumV3").upper() if quote not in processed: processed.append(quote) self.serumIndex[symbol].append({"id": address, "name": token["name"], "base": symbol, "quote": quote, "image": token.get("logoURI"), "program": "9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin"}) if len(self.serumIndex[symbol]) == 0: self.serumIndex.pop(symbol) elif self.serumIndex[symbol][0]["quote"] != "USDC": usdcMarket = None for index, market in enumerate(self.serumIndex[symbol]): if market["quote"] == "USDC": usdcMarket = self.serumIndex[symbol].pop(index) break if usdcMarket is not None: self.serumIndex[symbol].insert(0, usdcMarket) except Exception: print(format_exc())
class MirrorWatcher(Thread): """Watches a other server. """ def __init__(self, holder, host, pubport, reqport, sched): Thread.__init__(self) self._holder = holder self._pubaddress = "tcp://" + host + ":" + str(pubport) self._reqaddress = "tcp://" + host + ":" + str(reqport) self._req = SimpleRequester(host, reqport) self._subsocket = get_context().socket(SUB) self._subsocket.connect(self._pubaddress) self._subsocket.setsockopt_string(SUBSCRIBE, "pytroll") self._poller = Poller() self._poller.register(self._subsocket, POLLIN) self._lock = Lock() self._loop = True self._sched = sched def run(self): last_hb = datetime.now() minutes = 2 while self._loop: if datetime.now() - last_hb > timedelta(minutes=minutes): logger.error("No heartbeat from " + str(self._pubaddress)) last_hb = datetime.now() minutes = 1440 socks = dict(self._poller.poll(2000)) if (socks and self._subsocket in socks and socks[self._subsocket] == POLLIN): message = Message.decode(self._subsocket.recv()) else: continue if message.type == "have": sat = message.data["satellite"] key = strp_isoformat(message.data["timecode"]) elevation = message.data["elevation"] quality = message.data.get("quality", 100) data = _MirrorGetter(self._req, sat, key) self._holder.add(sat, key, elevation, quality, data) if message.type == "heartbeat": logger.debug("Got heartbeat from " + str(self._pubaddress) + ": " + str(message)) self._sched.mirror_next_pass = message.data["next_pass"] last_hb = datetime.now() minutes = 2 def stop(self): """Stop the watcher """ self._loop = False self._req.stop() self._subsocket.setsockopt(LINGER, 0) self._subsocket.close()
class ZMQPoller(object): """A poller that can be used in the tornado IOLoop. This simply wraps a regular zmq.Poller, scaling the timeout by 1000, so that it is in seconds rather than milliseconds. """ def __init__(self): self._poller = Poller() @staticmethod def _map_events(events): """translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR""" z_events = 0 if events & IOLoop.READ: z_events |= POLLIN if events & IOLoop.WRITE: z_events |= POLLOUT if events & IOLoop.ERROR: z_events |= POLLERR return z_events @staticmethod def _remap_events(z_events): """translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR""" events = 0 if z_events & POLLIN: events |= IOLoop.READ if z_events & POLLOUT: events |= IOLoop.WRITE if z_events & POLLERR: events |= IOLoop.ERROR return events def register(self, fd, events): return self._poller.register(fd, self._map_events(events)) def modify(self, fd, events): return self._poller.modify(fd, self._map_events(events)) def unregister(self, fd): return self._poller.unregister(fd) def poll(self, timeout): """poll in seconds rather than milliseconds. Event masks will be IOLoop.READ/WRITE/ERROR """ z_events = self._poller.poll(1000*timeout) return [ (fd,self._remap_events(evt)) for (fd,evt) in z_events ] def close(self): pass
def __stop(self): """ try to stop all of this Role's services """ # send commands poller = Poller() for (pipe, svc) in self.__services.items(): pipe.send_string('STOP') self.logger.debug('sent STOP command to %s service' % svc) poller.register(pipe, POLLIN) # give services a few seconds to cleanup and exit before checking responses sleep(1) max_attempts = len(self.__services) attempts = 0 while self.__some_alive() and attempts < max_attempts: attempts += 1 # poll for any replies items = dict(poller.poll(60000)) # wait for messages # mark responding services as stopped alive = dict(self.__services) # make copy for (pipe, svc) in alive.items(): if pipe in items: reply = pipe.recv_string() if 'STOPPED' == reply: self.logger.debug('received STOPPED control reply from %s service' % svc) svc.join(timeout=5) # STOPPED response should be sent right before svc exit if svc.is_alive(): self.logger.error('%s service is still alive; not waiting' % svc) else: self.logger.debug('%s service thread stopped' % svc) poller.unregister(pipe) pipe.close() del (self.__services[pipe]) else: self.logger.debug('unknown control reply: %s' % reply) # log some useful info if len(self.__services) > 0: msg = '%s services still alive after %d cycles; ' % ( [str(s) for s in self.__services.values()], attempts) if attempts < max_attempts: msg += 'waiting' else: msg += 'giving up' self.logger.debug(msg)
def _start_workers(context, sync_socket, count, timeout=None): """ :type context: zmq.Context :type sync_socker: zmq.Socket :type count: int :param timeout: Timeout for waiting for worker messages, in milliseconds. :type timeout: float """ _LOG.debug("Starting workers...") worker_threads = [ Thread( target=_handle_requests, name="worker {}".format(i), args=(context, )) for i in range(count)] for thread in worker_threads: thread.start() _LOG.debug("Synchronizing workers...") poller = Poller() poller.register(sync_socket, POLLIN) worker_identities = [] for _ in worker_threads: sockets = dict(poller.poll(timeout=timeout)) if sync_socket in sockets: worker_identities.append(sync_socket.recv_multipart()[0]) else: raise RuntimeError("Worker did not respond in time.") for worker_identity in worker_identities: sync_socket.send_multipart([worker_identity, b"", b""]) for _ in worker_identities: sockets = dict(poller.poll(timeout=timeout)) if sync_socket in sockets: sync_socket.recv_multipart() else: raise RuntimeError("Worker did not respond in time.") _LOG.debug("Workers synchronized.") return worker_threads, worker_identities
def listen(self, topics: Iterable[str] = ("log|", )): """ Listens to incoming log messages on the ZeroMQ socket(s). Blocks until `Ctrl + C` is pressed, or ``SIGINT`` or ``SIGTERM`` is received by the process. Returns immediately if either a bind address nor at least one connect address is set. :param topics: The topic to subscribe with. Can be used for selecting log messages from specific source. Defaults to ``["log|"]``. :type topics: Iterable[str] """ if not self._bind_address and not self._connect_addresses: return poller = Poller() if self._bind_address: self._pull_socket = self._ctx.socket(PULL) self._pull_socket.bind('tcp://' + self._bind_address) poller.register(self._pull_socket, POLLIN) if self._connect_addresses: self._sub_socket = self._ctx.socket(SUB) for topic in topics: self._sub_socket.setsockopt( SUBSCRIBE, topic.encode(encoding=_TOPIC_ENCODING)) for address in self._connect_addresses: self._sub_socket.connect('tcp://' + address) poller.register(self._sub_socket, POLLIN) try: while not is_stopped(): events = dict(poller.poll(self._interval * 1000)) if self._pull_socket in events \ and events[self._pull_socket] == POLLIN: data = self._pull_socket.recv_multipart(flags=NOBLOCK) self._handle_message(data) if self._sub_socket in events \ and events[self._sub_socket] == POLLIN: data = self._sub_socket.recv_multipart(flags=NOBLOCK) self._handle_message(data) finally: if self._pull_socket: self._pull_socket.close() self._pull_socket = None if self._sub_socket: self._sub_socket.close() self._sub_socket = None
def _request_quote(cls, request, ticker, context=None): try: socket = context.socket(REQ) socket.connect("tcp://serum-server:6900") socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) socket.send(bytes(dumps({"endpoint": "quote", "marketAddress": ticker.get("id"), "program": ticker.get("symbol")}), encoding='utf8')) responses = poller.poll(5000) if len(responses) != 0: response = socket.recv() socket.close() rawData = loads(response) else: socket.close() return [{}, ""] except: return [{}, ""] coinThumbnail = static_storage.icon if ticker.get("image") is None else ticker.get("image") price = [float(rawData["price"])] priceChange = 0 volume = 0 tokenImage = static_storage.icon base = "USD" if ticker.get("base") in AbstractProvider.stableCoinTickers else ticker.get("base") quote = "USD" if ticker.get("quote") in AbstractProvider.stableCoinTickers else ticker.get("quote") payload = { "quotePrice": "{:,.10f}".format(price[0]).rstrip('0').rstrip('.') + " " + quote, "quoteVolume": "{:,.4f}".format(volume).rstrip('0').rstrip('.') + " " + base, "title": ticker.get("name"), "thumbnailUrl": coinThumbnail, "messageColor": "amber" if priceChange == 0 else ("green" if priceChange > 0 else "red"), "sourceText": "Data from Serum DEX", "platform": "Serum", "raw": { "quotePrice": [price[0]], "quoteVolume": [volume], "timestamp": time() } } return [payload, ""]
def _request_translation(self, language, country, key, plural): """ Start up a worker, sync it and then send it a translation request. Returns the result, shuts down the worker at the end as well. Fails the current test, if something goes wrong. """ request = [ language, country if country is not None else "", key, str(plural) if plural is not None else ""] request = [x.encode(_ENCODING) for x in request] context = Context() # Create synchronization and backend sockets. try: sync_socket = context.socket(ROUTER) try: sync_socket.bind(_SYNC_ENDPOINT) backend = context.socket(DEALER) try: backend.bind(_REQUEST_ENDPOINT) worker_threads, worker_identities = _start_workers( context, sync_socket, 1, _TIMEOUT) poller = Poller() poller.register(backend, POLLIN) poller.register(sync_socket, POLLIN) # Send request. backend.send_multipart( [worker_identities[0], b""] + request) sockets = dict(poller.poll(_TIMEOUT)) # Shutdown worker. _shut_down_workers( sync_socket, worker_threads, worker_identities, _TIMEOUT / 1000.0) if backend in sockets: # Return translation. return backend.recv_multipart()[2].decode("utf-8") self.fail("Worker did not response the request in time.") finally: backend.set(LINGER, 0) backend.close() finally: sync_socket.set(LINGER, 0) sync_socket.close() finally: context.destroy(linger=0)
class Server(object): def __init__(self, port): self._ctx = Context() self._socket = self._ctx.socket(DEALER) self._socket.bind('tcp://*:%s' % port) self._poller = Poller() self._poller.register(self._socket, POLLIN) self._clients = {} self._actions = {'register': self._register_client} def _register_client(self, message): if message['pseudonym'] not in self._clients: self._clients[message['pseudonym']] = message['cert'] message = {'type': 'Accepted'} self._send_msg(message) else: message = {'type': 'Failed'} self._send_msg(message) def _send_msg(self, message): pickled_message = dumps(message) self._socket.send(pickled_message) print('Sent --> ', message) def _recv_msg(self): pickled_message = self._socket.recv() received_message = loads(pickled_message) print("Received --> ", received_message) return received_message def execute(self): while True: sockets = dict(self._poller.poll()) if sockets.get(self._socket) == POLLIN: received_message = self._recv_msg() self._actions[received_message['type']](received_message)
def _server(self, response): """ Wait for a client request, record it and send the response. """ context = Context() try: socket = context.socket(REP) try: socket.set(LINGER, 0) socket.bind("tcp://*:{}".format(_PORT)) poller = Poller() poller.register(socket, POLLIN) sockets = dict(poller.poll(_TIMEOUT)) if socket in sockets: self._client_request = socket.recv_multipart() if response: socket.send_multipart(response) finally: socket.close() finally: context.destroy(linger=0)
def execute_parser_request(endpoint, parameters, timeout=5): socket = TickerParser.zmqContext.socket(REQ) payload, responseText = None, None socket.connect("tcp://parser:6900") socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) socket.send_multipart([endpoint] + parameters) responses = poller.poll(timeout * 1000) if len(responses) != 0: response = socket.recv_multipart() socket.close() return response else: socket.close() raise Exception("time out") return None
def execute_database_request(endpoint, parameters, timeout=1): socket = DatabaseConnector.zmqContext.socket(REQ) payload, responseText = None, None socket.connect("tcp://database:6900") socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) socket.send_multipart([ endpoint, bytes(str(int((time() + timeout) * 1000)), encoding='utf8'), parameters ]) responses = poller.poll(timeout * 1000) if len(responses) != 0: [response] = socket.recv_multipart() socket.close() return loads(response) else: socket.close() return None
def run(self, *args): """Run the listener and answer to requests. """ del args arec = AddressReceiver( max_age=self._max_age, multicast_enabled=self._multicast_enabled, restrict_to_localhost=self._restrict_to_localhost) arec.start() port = PORT try: with nslock: self.listener = get_context().socket(REP) self.listener.bind("tcp://*:" + str(port)) logger.debug('Listening on port %s', str(port)) poller = Poller() poller.register(self.listener, POLLIN) while self.loop: with nslock: socks = dict(poller.poll(1000)) if socks: if socks.get(self.listener) == POLLIN: msg = self.listener.recv_string() else: continue logger.debug("Replying to request: " + str(msg)) msg = Message.decode(msg) active_address = get_active_address( msg.data["service"], arec) self.listener.send_unicode(six.text_type(active_address)) except KeyboardInterrupt: # Needed to stop the nameserver. pass finally: arec.stop() self.stop()
def directory_server(store: DirectoryServerStore, zmq_context: Context): # pylint: disable=no-member # These zmq.ROUTER and zmq.PUB must be actually exists print("Starting on libzmq {} with PyZMQ {}".format(zmq.zmq_version(), zmq.pyzmq_version())) entrypoint: Socket = zmq_context.socket(zmq.ROUTER) entrypoint.bind("tcp://127.0.0.1:5350") # This is just a PROTOTYPE! pub_file_changes: Socket = zmq_context.socket(zmq.PUB) pub_file_changes.bind("tcp://127.0.0.1:5351") poller = Poller() poller.register(entrypoint, flags=zmq.POLLIN) print( "Directory server is started on 127.0.0.1:5350 (commands) and 127.0.0.1:5351 (file_changes_push)" ) while True: events: List[Tuple[Socket, int]] = poller.poll() for socket, _ in events: frames: List[Frame] = socket.recv_multipart(copy=False) id_frame: Frame = frames.pop(0) empty_frame: Frame = frames.pop(0) assert (len(empty_frame.bytes) == 0) command_frame: Frame = frames.pop(0) command = str(command_frame.bytes, encoding='utf8') if command == 'ping': ping_handler(store, socket, frames, id_frame) elif command == 'device.cast_address': casting_address_handler(store, socket, frames, id_frame) elif command == 'device.get_addresses': get_addresses_handler(store, socket, frames, id_frame) elif command == 'fs.list': file_list_handler(store, socket, id_frame) elif command == 'fs.declare': file_declare_handler(store, socket, frames, id_frame, pub_file_changes) elif command == 'fs.disown': file_disown_handler(store, socket, frames, id_frame, pub_file_changes) elif command == 'fs.get': file_get_handler(store, socket, frames, id_frame)
def start_poll(self, start_seq_no=None): seq_no = start_seq_no if start_seq_no is not None else 0 last_process = time.time() poller = Poller() poller.register(self.pull, zmq.POLLIN) poller.register(self.router, zmq.POLLIN) while True: try: socks = dict(poller.poll(timeout=self.gather_time * 100)) except KeyboardInterrupt: return if self.pull in socks: self.recv_input() if self.router in socks: self.recv_from_comp() if self.worker_deque and self.ready_to_process() and time.time() - last_process >= self.gather_time: compute_id = self.worker_deque.popleft() last_process = time.time() try: self.process() self.send_to_comp(compute_id, seq_no) self.reset_inputs() if seq_no == 0xFFFF: seq_no = 0 else: seq_no += 1 except Exception as e: log.error("Could not process send and reset inputs -> {}".format(e)) else: log.debug('time since last {}, deque size {}', time.time() - last_process, len(self.worker_deque))
def _handle_requests(context): """ This is supposed to run as a background thread. It listens for translation requests and answers them until a message is arrived on the sync socket, whereupon the function (thread) ends. :type context: zmq.Context """ sync_socket = context.socket(REQ) sync_socket.connect(_SYNC_ENDPOINT) requests_socket = context.socket(REP) requests_socket.connect(_REQUEST_ENDPOINT) _LOG.debug("Synchronizing worker") sync_socket.send(b"") sync_socket.recv() sync_socket.send(b"") keep_running = True poller = Poller() poller.register(requests_socket, POLLIN) poller.register(sync_socket, POLLIN) _LOG.debug("Running worker") while keep_running: sockets = dict(poller.poll()) if requests_socket in sockets: try: response = _handle_request(requests_socket.recv_multipart()) except Exception: # pylint: disable=broad-except _LOG.critical("Handler crashed!", exc_info=True) response = [b""] requests_socket.send_multipart(response) if sync_socket in sockets: sync_socket.recv() keep_running = False _LOG.debug("Terminating worker")
class Distributor(Thread): def __init__(self, query): Thread.__init__(self) self._pool = ThreadPool(processes=4) self._workers = HashRing(WORKER_SERVERS) self._identity = bytes(uuid.uuid4()) self._query = query self._init_sock() def _log(self, text): if LOG_DISTRIBUTOR: log_debug(self, text) def _init_sock(self): self._context = Context(1) self._poller = Poller() self._set_sock() def _set_sock(self): self._socket = self._context.socket(DEALER) self._socket.setsockopt(IDENTITY, self._identity) self._poller.register(self._socket, POLLIN) self._socket.connect(zmqaddr(self._get_broker(), BROKER_PORT)) self._socket.send(PPP_READY) def _reset_sock(self): self._poller.unregister(self._socket) self._socket.setsockopt(LINGER, 0) self._socket.close() self._set_sock() def _get_broker(self): length = len(BROKER_SERVERS) return BROKER_SERVERS[randint(0, length - 1)] def _get_worker(self, uid): return self._workers.get_node(uid) def _get_user(self, buf): if len(buf) < USERNAME_SIZE: log_err(self, 'failed to get user, invalid length') raise Exception(log_get(self, 'failed to get user')) name = filter(lambda x:x != '*', buf[:USERNAME_SIZE]) if not name: log_err(self, 'failed to get user') raise Exception(log_get(self, 'failed to get user')) return name def _get_token(self, buf): if len(buf) < UID_SIZE: log_err(self, 'failed to get token, invalid length') raise Exception(log_get(self, 'failed to get token')) uid = None token = None if buf[UID_SIZE - 1] == '*': user = self._get_user(buf) uid, token = self._query.user.get(user, 'uid', 'password') else: uid = buf[0:UID_SIZE] token = self._query.token.get(uid) if uid and token: return (uid, token) else: log_err(self, 'failed to get token') raise Exception(log_get(self, 'failed to get token')) def _reply(self, identity, seq, buf): msg = [identity, '', seq, buf] self._socket.send_multipart(msg) def _request(self, addr, **args): sock = io.connect(addr, WORKER_PORT) try: buf = bson.dumps(args) io.send_pkt(sock, buf) res = io.recv_pkt(sock) return unicode2str(bson.loads(res)['result']) finally: io.close(sock) def _proc(self, identity, seq, buf): uid, token = self._get_token(buf) if not uid or not token: log_err(self, 'failed to process, cannot get token') return addr = self._get_worker(uid) ret = self._request(addr, uid=uid, token=token, buf=buf) self._reply(identity, seq, ret) def _proc_safe(self, identity, seq, buf): try: self._proc(identity, seq, buf) except: log_err(self, 'failed to process') def _handler(self, identity, seq, buf): if DEBUG and not SAFE: self._proc(identity, seq, buf) else: self._proc_safe(identity, seq, buf) def run(self): self._log('start ...') liveness = PPP_HEARTBEAT_LIVENESS timeout = time.time() + PPP_HEARTBEAT_INTERVAL while True: sockets = dict(self._poller.poll(PPP_HEARTBEAT_INTERVAL * 1000)) if sockets.get(self._socket) == POLLIN: frames = self._socket.recv_multipart() if not frames: log_err(self, 'invalid request') break if len(frames) == PPP_NR_FRAMES: self._pool.apply_async(self._proc, args=(frames[PPP_FRAME_ID], frames[PPP_FRAME_SEQ], frames[PPP_FRAME_BUF])) elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT: liveness = PPP_HEARTBEAT_LIVENESS else: log_err(self, "invalid request") else: liveness -= 1 if liveness == 0: time.sleep(random.randint(SLEEP_TIME / 2, SLEEP_TIME)) self._reset_sock() liveness = PPP_HEARTBEAT_LIVENESS if time.time() > timeout: timeout = time.time() + PPP_HEARTBEAT_INTERVAL self._socket.send(PPP_HEARTBEAT)
class JobContext(object): def __init__(self, job_id): self.job_id = job_id self.connections_by_name = {} self.connections_by_socket = {} self.poller = Poller() self.num_pending_reqs = 0 def send_req(self, worker_node, msg, callback): """Send a message and register callback for response. Message should be unserialized json """ node_name = worker_node["name"] if self.connections_by_name.has_key(node_name): conn = self.connections_by_name[node_name] assert conn.port==worker_node["worker_port"] assert conn.contact_address==worker_node["contact_address"] else: conn = Connection(node_name, worker_node["contact_address"], worker_node["worker_port"]) conn.connect() self.connections_by_name[node_name] = conn self.connections_by_socket[conn.socket] = conn self.poller.register(conn.socket, flags=POLLIN) self.num_pending_reqs += 1 conn.send(json.dumps(msg), callback) if msg.has_key("mtype"): mtype = msg["mtype"] else: mtype = "unknown" logger.debug("Sent message of type %s to %s" % (mtype, worker_node["name"])) def poll(self, timeout_in_ms): events = self.poller.poll(timeout_in_ms) if len(events)==0: return 0 for (socket, num) in events: if not self.connections_by_socket.has_key(socket): raise ZmqServerError("Poll: Could not find registered socket for socket %s" % socket) conn = self.connections_by_socket[socket] if not conn.callback: raise ZmqServerError("Received event for connection %s when no request pending" % conn.node_name) msg = socket.recv() logger.debug("message received from worker %s" % conn.node_name) callback = conn.callback # We clear the callback field of the connection just before we call # the callback. This lets us disconnect within the callback, # if we want to (e.g. after receiving a stop ack). conn.callback = None callback(msg) self.num_pending_reqs = self.num_pending_reqs - 1 return len(events) def disconnect(self, node_name): if not self.connections_by_name.has_key(node_name): raise ZmqServerError("Attempting to disconnect from %s but no conection found" % node_name) conn = self.connections_by_name[node_name] assert conn.callback==None del self.connections_by_name[node_name] del self.connections_by_socket[conn.socket] self.poller.unregister(conn.socket) conn.socket.close() def stop_job(self): node_names = [n for n in self.connections_by_name.keys()] for node_name in node_names: self.disconnect(node_name)
class Requester(object): """Make a request connection, waiting to get scanlines . """ def __init__(self, host, port): self._host = host self._port = port self._context = Context() self._socket = self._context.socket(REQ) self._socket.setsockopt(LINGER, 1) self._socket.connect("tcp://"+host+":"+str(port)) self._poller = Poller() self._poller.register(self._socket, POLLIN) def stop(self): """Close the socket. """ self._socket.close() def __del__(self, *args, **kwargs): self.stop() def send(self, msg): """Send a message. """ return self._socket.send(str(msg)) def recv(self, timeout=None): """Receive a message. *timeout* in ms. """ if self._poller.poll(timeout): return Message(rawstr=self._socket.recv()) else: raise IOError("Timeout from " + str(self._host) + ":" + str(self._port)) def get_line(self, satellite, utctime): """Get the scanline of *satellite* at *utctime*. """ msg = Message('/oper/polar/direct_readout/norrköping', 'request', {"type": "scanline", "satellite": satellite, "utctime": utctime.isoformat()}) self.send(msg) return self.recv(1000).data def get_slice(self, satellite, start_time, end_time): """Get a slice of scanlines. """ msg = Message('/oper/polar/direct_readout/norrköping', 'request', {"type": 'scanlines', "satellite": satellite, "start_time": start_time.isoformat(), "end_time": end_time.isoformat()}) self.send(msg) return self.recv(1000).data def send_lineinfo(self, sat, utctime, elevation, filename, pos): """Send information to our own server. """ msg = Message('/oper/polar/direct_readout/norrköping', 'notice', {"type": 'scanline', "satellite": sat, "utctime": utctime.isoformat(), "elevation": elevation, "filename": filename, "file_position": pos}) self.send(msg) self._socket.recv()
if __name__ == '__main__': context = Context() socket = context.socket(PUB) socket.bind('tcp://*:6667') # socket to receive control messages on controller = context.socket(SUB) controller.connect('tcp://localhost:6670') controller.setsockopt(SUBSCRIBE, "") # poller for handling receiver and controller messages poller = Poller() poller.register(socket, POLLIN) poller.register(controller, POLLIN) print 'Weather Update Server started ...' while True: socks = dict(poller.poll(timeout=1)) # exit the program if the kill command is executed if socks.get(controller) == POLLIN: controller.recv() print 'Got kill command' break # if not killed, then tranmit the next weather update zipcode = random.choice(zip_list) temperature = random.randrange(1, 215) - 80 relhumidity = random.randrange(1, 50) + 10 socket.send('%s %d %d' % (zipcode, temperature, relhumidity))
class Subscriber(object): def __init__(self, addresses, translate=False): self._addresses = addresses self._translate = translate self.subscribers = [] self._poller = Poller() for addr in self._addresses: subscriber = context.socket(SUB) subscriber.setsockopt(SUBSCRIBE, "pytroll") subscriber.connect(addr) self.subscribers.append(subscriber) self._poller.register(subscriber) self._lock = Lock() self._loop = True @property def sub_addr(self): return dict(zip(self.subscribers, self._addresses)) @property def addr_sub(self): return dict(zip(self._addresses, self.subscribers)) def reset(self, addr): with self._lock: idx = self._addresses.index(addr) self._poller.unregister(self.subscribers[idx]) self.subscribers[idx].setsockopt(LINGER, 0) self.subscribers[idx].close() self.subscribers[idx] = context.socket(SUB) self.subscribers[idx].setsockopt(SUBSCRIBE, "pytroll") self.subscribers[idx].connect(addr) self._poller.register(self.subscribers[idx], POLLIN) def recv(self, timeout=None): """Receive a message, timeout in seconds. """ if timeout: timeout *= 1000 while self._loop: with self._lock: if not self._loop: break subs = dict(self._poller.poll(timeout=timeout)) if subs: for sub in self.subscribers: if sub in subs and subs[sub] == POLLIN: msg = Message.decode(sub.recv()) if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] msg.sender = msg.sender.split("@")[0] + "@" + host yield msg else: yield None def stop(self): """Stop the subscriber """ with self._lock: self._loop = False for sub in self.subscribers: self._poller.unregister(sub) sub.setsockopt(LINGER, 0) sub.close()
class CollectorSOS(RecoveryThread): def __init__(self, ctx, ep, uuid, config_svc): RecoveryThread.__init__(self, ctx, ep, uuid) self.cfgsvc = config_svc # three sockets to send and receive election messages; it's awkward, but election PUBs # are received on the Node's SUB socket and passed to us via _get_from_queue() self.control_out = None self.control_in = None self.pub = None self.poller = Poller() # need separate endpoint for recovery activity self.recovery_ep = None # { election-uuid : Election } self.elections = {} self.last_message_time = None self.elected_leader = None def __init_sockets(self): # we send election commands (in response to a SUB'ed message) on this socket self.control_out = self.ctx.socket(DEALER) # we receive election commands (in response to a PUB'ed message) on this socket self.control_in = self.ctx.socket(ROUTER) bind_addr = self.control_in.bind_to_random_port("tcp://*") self.poller.register(self.control_in, POLLIN) # subtract CONTROL offset so the port calculated by the remote node matches the # random port to which we just bound self.recovery_ep = EndpntSpec("localhost", bind_addr - EndpntSpec.CONTROL) self.logger.debug('binding recovery socket to {}'.format(self.recovery_ep)) # we send election PUBs on this socket self.pub = self.ctx.socket(PUB) for c in self.cfgsvc.topo_get_all_collectors(): if c.endpoint == self.endpoint: self.logger.debug('skipping adding self in election') continue # don't add ourself try: self.pub.connect(c.endpoint.connect_uri(EndpntSpec.BASE)) self.logger.debug('connecting to {} for election'.format(c)) sleep(0.1) # dumb, but works; give socket time to connect except ZMQError as e: self.logger.error('unable to connect to endpoint {}: {}'.format(c.endpoint, e)) def _run(self): self.logger.error('EEEEEEKK!!! root node died... starting an election...') # create sockets in method called by recovery Thread instead of contructor which is called # by Node service thread. this avoids 0mq termination issues as described here: # http://zeromq.org/whitepapers:0mq-termination self.__init_sockets() # we can only exit once a leader has been elected while self.elected_leader is None: # the below loop may raise exceptions during poll() and recv(), but the parent class # will catch these and then call _cleanup() self.last_message_time = now_msecs() # just to seed the loop while (now_msecs() - self.last_message_time) < RECOVERY_ELECTION_WAIT_MS: # 1) process all messages on queue while True: try: msg = self._get_from_queue() self.__process_message(msg) except Again: break # 2) poll (timeout after 1s) items = dict(self.poller.poll(timeout=1000)) # 3) process all messages on socket if self.control_in in items: while True: try: msg = CONTROL.recv(self.control_in) self.__process_message(msg) except Again: break # 4) check if won any self-started elections for elect in self.elections.values(): if elect.node_uuid != self.uuid: continue if elect.result != 'pending': continue if (now_msecs() - elect.wutup_time) > RECOVERY_IWIN_WAIT_MS: self.last_message_time = now_msecs() elect.iwin(self.endpoint, self.uuid).send(self.pub) # if loop exited, time expired; check elections and find winner or start new election elected = self.__tally_results() if elected is None: self.logger.error('timed out waiting for leader; starting new election') self.__start_election() else: self.logger.info('new leader elected: {}'.format(elected)) self.elected_leader = elected return 'success' def __tally_results(self): # winner is election with most recent stop_time and "won" result # TODO: is time enough to determine winner? even if older winner has higher uuid? winner = None for elect in self.elections.values(): if elect.result == 'won': if winner is None: winner = elect else: winner = max(elect, winner, key=lambda e: e.iwin_time) return winner def __process_message(self, msg): # Expected Message Types: # SOS (CONTROL) : local message from Configuration service # WUTUP (TOPO) : remote message from other Collector # YO (CONTROL) : remote message from other Collector (response to WUTUP PUB) # IWIN (TOPO) : remote message from other Collector # # + SOS will come from the shared message queue since this is a local message from the # Configuration service. # + WUTUP and IWIN will also come from the shared message queue since these are received # on the SUB socket owned by the Node service. # + YO will come directly on the control_in socket if msg.is_error: self.logger.error('received error: {}'.format(msg)) return if isinstance(msg, TOPO): if msg.is_recovery: if msg.is_wutup: # track existing election self.last_message_time = now_msecs() elect_uuid = UUID(msg.content) elect = Election(msg.endpoint, msg.uuid, elect_uuid) self.elections[elect.elec_uuid] = elect if self.uuid > elect.node_uuid: peer_ep = msg.endpoint.connect_uri(EndpntSpec.CONTROL) self.control_out.connect(peer_ep) self.logger.debug('sending YO response to {}'.format(peer_ep)) sleep(0.1) # dumb, but works; give socket time to connect elect.yo(self.endpoint, self.uuid).send(self.control_out) self.control_out.disconnect(peer_ep) return elif msg.is_iwin: # record winner self.last_message_time = now_msecs() elect_uuid = UUID(msg.content) if elect_uuid not in self.elections: self.logger.warn('received IWIN for unknown election') elect = Election(msg.endpoint, msg.uuid, elect_uuid) self.elections[elect_uuid] = elect elect = self.elections[elect_uuid] elect.iwin(msg.endpoint, msg.uuid) if self.uuid > elect.node_uuid: # another node thinks they should be root, but we have higher uuid; here # comes the bully... # TODO: optimize to not start a new election if already winning another; # re-use logic in #4 of _run() loop self.__start_election() return elif isinstance(msg, CONTROL): if msg.is_sos: # sos is a local detection of the root failure assert msg.uuid == self.uuid # check for active election or start new election if len(self.elections) > 0: return # if any elections, we're done here self.__start_election() return elif msg.is_yo: elect_uuid = UUID(msg.properties['election-uuid']) if elect_uuid not in self.elections: self.logger.error('received YO for unknown election; ignoring') return elect = self.elections[elect_uuid] if elect.node_uuid != self.uuid: self.logger.error('received YO for unowned election; ignoring') return self.last_message_time = now_msecs() elect.yo(msg.endpoint, msg.uuid) return self.logger.error('unexpected message: {}'.format(msg)) return def __start_election(self): self.last_message_time = now_msecs() elect = Election(self.endpoint, self.uuid) self.elections[elect.elec_uuid] = elect # send recovery endpoint so replies are sent to us instead of Node service elect.wutup(self.recovery_ep, self.uuid).send(self.pub) def _cleanup(self): if self.control_out is not None: self.control_out.close() del self.control_out if self.control_in is not None: self.control_in.close() del self.control_in if self.pub is not None: self.pub.close() del self.pub self.logger.debug(str(self.elections))
class ServiceMixin(Thread): def __init__(self, pipe, local_ep, local_uuid, config_svc): Thread.__init__(self, name='dcamp.service.{}'.format(self)) self.ctx = Context.instance() self.__control_pipe = pipe assert isinstance(local_ep, EndpntSpec) self.__endpoint = local_ep assert isinstance(local_uuid, UUID) self.__uuid = local_uuid self.__cfgsvc = config_svc self.logger = getLogger('dcamp.service.%s' % self) self.poller = Poller() self.poller_timer = None self.poller.register(self.__control_pipe, POLLIN) def __str__(self): return self.__class__.__name__ @property def cfgsvc(self): return self.__cfgsvc @property def endpoint(self): return self.__endpoint @property def uuid(self): return self.__uuid def __send_control(self, message): self.__control_pipe.send_string(message) def __recv_control(self): return self.__control_pipe.recv_string() def _cleanup(self): # tell role we're done (if we can) if not self.in_errored_state: # @todo: this might raise an exception / issue #38 self.__send_control('STOPPED') self.logger.debug('sent STOPPED control reply') # shared context; will be term()'ed by caller self.__control_pipe.close() del self.__control_pipe self.logger.debug('service cleanup finished; exiting') def _pre_poll(self): pass def _post_poll(self, items): raise NotImplementedError('subclass must implement _post_poll()') def _do_control(self): """ Process control command on the pipe. """ msg = self.__recv_control() if 'STOP' == msg: self.logger.debug('received STOP control command') self.stop_state() else: self.__send_control('WTF') self.logger.error('unknown control command: %s' % msg) def run(self): self.run_state() # wait for configuration service to init if self.__cfgsvc is not None: self.__cfgsvc.wait_for_gogo() while self.in_running_state: try: self._pre_poll() items = dict(self.poller.poll(self.poller_timer)) self._post_poll(items) if self.__control_pipe in items: self._do_control() except ZMQError as e: if e.errno == ETERM: self.logger.debug('received ETERM: %s' % self.__class__) self.error_state() else: raise # thread is stopping; cleanup and exit return self._cleanup()
class Server(): """A node server. Each node has a request URL where request are accepted and a service URL where the node periodically publishes its list of known nodes and the time of the last contact (last publication from that node on the service URL). The request commands are: -new: register a new node -get: get a key -set: set a key Each node owns a Nodes object which tells to which node a key belongs and a Cache object to store key/value pairs. When keys need to be moved (node list did change) then the old and new owners are compared for each key and the key gets sent or deleted accordingly. """ PUB_INTERVALL = timedelta(seconds=5) IO_TIMEOUT = 5 * 1000 # milliseconds CMD_NEW = "new" CMD_GET = "get" CMD_SET = "set" def __init__(self, context, req_address, pub_address): self._context = context self._poller = Poller() self._nodes_sockets = {} # requests and service sockets # FIFO request queue for nodes: [in_progress, data, callback(response)] self._nodes_requests = defaultdict(list) # requests queues for nodes self._cache = Cache() # the cache node_id = uuid1().hex # unique node id, based on current time _LOG.info("node id: %s", node_id) self._nodes = Nodes(node_id, req_address, pub_address) # other nodes def _open_req_socket(self, addr): req_socket = self._context.socket(REQ) req_socket.setsockopt(RCVTIMEO, self.IO_TIMEOUT) req_socket.connect(addr) self._poller.register(req_socket, POLLIN) return req_socket def _open_sub_socket(self, addr): sub_socket = self._context.socket(SUB) sub_socket.setsockopt_string(SUBSCRIBE, "") sub_socket.connect(addr) self._poller.register(sub_socket, POLLIN) return sub_socket def _add_request(self, node_id, data, callback): """Adds a new request for a particular node. Each request in the queue consists of: - in progress flag - data to be send - function (with one argument, the response) to be called when the response is ready """ self._nodes_requests[node_id].append([False, data, callback]) def _hanlde_subscriptions(self, poll_sockets): """Read the nodes list from each other node and merge the information into the local node list. :return: true if nodes where added """ added = [] for node_id, (_, pub_socket) in self._nodes_sockets.items(): if pub_socket in poll_sockets: node_id, nodes = pub_socket.recv_json() # Convert string to datetime. nodes = { i: (req_addr, pub_addr, _str_to_dt(last)) for i, (req_addr, pub_addr, last) in nodes.items()} # Merge nodes list. new_nodes = self._nodes.update_nodes(nodes) # Create sockets for new nodes. for node_id, req_addr, pub_addr in new_nodes: _LOG.debug("adding node: %s", node_id) req_socket = self._open_req_socket(req_addr) sub_socket = self._open_sub_socket(pub_addr) added.append((node_id, (req_socket, sub_socket))) self._nodes_sockets.update(dict(added)) # save sockets return len(added) > 0 # nodes list got changed? def _add_node(self, node_id, req_addr, pub_addr, req_socket=None): """Add one node to the list and create the sockets for the request and service URLs (subscribe to the node). """ _LOG.debug("adding node: %s", node_id) # Add node to the nodes list. self._nodes.add_node(node_id, req_addr, pub_addr) if req_socket is None: # does that not yet exist? req_socket = self._open_req_socket(req_addr) sub_socket = self._open_sub_socket(pub_addr) # Remember sockets. self._nodes_sockets[node_id] = (req_socket, sub_socket) def _remove_nodes(self, removed_nodes): """Unregister and remove a node, remove all pending request, too. """ for node_id in removed_nodes: _LOG.debug("removing node: %s", node_id) # Unregister sockets from poller. req_socket, pub_socket = self._nodes_sockets.get( node_id, (None, None)) if req_socket is not None: self._poller.unregister(req_socket) if pub_socket is not None: self._poller.unregister(pub_socket) # Forget the sockets and erase request queue. self._nodes_sockets.pop(node_id, None) self._nodes_requests.pop(node_id, None) return len(removed_nodes) > 0 # nodes list got changed? def _handle_responses(self, poll_sockets): """When a request from the queue gets an answer call the function for that request, if possible. """ for node_id, (req_socket, _) in self._nodes_sockets.items(): # Some node did replay? if req_socket in poll_sockets: # Get the answer and call the callback function with it, if # there was one. response = req_socket.recv_json() requests = self._nodes_requests[node_id] if requests: in_progress, _, callback = requests[0] if in_progress: callback(response) # Request done, remove it from the queue. self._nodes_requests[node_id] = requests[1:] def _handle_request(self, node_changes, action, data): # Handle the requested action. _LOG.debug("Request %s: %s", action, str(data)) if action == self.CMD_NEW: # new node, register it node_id, req_addr, pub_addr = data self._add_node(node_id, req_addr, pub_addr) node_changes[0] = True # a node was added! return (self._nodes.id, self._nodes.pub_address) if action == self.CMD_SET: # set a key and value key, timestamp, value = data timestamp = _str_to_dt(timestamp) return self._cache.set( key, value, timestamp, self._nodes.get_nodes_for_index(key_index(key))) if action == self.CMD_GET: # return value for a key in_cache, timestamp, value = self._cache.get(data) return (in_cache, _dt_to_str(timestamp), value) def _rebalance(self): """ Check if key should be on some other nodes now, move/delete as necessary. """ send_entries = defaultdict(list) remove_entries = [] # Loop through all keys and compare on which nodes they should be. for key, ts, nodes, value, index in self._cache.key_indices: new_nodes = self._nodes.get_nodes_for_index(index) if new_nodes != nodes: # node list for that key changed? if self._nodes.id not in new_nodes: remove_entries.append(key) # no longer local for new_node in new_nodes - nodes: # where should they be? send_entries[new_node].append((key, ts, value)) self._cache.set(key, value, ts, new_nodes) # save new nodelist # Queue requests to set the keys on the new nodes. moves = 0 for new_node, entries in send_entries.items(): if new_node != self._nodes.id: for key, ts, value in entries: self._add_request( new_node, (self.CMD_SET, (key, _dt_to_str(ts), value)), lambda response: None) moves += 1 # Remove dead nodes from cache. for key in remove_entries: self._cache.delete_key(key) _LOG.debug( "adjusted distribution, %d moves, %d deletes", moves, len(remove_entries)) def _handle_api_get(self, api_pub_socket, req_id, key): """Handle a API get request from the local API. If the key is not local, send a request to one of the responsible nodes. Upon arrival of the response publish the response for the API, using the request id, which only the right listens for. """ req_id = req_id.encode("utf-8") # Publish the result. def send_response(resp, req_id=req_id, api_pub_socket=api_pub_socket): found, _, value = resp found = b"1" if found else b"0" value = value.encode("utf-8") if value else b"" api_pub_socket.send_multipart([req_id, found, value]) # Where is the key stored? nodes = self._nodes.get_nodes_for_index(key_index(key)) if nodes: if self._nodes.id in nodes: # local answer directly in_cache, _, value = self._cache.get(key) send_response((in_cache, None, value)) else: # remote, make request node_id = choice(list(nodes)) self._add_request(node_id, (self.CMD_GET, key), send_response) else: # can not be stored in cache, for whatever strange reason send_response((False, None, None)) def _handle_api_set(self, api_pub_socket, req_id, key, value): """Handle a API set request from the local API. Check where the key should be stored and make the appropriate requests. The publish all results to the API, alltough the API only cares about the fastest response. """ req_id = req_id.encode("utf-8") def send_response(resp, req_id=req_id, api_pub_socket=api_pub_socket): resp = str(resp).encode("utf-8") # "0", "-1", ... api_pub_socket.send_multipart([req_id, resp]) # Where should the key go? nodes = self._nodes.get_nodes_for_index(key_index(key)) if nodes: timestamp = datetime.now() if self._nodes.id in nodes: # save locally and publish response resp = self._cache.set(key, value, timestamp, nodes) send_response(resp) nodes.remove(self._nodes.id) # done with local node! for node_id in nodes: # make requests to all remote nodes self._add_request( node_id, (self.CMD_SET, (key, _dt_to_str(timestamp), value)), send_response) else: send_response(-2) # no nodes def loop(self, api_port, req_addr): """Event loop, listen to all sockets and handle all messages. If a req_address of an existing node is given the this node will register itself there, before entering the loop. The request on the api_port are handled by a Python WSGI instance running the Flask API app. """ # Register to existing node? if req_addr is not None: _LOG.debug("Contacting %s", req_addr) req_socket = self._open_req_socket(req_addr) req_socket.send_json( (self.CMD_NEW, (self._nodes.id, self._nodes.req_address, self._nodes.pub_address))) node_id, pub_addr = req_socket.recv_json() _LOG.debug("Received: %s %s", str(node_id), pub_addr) self._add_node(node_id, req_addr, pub_addr, req_socket) self._rebalance() # Create request and service sockets. nodes_publisher = self._context.socket(PUB) nodes_publisher.setsockopt(RCVTIMEO, self.IO_TIMEOUT) nodes_publisher.setsockopt(SNDTIMEO, self.IO_TIMEOUT) _LOG.debug("Publishing on %s", self._nodes.pub_address) nodes_publisher.bind(self._nodes.pub_address) req_socket = self._context.socket(REP) req_socket.setsockopt(RCVTIMEO, self.IO_TIMEOUT) req_socket.setsockopt(SNDTIMEO, self.IO_TIMEOUT) _LOG.debug("waiting for requests on %s", self._nodes.req_address) req_socket.bind(self._nodes.req_address) self._poller.register(req_socket, POLLIN) # Create in-process sockets to the API app. api_pull_socket = self._context.socket(PULL) api_pull_socket.bind(PUSH_ENDPOINT) self._poller.register(api_pull_socket, POLLIN) api_pub_socket = self._context.socket(PUB) api_pub_socket.bind(SUB_ENDPOINT) _LOG.info("Entering server loop") # Start web server. set_config(self._context) httpd = simple_server.make_server( '0.0.0.0', int(api_port), app) Thread(target=httpd.serve_forever).start() # Enter ZMQ loop. stop = False last_published = None try: while not stop: try: # Wait for messages, but not too long! sockets = dict( self._poller.poll(self.PUB_INTERVALL.seconds * 1000)) except KeyboardInterrupt: stop = True else: changes = [False] # Handle incoming node updates (subscriptions). if self._hanlde_subscriptions(sockets): changes[0] = True # Handle incoming responses. self._handle_responses(sockets) # Incoming requests? if req_socket in sockets: req_socket.send_json( self._handle_request( changes, *req_socket.recv_json())) # Remove dead nodes? if self._remove_nodes(self._nodes.remove_dead_nodes()): changes[0] = True # Did nodes change? if changes[0]: self._rebalance() # Request something? for node_id, requests in self._nodes_requests.items(): if requests and not requests[0][0]: request = requests[0] request[0] = True # in progress self._nodes_sockets[node_id][0].send_json( request[1]) # Publish something? now = datetime.now() if (last_published is None or now - last_published > self.PUB_INTERVALL): # Get nodes, convert last datetime to string nodes = { i: (req_addr, pub_addr, _dt_to_str(last)) for i, (req_addr, pub_addr, last) in self._nodes.nodes.items()} _LOG.debug( "publishing:\n%s", "\n".join( "{}: {}".format(i, str(n)) for i, n in nodes.items())) nodes_publisher.send_json((self._nodes.id, nodes)) last_published = now # Handle API get requests if api_pull_socket in sockets: req_id, action, key, value = ( api_pull_socket.recv_json()) if action == "get": self._handle_api_get(api_pub_socket, req_id, key) if action == "set": self._handle_api_set( api_pub_socket, req_id, key, value) finally: httpd.shutdown()
class Listener: def __init__(self, timeout=-1): self.controllerSocket = None self.timecodeSocket = None self.requestSocket = None self.subscribers = {} self.subscriberCallbacks = {} self.subscriberTopicFilters = {} self.periodicPublisher = None self.commandCb = None self.requestCb = None self.timeout = timeout self.poller = Poller() self.running = False self.pollCounter = 0 def connectController(self, address, commandCb=None): self.controllerSocket = zmq.Context().socket(zmq.PAIR) self.controllerSocket.bind(address) self.notifyConnected(address) self.poller.register(self.controllerSocket, zmq.POLLIN) self.commandCb = commandCb def sendController(self, message): wrappedMsg = IO.wrap(message) self.controllerSocket.send(wrappedMsg) def notifyConnected(self, address): msg = {'status': 'connected', 'address': address} self.sendController(msg) def notifyReady(self, address): msg = {'status': 'ready', 'address': address} self.sendController(msg) def listenToRequests(self, address, requestCb): self.requestCb = requestCb self.requestSocket = zmq.Context().socket(zmq.REP) self.requestSocket.bind(address) self.poller.register(self.requestSocket, zmq.POLLIN) def subscribeTimecode(self, address, timecodeCb): self.addSubscription(address, timecodeCb, label='_TC_', filters=['TIMECODE']) def addSubscription(self, address, subCb, label='default', filters=['default']): if label not in self.subscribers: self.subscribers[label] = zmq.Context().socket(zmq.SUB) self.subscribers[label].set_hwm(1) #self.subscribers[label].setsockopt(zmq.RCVHWM, 1) self.subscriberCallbacks[label] = subCb self.subscriberTopicFilters[label] = filters self.poller.register(self.subscribers[label], zmq.POLLIN) print "Subscribing", label, "to:", address self.subscribers[label].connect(address) if filters: for filter in filters: self.subscribers[label].setsockopt(zmq.SUBSCRIBE, filter) else: self.subscribers[label].setsockopt(zmq.SUBSCRIBE, '') def setPeriodicPublisher(self, publisher, overrideTimeout=True): self.periodicPublisher = publisher if overrideTimeout: self.timeout = publisher.getTimeout() def start(self): self.running = True self._main() def stop(self): self.running = False def pollRegistered(self, expected=-1, timeout=-1): if expected == -1: expected = len(self.poller.sockets) self.pollCounter = expected while self.pollCounter > 0: self.poll(timeout) def poll(self, timeout=-1): if timeout == -1: timeout = self.timeout socks = dict(self.poller.poll(timeout)) if self.controllerSocket in socks and socks[ self.controllerSocket] == zmq.POLLIN: command = self.controllerSocket.recv() command, _ = IO.unwrap(command) self.pollCounter -= 1 # Allow the user of this module to react to this command and to modify its contents if self.commandCb: modifiedCommand = self.commandCb(command) if modifiedCommand is not None: command = modifiedCommand if 'action' not in command: return action = command['action'] if action == 'exit': self.running = False if self.requestSocket in socks and socks[ self.requestSocket] == zmq.POLLIN: self.pollCounter -= 1 message = self.requestSocket.recv() message, _ = IO.unwrap(message) replyMessage = self.requestCb(message) if replyMessage is None: return wrappedReply = IO.wrap(replyMessage) self.requestSocket.send(wrappedReply) if socks: for label in self.subscribers.keys(): if self.subscribers[label] in socks and socks[ self.subscribers[label]] == zmq.POLLIN: self.pollCounter -= 1 data = self.subscribers[label].recv() # In case there is not space in the data, use find. Find returns -1 when the string isn't found sepIndex = data.find(' ') if sepIndex > -1 and not data[:4] == 'TIMS' and data[ 0:sepIndex] in self.subscriberTopicFilters[label]: topic, content = data[0:sepIndex], data[sepIndex + 1:] else: topic, content = '', data content, _ = IO.unwrap(content) if content is not None: self.subscriberCallbacks[label](content) if self.periodicPublisher: self.periodicPublisher.publish() def _main(self): while self.running: self.poll()
class RequestManager(Thread): """Manage requests. """ def __init__(self, port, attrs=None): Thread.__init__(self) self._loop = True self.out_socket = get_context().socket(ROUTER) self.out_socket.bind("tcp://*:" + str(port)) self.port = port self.in_socket = get_context().socket(PULL) self.in_socket.bind("inproc://replies" + str(port)) self._poller = Poller() self._poller.register(self.out_socket, POLLIN) self._poller.register(self.in_socket, POLLIN) self._attrs = attrs try: # Checking the validity of the file pattern _pattern = globify(attrs["origin"]) except ValueError as err: raise ConfigError('Invalid file pattern: ' + str(err)) except KeyError: if 'listen' not in attrs: raise self._deleter = Deleter() try: self._station = self._attrs["station"] except (KeyError, TypeError): LOGGER.warning("Station is not defined in config file") self._station = "unknown" LOGGER.debug("Station is '%s'" % self._station) def start(self): self._deleter.start() Thread.start(self) def pong(self, message): """Reply to ping """ return Message(message.subject, "pong", {"station": self._station}) def push(self, message): """Reply to push request """ for the_dict in gen_dict_contains(message.data, 'uri'): uri = urlparse(the_dict['uri']) rel_path = the_dict.get('path', '') pathname = uri.path # FIXME: check against file_cache if 'origin' in self._attrs and not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reachable".format(pathname)) try: move_it(pathname, message.data['destination'], self._attrs, rel_path=rel_path) except Exception as err: return Message(message.subject, "err", data=str(err)) else: if (self._attrs.get('compression') or self._attrs.get('delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) if 'dataset' in message.data: mtype = 'dataset' elif 'collection' in message.data: mtype = 'collection' elif 'uid' in message.data: mtype = 'file' else: raise KeyError('No known metadata in message.') new_msg = Message(message.subject, mtype, data=message.data.copy()) new_msg.data['destination'] = clean_url(new_msg.data['destination']) return new_msg def ack(self, message): """Reply with ack to a publication """ for url in gen_dict_extract(message.data, 'uri'): uri = urlparse(url) pathname = uri.path if 'origin' in self._attrs and not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reacheable".format(pathname)) if (self._attrs.get('compression') or self._attrs.get( 'delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) new_msg = Message(message.subject, "ack", data=message.data.copy()) try: new_msg.data['destination'] = clean_url( new_msg.data['destination']) except KeyError: pass return new_msg def info(self, message): topic = message.subject max_count = 2256 # Let's set a (close to arbitrary) limit on messages size. try: max_count = min(message.data.get("max_count", max_count), max_count) except AttributeError: pass uptime = datetime.datetime.utcnow() - START_TIME files = [] with file_cache_lock: for i in file_cache: if i.startswith(topic): files.append(i) if len(files) == max_count: break return Message(message.subject, "info", data={ "files": files, "max_count": max_count, "uptime": str(uptime) }) def unknown(self, message): """Reply to any unknown request. """ return Message(message.subject, "unknown") def reply_and_send(self, fun, address, message): in_socket = get_context().socket(PUSH) in_socket.connect("inproc://replies" + str(self.port)) reply = Message(message.subject, "error") try: reply = fun(message) except Exception: LOGGER.exception( "Something went wrong" " when processing the request: %s", str(message)) finally: LOGGER.debug("Response: " + str(reply)) try: in_socket.send_multipart([address, b'', str(reply)]) except TypeError: in_socket.send_multipart( [address, b'', bytes(str(reply), 'utf-8')]) def run(self): while self._loop: try: socks = dict(self._poller.poll(timeout=2000)) except ZMQError: LOGGER.info("Poller interrupted.") continue if socks.get(self.out_socket) == POLLIN: LOGGER.debug("Received a request") address, empty, payload = self.out_socket.recv_multipart( NOBLOCK) message = Message(rawstr=payload) fake_msg = Message(rawstr=str(message)) try: urlparse(message.data['destination']) except (KeyError, TypeError): pass else: fake_msg.data['destination'] = clean_url( message.data['destination']) LOGGER.debug("processing request: " + str(fake_msg)) if message.type == "ping": Thread(target=self.reply_and_send, args=(self.pong, address, message)).start() elif message.type == "push": Thread(target=self.reply_and_send, args=(self.push, address, message)).start() elif message.type == "ack": Thread(target=self.reply_and_send, args=(self.ack, address, message)).start() elif message.type == "info": Thread(target=self.reply_and_send, args=(self.info, address, message)).start() else: # unknown request Thread(target=self.reply_and_send, args=(self.unknown, address, message)).start() elif socks.get(self.in_socket) == POLLIN: self.out_socket.send_multipart( self.in_socket.recv_multipart(NOBLOCK)) else: # timeout pass def stop(self): """Stop the request manager.""" self._loop = False self._deleter.stop() self.out_socket.close(1) self.in_socket.close(1)
class PushRequester(object): """Base requester class. """ request_retries = 3 def __init__(self, host, port): self._socket = None self._reqaddress = "tcp://" + host + ":" + str(port) self._poller = Poller() self._lock = Lock() self.failures = 0 self.jammed = False self.running = True self.connect() def connect(self): """Connect to the server """ self._socket = context.socket(REQ) self._socket.connect(self._reqaddress) self._poller.register(self._socket, POLLIN) def stop(self): """Close the connection to the server """ self.running = False self._socket.setsockopt(LINGER, 0) self._socket.close() self._poller.unregister(self._socket) def reset_connection(self): """Reset the socket """ self.stop() self.connect() def __del__(self, *args, **kwargs): self.stop() def send_and_recv(self, msg, timeout=DEFAULT_REQ_TIMEOUT): with self._lock: retries_left = self.request_retries request = str(msg) self._socket.send(request) rep = None small_timeout = 0.1 while retries_left and self.running: now = time.time() while time.time() < now + timeout: if not self.running: return rep socks = dict(self._poller.poll(small_timeout)) if socks.get(self._socket) == POLLIN: reply = self._socket.recv() if not reply: LOGGER.error("Empty reply!") break try: rep = Message(rawstr=reply) except MessageError as err: LOGGER.error('Message error: %s', str(err)) break LOGGER.debug("Receiving (REQ) %s", str(rep)) self.failures = 0 self.jammed = False return rep # During big file transfers, give some time to a friend. time.sleep(0.1) LOGGER.warning("Timeout from " + str(self._reqaddress) + ", retrying...") # Socket is confused. Close and remove it. self.stop() retries_left -= 1 if retries_left <= 0: LOGGER.error("Server doesn't answer, abandoning... " + str(self._reqaddress)) self.connect() self.failures += 1 if self.failures == 5: LOGGER.critical("Server jammed ? %s", self._reqaddress) self.jammed = True break LOGGER.info("Reconnecting and resending " + str(msg)) # Create new connection self.connect() self._socket.send(request) return rep
class Cornerstone(Scaffold): """ Cornerstone can be used to create a 0mq poll loop. Upon creation of a Cornerstone instance, the initial state of the instance internal xmq poll loop is passive. To start the loop call Cornerstone run(). To stop the Cornerstone instance call Cornerstone.kill(). Cornerstone only allows for one zmq input port and one zmq output port. Cornerstone support respectively; Cornerstone.register_input_sock() and Cornerstone.register_output_sock() methods. Cornerstone implements an internal signal handler for detection of interrupt signals to handle shutdown of connection resources. Example Usage: >>> import threading >>> import time >>> from zmq import Context, SUB, SUBSCRIBE >>> # create, configure, and run a Cornerstone instance >>> foo = Cornerstone() >>> property_bag = foo.__create_property_bag__() >>> property_bag.heartbeat = 1 >>> foo.configure(args=property_bag) >>> t = threading.Thread(target=foo.run) >>> t.start() >>> time.sleep(1) >>> assert t.is_alive() >>> foo.kill() >>> t.join(1) >>> assert not t.is_alive() >>> # register an input socket >>> ctx = foo.zmq_ctx >>> sock = ctx.socket(SUB) >>> sock.connect('tcp://localhost:6670') >>> sock.setsockopt(SUBSCRIBE, "") >>> foo.register_input_sock(sock) >>> t = threading.Thread(target=foo.run) >>> t.start() >>> time.sleep(1) >>> foo.kill() >>> t.join(1) >>> assert not t.is_alive() """ def __init__(self, **kwargs): self._input_sock = None self._output_sock = None self._control_sock = None # determine if outgoing messages should enable NOBLOCK on send # default behaviour is to block on a send call till receiver is present self.no_block_send = False # configure the interrupt handling self._stop = True signal.signal(signal.SIGINT, self._signal_interrupt_handler) # a regular hearbeat interval must be set to the default. self.heartbeat = 3 # seconds # create the zmq context self.zmq_ctx = Context() # set the default input receive handler, if none has been assigned if not hasattr(self, 'input_recv_handler'): self.input_recv_handler = self._default_recv_handler # set the default handler, if none has been assigned. if not hasattr(self, '_command_handler'): self._command_handler = self._default_command_handler # construct the poller self._poll = Poller() # monitoring of message stream is off by default self.monitor_stream = False Scaffold.__init__(self, **kwargs) def configuration_options(self, arg_parser=None): """ The configuration_options method utilizes the arg_parser parameter to add arguments that should be handled during configuration. Keyword Arguments: arg_parser - argparse.ArgumentParser object. Sample invocation: >>> import argparse >>> parser = argparse.ArgumentParser(prog='app.py') >>> foo = Cornerstone() >>> foo.configuration_options(arg_parser=parser) >>> args = parser.print_usage() # doctest: +NORMALIZE_WHITESPACE usage: app.py [-h] [--heartbeat HEARTBEAT] [--monitor_stream] [--no_block_send] """ assert arg_parser arg_parser.add_argument('--heartbeat', type=int, default=3, help="Set the heartbeat rate in seconds of " "the core 0mq poller timeout.") arg_parser.add_argument('--monitor_stream', action='store_true', help='Enable the sampling of message flow.') arg_parser.add_argument('--no_block_send', action='store_true', help='Enable NOBLOCK on the sending of messages.' ' This will cause an message to be dropped ' 'if no receiver is present.') def configure(self, args=None): """ The configure method configures a Cornerstone instance by prior to the invocation of start. Keyword Arguments: args - an object with attributes set to the argument values.e """ assert args def register_input_sock(self, sock): """ Register a given input socket as the ingest point for a Cornerstone instance. Keyward Arguments: sock - the input socket that is to be registered. Return: None Cornerstone does not support multiple input sockets, so any currently registered input socket will be discarded. This is a per instance limitation, in which case the primary concern is ipaddress collision. Example Usage: >>> from zmq import SUB, SUBSCRIBE >>> foo = Cornerstone() >>> ctx = foo.zmq_ctx >>> sock1 = ctx.socket(SUB) >>> sock1.connect('tcp://localhost:2880') >>> sock1.setsockopt(SUBSCRIBE, "") >>> assert foo._poll.sockets == {} >>> foo.register_input_sock(sock1) >>> assert foo._poll.sockets.has_key(sock1) >>> sock2 = ctx.socket(SUB) >>> sock2.connect('tcp://localhost:2881') >>> sock2.setsockopt(SUBSCRIBE, "") >>> foo.register_input_sock(sock2) >>> assert not foo._poll.sockets.has_key(sock1) >>> assert foo._poll.sockets.has_key(sock2) """ # if there is an existing input socket, then it will be removed. if self._input_sock is not None: self._poll.unregister(self._input_sock) self._input_sock.close() self._input_sock = None self._input_sock = sock if self._input_sock is not None: self._poll.register(self._input_sock, POLLIN) def register_output_sock(self, sock): """ Register a given output socket as the egress point for a Cornerstone instance. Keyward Arguments: sock - the output socket that is to be registered. Return: none Cornerstone does not support multiple output sockets, so any currently registered output socket will be discarded. This is a per instance limitation. In which case the primary concern is ipaddress collision. Example Usage: >>> from zmq import PUB >>> foo = Cornerstone() >>> ctx = foo.zmq_ctx >>> sock1 = ctx.socket(PUB) >>> sock1.bind('tcp://*:2880') >>> assert foo._output_sock == None >>> foo.register_output_sock(sock1) >>> assert foo._output_sock == sock1 >>> sock2 = ctx.socket(PUB) >>> sock2.bind('tcp://*:28881') >>> foo.register_output_sock(sock2) >>> assert foo._output_sock == sock2 """ # if there is an existing output socket, then it will be removed. if self._output_sock is not None: self._output_sock.close() self._output_sock = None self._output_sock = sock def send(self, msg): assert msg if self.monitor_stream: self.log.info('o: %s', msg) if not self.no_block_send: self._output_sock.send(msg) else: try: self._output_sock.send(msg, NOBLOCK) except: self.log.error("Unexpected error:", sys.exc_info()[0]) def setRun(self): self._stop = False def isStopped(self): return self._stop def run(self): """ Comment: -- AAA -- What needs to occur here si to see if there is a 0mq connection configured. If so, then we will simply push to that connector. This will be the default behavior, at least for now. There should be a mechanism for transmitting the data out to a registered handler. """ self._stop = False self.log.info('Beginning run() with configuration: %s', self._args) #todo: raul - move this section to command configuraiton layer # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # of course this is when a command configuration layer get's added controller = self.zmq_ctx.socket(SUB) controller.connect('tcp://localhost:7885') controller.setsockopt(SUBSCRIBE, "") self._control_sock = controller self._poll.register(self._control_sock) # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ loop_count = 0 input_count = 0 while True: try: socks = dict(self._poll.poll(timeout=self.heartbeat)) loop_count += 1 if self.monitor_stream and (loop_count % 1000) == 0: sys.stdout.write('loop(%s)' % loop_count) sys.stdout.flush() if self._input_sock and socks.get(self._input_sock) == POLLIN: #todo: raul - this whole section needs to be redone, # see additional comment AAA above. msg = self.input_recv_handler(self._input_sock) input_count += 1 if self.monitor_stream: # and (input_count % 10) == 0: self.log.info('i:%s- %s', input_count, msg) if (self._control_sock and socks.get(self._control_sock) == POLLIN): msg = self._control_sock.recv() if self._command_handler is not None: self._command_handler(msg) if self._stop: self.log.info('Stop flag triggered ... shutting down.') break except ZMQError, ze: if ze.errno == 4: # Known exception due to keyboard ctrl+c self.log.info('System interrupt call detected.') else: # exit hard on unhandled exceptions self.log.error('Unhandled exception in run execution:%d - %s' % (ze.errno, ze.strerror)) exit(-1) # close the sockets held by the poller self._control_sock.close() self.register_input_sock(sock=None) self.register_output_sock(sock=None) self.log.info('Run terminated for %s', self.name)
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr): '''Connect to and marshall messages between broker and writer sockets. Parameters ---------- bot : zio.Message The BOT message rule_object: dicionary A ruleset rule object. writer_addr :: string The address of the writer's PULL socket to connect. broker_addr : string The address of the broker's SERVER socket to connect. ''' # An HDF path to be added to every message we send to writer. mattr = message_to_dict(bot) rattr = dict(rule_object.get("attr",{}), **mattr) log.info(f'writer: attrs: {rattr}') try: base_path = rule_object.get("grouppat","/").format(**rattr) except KeyError as e: log.error(f'writer: missing attribute: {e}') raise log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")') log.debug(bot) pipe.signal() push = ctx.socket(PUSH) push.connect(writer_addr) sock = ctx.socket(CLIENT) port = Port("write-handler", sock) port.connect(broker_addr) port.online(None) direction = mattr["direction"] if direction != "inject": raise RuntimeError(f'zio.flow.hdf.writer bad direction: "{direction}"') credit = mattr["credit"] flow = Flow(port, direction, credit) log.debug (f'writer({base_path}) send BOT to {broker_addr}') bot = flow.bot(bot) # this introduces us to the server log.debug (f'writer({base_path}) got response:\n{bot}') flow.begin() def push_message(m): log.debug (f'write_handler({base_path}) push {m}') attr = message_to_dict(m) attr['hdfgroup'] = base_path m.label = json.dumps(attr) push.send(m.encode()) #push_message(bot) poller = Poller() poller.register(pipe, POLLIN) poller.register(sock, POLLIN) while True: for which,_ in poller.poll(): if not which: return if which == pipe: # signal exit log.debug ('write_handler pipe hit') return # o.w. we have flow try: msg = flow.get() except TransmissionEnd as te: flow.eotsend() break push_message(msg) continue log.debug ('write_handler exiting') pipe.signal()
def file_handler(ctx, pipe, filename, *wargs): '''An actor that marshals messages from socket to file. Parameters ----------- filename : string Name of an HDF file in which to write wargs : tuple of args wargs[0] : string (address pattern) An f-string formatted with a "port" parameter that should resolve to a legal ZeroMQ socket address. When a successful bind() can be done on the result, the resolved address is returned through the pipe. If no successful address can be bound, an empty string is returned as an error indicator. ''' wargs = list(wargs) addrpat = wargs.pop(0) log.debug(f'actor: writer("{filename}", "{addrpat}")') fp = h5py.File(filename,'w') log.debug(f'writer: opened {filename}') pipe.signal() pull = ctx.socket(PULL) minport,maxport = 49999,65000 for port in range(minport,maxport): writer_addr = addrpat.format(port=port) pull.bind(writer_addr) log.debug(f'writer: bind to {writer_addr}') pipe.send_string(writer_addr) break flow_writer = dict() poller = Poller() poller.register(pipe, POLLIN) poller.register(pull, POLLIN) while True: for which,_ in poller.poll(): if not which or which == pipe: # signal exit log.debug(f'writer: {filename} exiting') fp.close() return # o.w. we have flow data = pull.recv() if not data: continue msg = Message(encoded=data) fobj = objectify(msg) path = fobj.pop("hdfgroup") # must be supplied msg.label = json.dumps(fobj) log.debug(f'writer: {filename}:/{path} writing: {msg}') fw = flow_writer.get(path, None) if fw is None: sg = fp.get(path, None) or fp.create_group(path) # note: in principle/future, TensWriter type can be # made an arg to support other message formats. fw = flow_writer[path] = TensWriter(sg, *wargs) fw.save(msg) #log.debug(f'writer: flush {filename}') fp.flush() return
def run(self): poller = Poller() poller.register(self._socket, POLLIN) while self._loop: socks = dict(poller.poll(timeout=2000)) if self._socket in socks and socks[self._socket] == POLLIN: message = Message(rawstr=self._socket.recv(NOBLOCK)) logger.debug("Request: " + str(message)) # send list of scanlines if(message.type == "request" and message.data["type"] == "scanlines"): sat = message.data["satellite"] epoch = "1950-01-01T00:00:00" start_time = strp_isoformat(message.data.get("start_time", epoch)) end_time = strp_isoformat(message.data.get("end_time", epoch)) resp = Message('/oper/polar/direct_readout/' + self._station, "scanlines", [(utctime.isoformat(), self._holder[sat][utctime][2]) for utctime in self._holder.get(sat, []) if utctime >= start_time and utctime <= end_time]) self._socket.send(str(resp)) # send one scanline elif(message.type == "request" and message.data["type"] == "scanline"): sat = message.data["satellite"] utctime = strp_isoformat(message.data["utctime"]) try: url = urlparse(self._holder[sat][utctime][1]) except KeyError: resp = Message('/oper/polar/direct_readout/' + self._station, "missing") else: if url.scheme in ["", "file"]: # data is locally stored. resp = Message('/oper/polar/direct_readout/' + self._station, "scanline", self._holder.get_scanline(sat, utctime), binary=True) else: # it's the address of a remote server. resp = self.forward_request(urlunparse(url), message) self._socket.send(str(resp)) # take in a new scanline elif(message.type == "notice" and message.data["type"] == "scanline"): sat = message.data["satellite"] utctime = message.data["utctime"] elevation = message.data["elevation"] filename = message.data["filename"] line_start = message.data["file_position"] utctime = strp_isoformat(utctime) self._holder.add_scanline(sat, utctime, elevation, line_start, filename) resp = Message('/oper/polar/direct_readout/' + self._station, "notice", "ack") self._socket.send(str(resp)) elif(message.type == "ping"): resp = Message('/oper/polar/direct_readout/' + self._station, "pong", {"station": self._station}) self._socket.send(str(resp)) if resp.binary: logger.debug("Response: " + " ".join(str(resp).split()[:6])) else: logger.debug("Response: " + str(resp))
# socket to receive control messages on controller = context.socket(SUB) controller.connect("tcp://localhost:6670") controller.setsockopt(SUBSCRIBE, "") # poler for handling receiver and controller messages poller = Poller() poller.register(receiver, POLLIN) poller.register(controller, POLLIN) print "Waiting for job ..." # poller handling while True: socks = dict(poller.poll()) if socks.get(receiver) == POLLIN: msg = receiver.recv() # process task workload_msec = int(msg) time.sleep(workload_msec * 0.001) # send result to sink sender.send(msg) # progress indicator stdout.write('.') stdout.flush()
class PushRequester(object): """Base requester class. """ request_retries = 3 def __init__(self, host, port): self._socket = None self._reqaddress = "tcp://" + host + ":" + str(port) self._poller = Poller() self._lock = Lock() self.failures = 0 self.jammed = False self.connect() def connect(self): """Connect to the server """ self._socket = context.socket(REQ) self._socket.connect(self._reqaddress) self._poller.register(self._socket, POLLIN) def stop(self): """Close the connection to the server """ self._socket.setsockopt(LINGER, 0) self._socket.close() self._poller.unregister(self._socket) def reset_connection(self): """Reset the socket """ self.stop() self.connect() def __del__(self, *args, **kwargs): self.stop() def send_and_recv(self, msg, timeout=REQ_TIMEOUT): with self._lock: retries_left = self.request_retries request = str(msg) self._socket.send(request) rep = None while retries_left: socks = dict(self._poller.poll(timeout)) if socks.get(self._socket) == POLLIN: reply = self._socket.recv() if not reply: LOGGER.error("Empty reply!") break rep = Message(rawstr=reply) self.failures = 0 self.jammed = False break else: LOGGER.warning("Timeout from " + str(self._reqaddress) + ", retrying...") # Socket is confused. Close and remove it. self.stop() retries_left -= 1 if retries_left <= 0: LOGGER.error("Server doesn't answer, abandoning... " + str(self._reqaddress)) self.connect() self.failures += 1 if self.failures == 5: LOGGER.critical("Server jammed ? %s", self._reqaddress) self.jammed = True break LOGGER.info("Reconnecting and resending " + str(msg)) # Create new connection self.connect() self._socket.send(request) return rep
class ZmqSelector(BaseSelector): """A selector that can be used with asyncio's selector base event loops.""" def __init__(self): # this maps file descriptors to keys self._fd_to_key = {} # read-only mapping returned by get_map() self._map = _SelectorMapping(self) self._poller = ZMQPoller() def _fileobj_lookup(self, fileobj): """Return a file descriptor from a file object. This wraps _fileobj_to_fd() to do an exhaustive search in case the object is invalid but we still have it in our map. This is used by unregister() so we can unregister an object that was previously registered even if it is closed. It is also used by _SelectorMapping. """ try: return _fileobj_to_fd(fileobj) except ValueError: # Do an exhaustive search. for key in self._fd_to_key.values(): if key.fileobj is fileobj: return key.fd # Raise ValueError after all. raise def register(self, fileobj, events, data=None): if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): raise ValueError("Invalid events: {!r}".format(events)) key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) if key.fd in self._fd_to_key: raise KeyError("{!r} (FD {}) is already registered" .format(fileobj, key.fd)) z_events = 0 if events & EVENT_READ: z_events |= POLLIN if events & EVENT_WRITE: z_events |= POLLOUT try: self._poller.register(key.fd, z_events) except ZMQError as exc: raise OSError(exc.errno, exc.strerror) from exc self._fd_to_key[key.fd] = key return key def unregister(self, fileobj): try: key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) except KeyError: raise KeyError("{!r} is not registered".format(fileobj)) from None try: self._poller.unregister(key.fd) except ZMQError as exc: self._fd_to_key[key.fd] = key raise OSError(exc.errno, exc.strerror) from exc return key def modify(self, fileobj, events, data=None): try: fd = self._fileobj_lookup(fileobj) key = self._fd_to_key[fd] except KeyError: raise KeyError("{!r} is not registered".format(fileobj)) from None if data == key.data and events == key.events: return key if events != key.events: z_events = 0 if events & EVENT_READ: z_events |= POLLIN if events & EVENT_WRITE: z_events |= POLLOUT try: self._poller.modify(fd, z_events) except ZMQError as exc: raise OSError(exc.errno, exc.strerror) from exc key = key._replace(data=data, events=events) self._fd_to_key[key.fd] = key return key def close(self): self._fd_to_key.clear() self._poller = None def get_map(self): return self._map def _key_from_fd(self, fd): """Return the key associated to a given file descriptor. Parameters: fd -- file descriptor Returns: corresponding key, or None if not found """ try: return self._fd_to_key[fd] except KeyError: return None def select(self, timeout=None): if timeout is None: timeout = None elif timeout <= 0: timeout = 0 else: # poll() has a resolution of 1 millisecond, round away from # zero to wait *at least* timeout seconds. timeout = math.ceil(timeout * 1e3) ready = [] try: z_events = self._poller.poll(timeout) except ZMQError as exc: if exc.errno == EINTR: return ready else: raise OSError(exc.errno, exc.strerror) from exc for fd, evt in z_events: events = 0 if evt & POLLIN: events |= EVENT_READ if evt & POLLOUT: events |= EVENT_WRITE if evt & POLLERR: events = EVENT_READ | EVENT_WRITE key = self._key_from_fd(fd) if key: ready.append((key, events & key.events)) return ready
class Subscriber(object): """Subscriber Subscribes to *addresses* for *topics*, and perform address translation of *translate* is true. The function *message_filter* can be used to discriminate some messages on the subscriber side. *topics* on the other hand performs filtering on the publishing side (from zeromq 3). Example:: from posttroll.subscriber import Subscriber, get_pub_address addr = get_pub_address(service, timeout=2) sub = Subscriber([addr], 'my_topic') try: for msg in sub(timeout=2): print("Consumer got", msg) except KeyboardInterrupt: print("terminating consumer...") sub.close() """ def __init__(self, addresses, topics='', message_filter=None, translate=False): self._topics = self._magickfy_topics(topics) self._filter = message_filter self._translate = translate self.sub_addr = {} self.addr_sub = {} self.poller = None self._hooks = [] self._hooks_cb = {} self.poller = Poller() self._lock = Lock() self.update(addresses) self._loop = True def add(self, address, topics=None): """Add *address* to the subscribing list for *topics*. It topics is None we will subscibe to already specified topics. """ with self._lock: if address in self.addresses: return False topics = self._magickfy_topics(topics) or self._topics LOGGER.info("Subscriber adding address %s with topics %s", str(address), str(topics)) subscriber = get_context().socket(SUB) for t__ in topics: subscriber.setsockopt_string(SUBSCRIBE, six.text_type(t__)) subscriber.connect(address) self.sub_addr[subscriber] = address self.addr_sub[address] = subscriber if self.poller: self.poller.register(subscriber, POLLIN) return True def remove(self, address): """Remove *address* from the subscribing list for *topics*. """ with self._lock: try: subscriber = self.addr_sub[address] except KeyError: return False LOGGER.info("Subscriber removing address %s", str(address)) if self.poller: self.poller.unregister(subscriber) del self.addr_sub[address] del self.sub_addr[subscriber] subscriber.close() return True def update(self, addresses): """Updating with a set of addresses. """ if isinstance(addresses, six.string_types): addresses = [addresses, ] s0_, s1_ = set(self.addresses), set(addresses) sr_, sa_ = s0_.difference(s1_), s1_.difference(s0_) for a__ in sr_: self.remove(a__) for a__ in sa_: self.add(a__) return bool(sr_ or sa_) def add_hook_sub(self, address, topics, callback): """Specify a *callback* in the same stream (thread) as the main receive loop. The callback will be called with the received messages from the specified subscription. Good for operations, which is required to be done in the same thread as the main recieve loop (e.q operations on the underlying sockets). """ LOGGER.info("Subscriber adding SUB hook %s for topics %s", str(address), str(topics)) socket = get_context().socket(SUB) for t__ in self._magickfy_topics(topics): socket.setsockopt_string(SUBSCRIBE, six.text_type(t__)) socket.connect(address) self._add_hook(socket, callback) def add_hook_pull(self, address, callback): """Same as above, but with a PULL socket. (e.g good for pushed 'inproc' messages from another thread). """ LOGGER.info("Subscriber adding PULL hook %s", str(address)) socket = get_context().socket(PULL) socket.connect(address) self._add_hook(socket, callback) def _add_hook(self, socket, callback): """Generic hook. The passed socket has to be "receive only". """ self._hooks.append(socket) self._hooks_cb[socket] = callback if self.poller: self.poller.register(socket, POLLIN) @property def addresses(self): """Get the addresses """ return self.sub_addr.values() @property def subscribers(self): """Get the subscribers """ return self.sub_addr.keys() def recv(self, timeout=None): """Receive, optionally with *timeout* in seconds. """ if timeout: timeout *= 1000. for sub in list(self.subscribers) + self._hooks: self.poller.register(sub, POLLIN) self._loop = True try: while self._loop: sleep(0) try: socks = dict(self.poller.poll(timeout=timeout)) if socks: for sub in self.subscribers: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) if not self._filter or self._filter(m__): if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] m__.sender = (m__.sender.split("@")[0] + "@" + host) yield m__ for sub in self._hooks: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) self._hooks_cb[sub](m__) else: # timeout yield None except ZMQError as err: LOGGER.exception("Receive failed: %s", str(err)) finally: for sub in list(self.subscribers) + self._hooks: self.poller.unregister(sub) def __call__(self, **kwargs): return self.recv(**kwargs) def stop(self): """Stop the subscriber. """ self._loop = False def close(self): """Close the subscriber: stop it and close the local subscribers. """ self.stop() for sub in list(self.subscribers) + self._hooks: try: sub.setsockopt(LINGER, 1) sub.close() except ZMQError: pass @staticmethod def _magickfy_topics(topics): """Add the magick to the topics if missing. """ # If topic does not start with messages._MAGICK (pytroll:/), it will be # prepended. if topics is None: return None if isinstance(topics, six.string_types): topics = [topics, ] ts_ = [] for t__ in topics: if not t__.startswith(_MAGICK): if t__ and t__[0] == '/': t__ = _MAGICK + t__ else: t__ = _MAGICK + '/' + t__ ts_.append(t__) return ts_ def __del__(self): for sub in list(self.subscribers) + self._hooks: try: sub.close() except: pass