class AsyncServerAdapter(object): producer_class = Producer def __init__(self, backend_rep_uri, frontend_rep_uri, frontend_pub_uri, control_pipe=None): self.uris = OrderedDict([ ('backend_rep', backend_rep_uri), ('consumer_push_be', unique_ipc_uri()), ('consumer_pull_be', unique_ipc_uri()), ('frontend_rep_uri', frontend_rep_uri), ('frontend_pub_uri', frontend_pub_uri) ]) self.control_pipe = control_pipe self.done = False logging.getLogger(log_label(self)).info("uris: %s", self.uris) def watchdog(self): if self.control_pipe is None: return elif not self.done and self.control_pipe.poll(): self.done = True self.finish() def run(self): consumer = Process(target=Consumer(self.uris['backend_rep'], self.uris['consumer_push_be'], self.uris['consumer_pull_be']).run ) producer = Process(target=self.producer_class( self.uris['frontend_rep_uri'], self.uris['frontend_pub_uri'], self.uris['consumer_pull_be'], self.uris['consumer_push_be']).run ) self.io_loop = IOLoop() periodic_callback = PeriodicCallback(self.watchdog, 500, self.io_loop) periodic_callback.start() try: consumer.start() producer.start() self.io_loop.start() except KeyboardInterrupt: pass producer.terminate() consumer.terminate() logging.getLogger(log_label(self)).info('PRODUCER and CONSUMER have ' 'been terminated') def __del__(self): uris = [self.uris[label] for label in ('consumer_push_be', 'consumer_pull_be', )] cleanup_ipc_uris(uris) def finish(self): logging.getLogger(log_label(self)).debug('"finish" request received') self.io_loop.stop()
class WorkerConnection(object): def __init__(self, address, retry_ms=10000, compression=None, serialization=None): self.address = address self.message_address = 'inproc://WorkerConnection%s' % id(self) self.__context = zmq.Context() self.__queue_socket = self.__context.socket(zmq.PUSH) self.__queue_socket.bind(self.message_address) self.__thread = None self.__retry_ms = retry_ms self.__callbacks = {} self.__queued_messages = {} self.__message_timeouts = {} self.__ioloop = None self.loads = serialization and serialization.loads or pickle.loads self.dumps = serialization and serialization.dumps or pickle.dumps self.compress = compression and compression.compress or (lambda x: x) self.decompress = compression and compression.decompress or (lambda x: x) def invoke(self, method, parameters, callback=None, retry_ms=0): self._queue_message(self.compress(self.dumps({'method': method, 'parameters': parameters})), callback, retry_ms) def __len__(self): return len(self.__queued_messages) def __getattr__(self, path): return WorkerInvocation(path, self) def _queue_message(self, message, callback=None, retry_ms=0): if not self.__ioloop: self.start() message_id = str(uuid4()) if callback: self.__callbacks[message_id] = callback if retry_ms > 0: self.__message_timeouts[message_id] = retry_ms self.__queue_socket.send_multipart(('', message_id, message)) def log_error(self, error): logging.error(repr(error)) def start(self): def loop(): self.__ioloop = IOLoop() queue_socket = self.__context.socket(zmq.PULL) queue_socket.connect(self.message_address) queue_stream = ZMQStream(queue_socket, self.__ioloop) worker_socket = self.__context.socket(zmq.DEALER) worker_socket.connect(self.address) worker_stream = ZMQStream(worker_socket, self.__ioloop) def receive_response(message): self.__queued_messages.pop(message[1], None) self.__message_timeouts.pop(message[1], None) callback = self.__callbacks.pop(message[1], None) if callback: try: callback(self.loads(self.decompress(message[2]))) except Exception as e: self.log_error(e) worker_stream.on_recv(receive_response) def queue_message(message): self.__queued_messages[message[1]] = (time() * 1000, message) try: worker_stream.send_multipart(message) except Exception as e: self.log_error(e) queue_stream.on_recv(queue_message) def requeue_message(): now = time() * 1000 for message in (item[1] for item in self.__queued_messages.itervalues() if item[0] + self.__message_timeouts.get(item[1][1], self.__retry_ms) < now): queue_message(message) requeue_callback = PeriodicCallback(requeue_message, self.__retry_ms, io_loop = self.__ioloop) requeue_callback.start() self.__ioloop.start() self.__thread = None self.__thread = Thread(target=loop) self.__thread.daemon = True self.__thread.start() def stop(self): if self.__ioloop: self.__ioloop.stop() def join(self): if self.__thread: self.__thread.join() def enable_traceback_logging(self): from new import instancemethod from traceback import format_exc def log_error(self, e): logging.error(format_exc()) self.log_error = instancemethod(log_error, self) @classmethod def instance(cls): if not hasattr(cls, '_instance'): cls._instance = cls(options.worker_address, retry_ms=options.worker_retry_ms, compression=options.worker_compression_module and __import__(options.worker_compression_module), serialization=options.worker_serialization_module and __import__(options.worker_serialization_module)) return cls._instance
class WorkerConnection(object): '''Use a ``WorkerConnection`` to make RPCs to the remote worker service(s) or worker/router specified by ``address``. ``address`` may be either an enumerable of address strings or a string of comma separated addresses. RPC retries and timeouts will happen by at most every ``abs(timeout)`` seconds when a periodic callback runs through all active messages and checks for prolonged requests. This is also the default timeout for any new calls. ``timeout`` must not be ``0``. Optionally pass any object or module with ``compress`` and ``decompress`` methods as the ``compression`` parameter to compress messages. The module must implement the same algorithm used on the worker service. By default, messages are not compressed. Optionally pass any object or module with ``dumps`` and ``loads`` methods that convert an ``object`` to and from a ``str`` to replace the default ``cPickle`` serialization with a protocol of your choice. Use ``auto_retry`` to specify whether or not messages should be retried by default. Retrying messages can cause substantial congestion in your worker service. Use with caution. ''' def __init__(self, address, timeout=10.0, compression=None, serialization=None, auto_retry=False): if isinstance(address, str): self.active_connections = {i.strip() for i in address.split(',')} else: self.active_connections = set(address) self.message_address = 'inproc://WorkerConnection%s' % id(self) self.__context = zmq.Context() self.__queue_socket = self.__context.socket(zmq.PUSH) self.__queue_socket.bind(self.message_address) self.__thread = None self.__timeout = timeout self.__callbacks = {} self.__queued_messages = {} self.__message_auto_retry = {} self.__message_timeouts = {} self.__ioloop = None self.__auto_retry = auto_retry self.loads = serialization and serialization.loads or pickle.loads self.dumps = serialization and serialization.dumps or pickle.dumps self.compress = compression and compression.compress or (lambda x: x) self.decompress = compression and compression.decompress or (lambda x: x) def invoke(self, method, parameters={}, callback=None, timeout=0, auto_retry=None): '''Invoke a ``method`` to be run on a remote worker process with the given ``parameters``. If specified, ``callback`` will be invoked with any response from the remote worker. By default the worker will timeout or retry based on the settings of the current ``WorkerConnection`` but ``timeout`` and ``auto_retry`` can be used for invocation specific behavior. Note: ``callback`` will be invoked with ``{'error': 'timeout'}`` on ``timeout`` if ``auto_retry`` is false. Invocations set to retry will never timeout and will instead be re-sent until a response is received. This behavior can be useful for critical operations but has the potential to cause substantial congestion in the worker system. Use with caution. Negative values of ``timeout`` will prevent messages from ever expiring or retrying regardless of ``auto_retry``. The default values of ``timeout`` and ``auto_retry`` cause a fallback to the values used to initialize ``WorkerConnection``. Alternativly, you can invoke methods with ``WorkerConnection.<module>.<method>(parameters, callback=None, timeout=0, auto_retry=None)`` where ``"<module>.<method>"`` will be passed as the ``method`` argument to ``invoke()``. ''' self._queue_message(self.compress(self.dumps({'method': method, 'parameters': parameters})), callback, timeout, auto_retry) def add_connection(self, address): '''Connect to the worker at ``address``. Worker invocations will be round robin load balanced between all connected workers.''' self._queue_message(address, command=WORKER_SOCKET_CONNECT) def remove_connection(self, address): '''Disconnect from the worker at ``address``. Worker invocations will be round robin load balanced between all connected workers.''' self._queue_message(address, command=WORKER_SOCKET_DISCONNECT) def set_connections(self, addresses): '''A convenience method to set the connected addresses. A connection will be made to any new address included in the ``addresses`` enumerable and any currently connected address not included in ``addresses`` will be disconnected. If an address in ``addresses`` is already connected, it will not be affected. ''' addresses = set(addresses) to_remove = {a for a in self.active_connections if a not in addresses} to_add = {a for a in addresses if a not in self.active_connections} for a in to_remove: self.remove_connection(a) for a in to_add: self.add_connection(a) def __len__(self): return len(self.__queued_messages) def __getattr__(self, path): return WorkerInvocation(path, self) def _queue_message(self, message, callback=None, timeout=0, auto_retry=None, command=''): if not self.__ioloop: self.start() message_id = str(uuid4()) if callback: self.__callbacks[message_id] = callback if timeout != 0: self.__message_timeouts[message_id] = timeout if auto_retry is not None: self.__message_auto_retry[message_id] = auto_retry self.__queue_socket.send_multipart((command, message_id, message)) def log_error(self, error): logging.error(repr(error)) def start(self): def loop(): self.__ioloop = IOLoop() queue_socket = self.__context.socket(zmq.PULL) queue_socket.connect(self.message_address) queue_stream = ZMQStream(queue_socket, self.__ioloop) worker_socket = self.__context.socket(zmq.DEALER) for address in self.active_connections: worker_socket.connect(address) worker_stream = ZMQStream(worker_socket, self.__ioloop) def receive_response(message, response_override=None): self.__queued_messages.pop(message[1], None) self.__message_timeouts.pop(message[1], None) callback = self.__callbacks.pop(message[1], None) if callback: try: callback(response_override or self.loads(self.decompress(message[2]))) except Exception as e: self.log_error(e) callback({'error': e}) worker_stream.on_recv(receive_response) def queue_message(message): if message[0]: if message[0] == WORKER_SOCKET_CONNECT and message[2] not in self.active_connections: self.active_connections.add(message[2]) worker_stream.socket.connect(message[2]) elif message[0] == WORKER_SOCKET_DISCONNECT and message[2] in self.active_connections: self.active_connections.remove(message[2]) worker_stream.socket.disconnect(message[2]) return self.__queued_messages[message[1]] = (time(), message) try: worker_stream.send_multipart(message) except Exception as e: self.log_error(e) queue_stream.on_recv(queue_message) def timeout_message(): now = time() for message, retry in [(item[1], self.__message_auto_retry.get(item[1][1], self.__auto_retry)) for item, t in ((i, self.__message_timeouts.get(i[1][1], self.__timeout)) for i in self.__queued_messages.itervalues()) if t >= 0 and (item[0] + t < now)]: if retry: logging.info('Worker timeout, requeuing ' + message[1]) queue_message(message) else: receive_response(('', message[1]), {'error': 'timeout'}) timeout_callback = PeriodicCallback(timeout_message, int(abs(self.__timeout * 1000.0)), io_loop = self.__ioloop) timeout_callback.start() self.__ioloop.start() self.__thread = None self.__thread = Thread(target=loop) self.__thread.daemon = True self.__thread.start() def stop(self): if self.__ioloop: self.__ioloop.stop() def join(self): if self.__thread: self.__thread.join() def enable_traceback_logging(self): from new import instancemethod from traceback import format_exc def log_error(self, e): logging.error(format_exc()) self.log_error = instancemethod(log_error, self) @classmethod def instance(cls): '''Returns the default instance of ``WorkerConnection`` as configured by the options prefixed with ``worker_``, instantiating it if necessary. Import the ``workerconnection`` module within your ``TotoService`` and run it with ``--help`` to see all available options. ''' if not hasattr(cls, '_instance'): cls._instance = cls(options.worker_address, timeout=options.worker_timeout, compression=options.worker_compression_module and __import__(options.worker_compression_module), serialization=options.worker_serialization_module and __import__(options.worker_serialization_module), auto_retry=options.worker_auto_retry) return cls._instance