class test_LogMixin(Case): def setup(self): self.log = Log('Log', Mock()) self.logger = self.log.logger def test_debug(self): self.log.debug('debug') self.logger.log.assert_called_with(logging.DEBUG, 'Log - debug') def test_info(self): self.log.info('info') self.logger.log.assert_called_with(logging.INFO, 'Log - info') def test_warning(self): self.log.warn('warning') self.logger.log.assert_called_with(logging.WARN, 'Log - warning') def test_error(self): self.log.error('error', exc_info='exc') self.logger.log.assert_called_with( logging.ERROR, 'Log - error', exc_info='exc', ) def test_critical(self): self.log.critical('crit', exc_info='exc') self.logger.log.assert_called_with( logging.CRITICAL, 'Log - crit', exc_info='exc', ) def test_error_when_DISABLE_TRACEBACKS(self): from kombu import log log.DISABLE_TRACEBACKS = True try: self.log.error('error') self.logger.log.assert_called_with(logging.ERROR, 'Log - error') finally: log.DISABLE_TRACEBACKS = False def test_get_loglevel(self): self.assertEqual(self.log.get_loglevel('DEBUG'), logging.DEBUG) self.assertEqual(self.log.get_loglevel('ERROR'), logging.ERROR) self.assertEqual(self.log.get_loglevel(logging.INFO), logging.INFO) def test_is_enabled_for(self): self.logger.isEnabledFor.return_value = True self.assertTrue(self.log.is_enabled_for('DEBUG')) self.logger.isEnabledFor.assert_called_with(logging.DEBUG) def test_LogMixin_get_logger(self): self.assertIs(LogMixin().get_logger(), logging.getLogger('LogMixin')) def test_Log_get_logger(self): self.assertIs(Log('test_Log').get_logger(), logging.getLogger('test_Log')) def test_log_when_not_enabled(self): self.logger.isEnabledFor.return_value = False self.log.debug('debug') self.assertFalse(self.logger.log.called) def test_log_with_format(self): self.log.debug('Host %r removed', 'example.com') self.logger.log.assert_called_with( logging.DEBUG, 'Log - Host %s removed', ANY, ) self.assertEqual( self.logger.log.call_args[0][2].strip('u'), "'example.com'", )
class Actor: AsyncResult = AsyncResult Error = exceptions.CellError Next = exceptions.Next NoReplyError = exceptions.NoReplyError NoRouteError = exceptions.NoRouteError NotBoundError = exceptions.NotBoundError #: Actor name. #: Defaults to the defined class name. name = None #: Default exchange(direct) used for messages to this actor. exchange = None #: Default routing key used if no ``to`` argument passed. default_routing_key = None #: Delivery mode: persistent or transient. Default is persistent. delivery_mode = 'persistent' #: Set to True to disable acks. no_ack = False #: List of calling types this actor should handle. #: Valid types are: #: #: * direct #: Send the message directly to an agent by exact routing key. #: * round-robin #: Send the message to an agent by round-robin. #: * scatter #: Send the message to all of the agents (broadcast). types = (ACTOR_TYPE.DIRECT, ACTOR_TYPE.SCATTER, ACTOR_TYPE.RR) #: Default serializer used to send messages and reply messages. serializer = 'json' #: Default timeout in seconds as a float which after #: we give up waiting for replies. default_timeout = 5.0 #: Time in seconds as a float which after replies expires. reply_expires = 100.0 #: Exchange used for replies. reply_exchange = Exchange('cl.reply', 'direct') #: Exchange used for forwarding/binding with other actors. outbox_exchange = None #: Exchange used for receiving broadcast commands for this actor type. _scatter_exchange = None #: Exchange used for round-robin commands for this actor type. _rr_exchange = None #: Should we retry publishing messages by default? #: Default: NO retry = None #: time-to-live for the actor before becoming Idle ttl = 20 idle = 40 #: Default policy used when retrying publishing messages. #: see :meth:`kombu.BrokerConnection.ensure` for a list #: of supported keys. retry_policy = { 'max_retries': 100, 'interval_start': 0, 'interval_max': 1, 'interval_step': 0.2 } #: returns the next anonymous ticket number #: used fo+r identifying related logs. ticket_count = count(1) #: Additional fields added to reply messages by default. default_fields = {} #: Map of calling types and their special routing keys. type_to_rkey = { 'rr': '__rr__', ACTOR_TYPE.RR: '__rr__', ACTOR_TYPE.SCATTER: '__scatter__' } meta = {} consumer = None class state: """Placeholder class for actor's supported methods.""" pass def __init__(self, connection=None, id=None, name=None, exchange=None, logger=None, agent=None, outbox_exchange=None, group_exchange=None, **kwargs): self.connection = connection self.id = id or uuid() self.name = name or self.name or self.__class__.__name__ self.outbox_exchange = outbox_exchange or self.outbox_exchange self.agent = agent if self.default_fields is None: self.default_fields = {} # - setup exchanges and queues self.exchange = exchange or self.get_direct_exchange() if group_exchange: self._scatter_exchange = Exchange(group_exchange, 'fanout', auto_delete=True) typemap = { ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct], ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr], ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter] } self.type_to_queue = {k: v[0] for k, v in items(typemap)} self.type_to_exchange = {k: v[1] for k, v in items(typemap)} if not self.outbox_exchange: self.outbox_exchange = Exchange( 'cl.%s.output' % self.name, type='topic', ) # - setup logging logger_name = self.name if self.agent: logger_name = '%s#%s' % (self.name, shortuuid(self.id)) self.log = Log('!<%s>' % logger_name, logger=logger) self.state = self.contribute_to_state(self.construct_state()) # actor specific initialization. self.construct() def _add_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): source_exchange = Exchange(**source) binder = self.get_binder(inbox_type) maybe_declare(source_exchange, self.connection.default_channel) binder(exchange=source_exchange, routing_key=routing_key) def _remove_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): source_exchange = Exchange(**source) unbinder = self.get_unbinder(inbox_type) unbinder(exchange=source_exchange, routing_key=routing_key) def get_binder(self, type): if type == ACTOR_TYPE.DIRECT: entity = self.type_to_queue[type]() elif type in self.types: entity = self.type_to_exchange[type]() else: raise ValueError('Unsupported type: {0}'.format(type)) binder = entity.bind_to # @TODO: Declare probably should not happened here entity.maybe_bind(self.connection.default_channel) maybe_declare(entity, entity.channel) return binder def get_unbinder(self, type): if type == ACTOR_TYPE.DIRECT: entity = self.type_to_queue[type]() unbinder = entity.unbind_from else: entity = self.type_to_exchange[type]() unbinder = entity.exchange_unbind entity = entity.maybe_bind(self.connection.default_channel) # @TODO: Declare probably should not happened here return unbinder def add_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): self.call('add_binding', { 'source': source.as_dict(), 'routing_key': routing_key, 'inbox_type': inbox_type, }, type=ACTOR_TYPE.DIRECT) def remove_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): self.call('remove_binding', { 'source': source.as_dict(), 'routing_key': routing_key, 'inbox_type': inbox_type, }, type=ACTOR_TYPE.DIRECT) def construct(self): """Actor specific initialization.""" pass def construct_state(self): """Instantiates the state class of this actor.""" return self.state() def on_agent_ready(self): pass def contribute_to_object(self, obj, map): for attr, value in items(map): setattr_default(obj, attr, value) return obj def contribute_to_state(self, state): try: contribute = state.contribute_to_state except AttributeError: # set default state attributes. return self.contribute_to_object( state, { 'actor': self, 'agent': self.agent, 'connection': self.connection, 'log': self.log, 'Next': self.Next, 'NoRouteError': self.NoRouteError, 'NoReplyError': self.NoReplyError, 'add_binding': self._add_binding, 'remove_binding': self._remove_binding, }) else: return contribute(self) def send(self, method, args={}, to=None, nowait=False, **kwargs): """Call method on agent listening to ``routing_key``. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. j """ if to is None: to = self.routing_key r = self.call_or_cast(method, args, routing_key=to, nowait=nowait, **kwargs) if not nowait: return r.get() def throw(self, method, args={}, nowait=False, **kwargs): """Call method on one of the agents in round robin. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. """ r = self.call_or_cast(method, args, type=ACTOR_TYPE.RR, nowait=nowait, **kwargs) if not nowait: return r def scatter(self, method, args={}, nowait=False, timeout=None, **kwargs): """Broadcast method to all agents. if nowait is False, returns generator to iterate over the results. :keyword limit: Limit number of reads from the queue. Unlimited by default. :keyword timeout: the timeout (in float seconds) waiting for replies. Default is :attr:`default_timeout`. **Examples** ``scatter`` is a generator (if nowait is False):: >>> res = scatter() >>> res.next() # one event consumed, or timed out. >>> res = scatter(limit=2): >>> for i in res: # two events consumed or timeout >>> pass See :meth:`call_or_cast` for a full list of supported arguments. """ timeout = timeout if timeout is not None else self.default_timeout r = self.call_or_cast(method, args, type=ACTOR_TYPE.SCATTER, nowait=nowait, timeout=timeout, **kwargs) if not nowait: return r.gather(timeout=timeout, **kwargs) def call_or_cast(self, method, args={}, nowait=False, **kwargs): """Apply remote `method` asynchronously or synchronously depending on the value of `nowait`. :param method: The name of the remote method to perform. :param args: Dictionary of arguments for the method. :keyword nowait: If false the call will block until the result is available and return it (default), if true the call will be non-blocking and no result will be returned. :keyword retry: If set to true then message sending will be retried in the event of connection failures. Default is decided by the :attr:`retry` attributed. :keyword retry_policy: Override retry policies. See :attr:`retry_policy`. This must be a dictionary, and keys will be merged with the default retry policy. :keyword timeout: Timeout to wait for replies in seconds as a float (**only relevant in blocking mode**). :keyword limit: Limit number of replies to wait for (**only relevant in blocking mode**). :keyword callback: If provided, this callback will be called for every reply received (**only relevant in blocking mode**). :keyword \*\*props: Additional message properties. See :meth:`kombu.Producer.publish`. """ return (nowait and self.cast or self.call)(method, args, **kwargs) def get_scatter_exchange(self): """Returns a :class:'kombu.Exchange' for type fanout""" return Exchange('cl.scatter.%s' % self.name, 'fanout', auto_delete=True) def get_rr_exchange(self): """Returns a :class:'kombu.Exchange' instance with type set to fanout. The exchange is used for sending in a round-robin style""" return Exchange('cl.rr.%s' % self.name, 'fanout', auto_delete=True) def get_direct_exchange(self): """Returns a :class:'kombu.Exchange' with type direct""" return Exchange('cl.%s' % self.name, 'direct', auto_delete=True) def get_queues(self): return [self.type_to_queue[type]() for type in self.types] def get_direct_queue(self): """Returns a :class: `kombu.Queue` instance to be used to listen for messages send to this specific Actor instance""" return Queue(self.id, self.inbox_direct, routing_key=self.routing_key, auto_delete=True) def get_scatter_queue(self): """Returns a :class: `kombu.Queue` instance for receiving broadcast commands for this actor type.""" return Queue('%s.%s.scatter' % (self.name, self.id), self.inbox_scatter, auto_delete=True) def get_rr_queue(self): """Returns a :class: `kombu.Queue` instance for receiving round-robin commands for this actor type.""" return Queue(self.inbox_rr.name + '.rr', self.inbox_rr, auto_delete=True) def get_reply_queue(self, ticket): return Queue( ticket, self.reply_exchange, ticket, auto_delete=True, queue_arguments={'x-expires': int(self.reply_expires * 1000)}) def Consumer(self, channel, **kwargs): """Returns a :class:`kombu.Consumer` instance for this Actor""" kwargs.setdefault('no_ack', self.no_ack) return Consumer(channel, self.get_queues(), callbacks=[self.on_message], **kwargs) def emit(self, method, args={}, retry=None): return self.cast(method, args, retry=retry, exchange=self.outbox) def cast(self, method, args={}, declare=None, retry=None, retry_policy=None, type=None, exchange=None, **props): """Send message to actor. Discarding replies.""" retry = self.retry if retry is None else retry body = {'class': self.name, 'method': method, 'args': args} _retry_policy = self.retry_policy if retry_policy: # merge default and custom policies. _retry_policy = dict(_retry_policy, **retry_policy) if type and type not in self.types: raise ValueError('Unsupported type: {0}'.format(type)) elif not type: type = ACTOR_TYPE.DIRECT props.setdefault('routing_key', self.routing_key) props.setdefault('serializer', self.serializer) exchange = exchange or self.type_to_exchange[type]() declare = (maybe_list(declare) or []) + [exchange] with producers[self._connection].acquire(block=True) as producer: return producer.publish(body, exchange=exchange, declare=declare, retry=retry, retry_policy=retry_policy, **props) def call(self, method, args={}, retry=False, retry_policy=None, ticket=None, **props): """Send message to the same actor and return :class:`AsyncResult`.""" ticket = ticket or uuid() reply_q = self.get_reply_queue(ticket) self.cast(method, args, declare=[reply_q], reply_to=ticket, **props) return self.AsyncResult(ticket, self) def handle_cast(self, body, message): """Handle cast message.""" self._DISPATCH(body) def handle_call(self, body, message): """Handle call message.""" try: r = self._DISPATCH(body, ticket=message.properties['reply_to']) except self.Next: # don't reply, delegate to other agents. pass else: self.reply(message, r) def reply(self, req, body, **props): with producers[self._connection].acquire(block=True) as producer: content_type = req.content_type serializer = serialization.registry.type_to_name[content_type] return producer.publish( body, declare=[self.reply_exchange], routing_key=req.properties['reply_to'], correlation_id=req.properties.get('correlation_id'), serializer=serializer, **props) def on_message(self, body, message): self.agent.process_message(self, body, message) def _on_message(self, body, message): """What to do when a message is received. This is a kombu consumer callback taking the standard ``body`` and ``message`` arguments. Note that if the properties of the message contains a value for ``reply_to`` then a proper implementation is expected to send a reply. """ if message.properties.get('reply_to'): handler = self.handle_call else: handler = self.handle_cast def handle(): # Do not ack the message if an exceptional error occurs, # but do ack the message if SystemExit or KeyboardInterrupt # is raised, as this is probably intended. try: handler(body, message) except Exception: raise except BaseException: message.ack() raise else: message.ack() handle() def _collect_replies(self, conn, channel, ticket, *args, **kwargs): kwargs.setdefault('timeout', self.default_timeout) if 'limit' not in kwargs and self.agent: kwargs['limit'] = self.agent.get_default_scatter_limit() if 'ignore_timeout' not in kwargs and not kwargs.get('limit', None): kwargs.setdefault('ignore_timeout', False) return collect_replies(conn, channel, self.get_reply_queue(ticket), *args, **kwargs) def lookup_action(self, name): try: if not name: method = self.default_receive else: method = getattr(self.state, name) except AttributeError: raise KeyError(name) if not callable(method) or name.startswith('_'): raise KeyError(method) return method def default_receive(self, msg_body): """Override in the derived classes.""" pass def _DISPATCH(self, body, ticket=None): """Dispatch message to the appropriate method in :attr:`state`, handle possible exceptions, and return a response suitable to be used in a reply. To protect from calling special methods it does not dispatch method names starting with underscore (``_``). This returns the return value or exception error with defaults fields in a suitable format to be used as a reply. The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt` will not be handled, and will propagate. In the case of a successful call the return value will be:: {'ok': return_value, **default_fields} If the method raised an exception the return value will be:: {'nok': [repr exc, str traceback], **default_fields} :raises KeyError: if the method specified is unknown or is a special method (name starting with underscore). """ if ticket: sticket = '%s' % (shortuuid(ticket), ) else: ticket = sticket = str(next(self.ticket_counter)) try: method, args = itemgetter('method', 'args')(body) self.log.info('#%s --> %s', sticket, self._reprcall(method, args)) act = self.lookup_action(method) r = {'ok': act(args or {})} self.log.info('#%s <-- %s', sticket, reprkwargs(r)) except self.Next: raise except Exception as exc: einfo = sys.exc_info() r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]} self.log.error('#%s <-- nok=%r', sticket, exc) return dict(self._default_fields, **r) def _get_traceback(self, exc_info): return ''.join(traceback.format_exception(*exc_info)) def _reprcall(self, method, args): return '%s.%s' % (self.name, reprcall(method, (), args)) def bind(self, connection, agent=None): return self.__class__(connection, self.id, self.name, self.exchange, agent=agent) def is_bound(self): return self.connection is not None def __copy__(self): cls, args = self.__reduce__() return cls(*args) def __reduce__(self): return (self.__class__, (self.connection, self.id, self.name, self.exchange)) @property def outbox(self): return self.outbox_exchange def _inbox_rr(self): if not self._rr_exchange: self._rr_exchange = self.get_rr_exchange() return self._rr_exchange @property def inbox_rr(self): return self._inbox_rr() def _inbox_direct(self): return self.exchange @property def inbox_direct(self): return self._inbox_direct() def _inbox_scatter(self): if not self._scatter_exchange: self._scatter_exchange = self.get_scatter_exchange() return self._scatter_exchange @property def inbox_scatter(self): return self._inbox_scatter() @property def _connection(self): if not self.is_bound(): raise self.NotBoundError('Actor is not bound to any connection.') return self.connection @cached_property def _default_fields(self): return dict(BUILTIN_FIELDS, **self.default_fields) @property def routing_key(self): if self.default_routing_key: return self.default_routing_key else: return self.id
class test_LogMixin(Case): def setUp(self): self.log = Log('Log', Mock()) self.logger = self.log.logger def test_debug(self): self.log.debug('debug') self.logger.log.assert_called_with(logging.DEBUG, 'Log - debug') def test_info(self): self.log.info('info') self.logger.log.assert_called_with(logging.INFO, 'Log - info') def test_warning(self): self.log.warn('warning') self.logger.log.assert_called_with(logging.WARN, 'Log - warning') def test_error(self): self.log.error('error', exc_info='exc') self.logger.log.assert_called_with( logging.ERROR, 'Log - error', exc_info='exc', ) def test_critical(self): self.log.critical('crit', exc_info='exc') self.logger.log.assert_called_with( logging.CRITICAL, 'Log - crit', exc_info='exc', ) def test_error_when_DISABLE_TRACEBACKS(self): from kombu import log log.DISABLE_TRACEBACKS = True try: self.log.error('error') self.logger.log.assert_called_with(logging.ERROR, 'Log - error') finally: log.DISABLE_TRACEBACKS = False def test_get_loglevel(self): self.assertEqual(self.log.get_loglevel('DEBUG'), logging.DEBUG) self.assertEqual(self.log.get_loglevel('ERROR'), logging.ERROR) self.assertEqual(self.log.get_loglevel(logging.INFO), logging.INFO) def test_is_enabled_for(self): self.logger.isEnabledFor.return_value = True self.assertTrue(self.log.is_enabled_for('DEBUG')) self.logger.isEnabledFor.assert_called_with(logging.DEBUG) def test_LogMixin_get_logger(self): self.assertIs(LogMixin().get_logger(), logging.getLogger('LogMixin')) def test_Log_get_logger(self): self.assertIs(Log('test_Log').get_logger(), logging.getLogger('test_Log')) def test_log_when_not_enabled(self): self.logger.isEnabledFor.return_value = False self.log.debug('debug') self.assertFalse(self.logger.log.called) def test_log_with_format(self): self.log.debug('Host %r removed', 'example.com') self.logger.log.assert_called_with( logging.DEBUG, 'Log - Host %s removed', "'example.com'", )
class Actor(object, metaclass=ActorType): AsyncResult = AsyncResult Error = exceptions.CellError Next = exceptions.Next NoReplyError = exceptions.NoReplyError NoRouteError = exceptions.NoRouteError NotBoundError = exceptions.NotBoundError #: Actor name. #: Defaults to the defined class name. name = None #: Default exchange used for messages to this actor. exchange = None #: Default routing key used if no ``to`` argument passed. default_routing_key = None #: Delivery mode: persistent or transient. Default is persistent. delivery_mode = 'persistent' #: Set to True to disable acks. no_ack = False #: List of calling types this actor should handle. #: Valid types are: #: #: * direct #: Send the message directly to an agent by exact routing key. #: * round-robin #: Send the message to an agent by round-robin. #: * scatter #: Send the message to all of the agents (broadcast). types = ('direct', ) #: Default serializer used to send messages and reply messages. serializer = 'json' #: Default timeout in seconds as a float which after #: we give up waiting for replies. default_timeout = 10.0 #: Time in seconds as a float which after replies expires. reply_expires = 100.0 #: Exchanged used for replies. reply_exchange = Exchange('cl.reply', 'direct') #: Should we retry publishing messages by default? #: Default: NO retry = None #: Default policy used when retrying publishing messages. #: see :meth:`kombu.BrokerConnection.ensure` for a list #: of supported keys. retry_policy = { 'max_retries': 100, 'interval_start': 0, 'interval_max': 1, 'interval_step': 0.2 } #: returns the next anonymous ticket number #: used for identifying related logs. next_anon_ticket = count(1).__next__ #: Additional fields added to reply messages by default. default_fields = {} #: Map of calling types and their special routing keys. type_to_rkey = { 'rr': '__rr__', 'round-robin': '__rr__', 'scatter': '__scatter__' } meta = {} class state: pass def __init__(self, connection=None, id=None, name=None, exchange=None, logger=None, agent=None, **kwargs): self.connection = connection self.id = id or uuid() self.name = name or self.name or self.__class__.__name__ self.exchange = exchange or self.exchange self.agent = agent self.type_to_queue = { 'direct': self.get_direct_queue, 'round-robin': self.get_rr_queue, 'scatter': self.get_scatter_queue } if self.default_fields is None: self.default_fields = {} if not self.exchange: self.exchange = Exchange('cl.%s' % (self.name, ), 'direct', auto_delete=True) logger_name = self.name if self.agent: logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, )) self.log = Log('!<%s>' % (logger_name, ), logger=logger) self.state = self.contribute_to_state(self.construct_state()) self.setup() def setup(self): pass def construct_state(self): """Instantiates the state class of this actor.""" return self.state() def maybe_setattr(self, obj, attr, value): if not hasattr(obj, attr): setattr(obj, attr, value) def on_agent_ready(self): pass def contribute_to_object(self, obj, map): for attr, value in map.items(): self.maybe_setattr(obj, attr, value) return obj def contribute_to_state(self, state): try: contribute = state.contribute_to_state except AttributeError: return self.contribute_to_object( state, { 'actor': self, 'agent': self.agent, 'log': self.log, 'Next': self.Next, 'NoRouteError': self.NoRouteError, 'NoReplyError': self.NoReplyError }) else: return contribute(self) def send(self, method, args={}, to=None, nowait=False, **kwargs): """Call method on agent listening to ``routing_key``. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. """ if to is None: to = self.routing_key r = self.call_or_cast(method, args, routing_key=to, nowait=nowait, **kwargs) if not nowait: return r.get() def throw(self, method, args={}, nowait=False, **kwargs): """Call method on one of the agents in round robin. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. """ r = self.call_or_cast(method, args, type='round-robin', nowait=nowait, **kwargs) if not nowait: return r.get() def scatter(self, method, args={}, nowait=False, **kwargs): """Broadcast method to all agents. In this context the reply limit is disabled, and the timeout is set to 1 by default, which means we collect all the replies that managed to be sent within the requested timeout. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the replies. """ kwargs.setdefault('timeout', 2) r = self.call_or_cast(method, args, type='scatter', nowait=nowait, **kwargs) if not nowait: return r.gather(**kwargs) def get_default_scatter_limit(self): if self.agent: return self.agent.get_default_scatter_limit(self.name) return None def call_or_cast(self, method, args={}, nowait=False, **kwargs): """Apply remote `method` asynchronously or synchronously depending on the value of `nowait`. :param method: The name of the remote method to perform. :keyword args: Dictionary of arguments for the method. :keyword nowait: If false the call will be block until the result is available and return it (default), if true the call will be non-blocking. :keyword retry: If set to true then message sending will be retried in the event of connection failures. Default is decided by the :attr:`retry` attributed. :keyword retry_policy: Override retry policies. See :attr:`retry_policy`. This must be a dictionary, and keys will be merged with the default retry policy. :keyword timeout: Timeout to wait for replies in seconds as a float (**only relevant in blocking mode**). :keyword limit: Limit number of replies to wait for (**only relevant in blocking mode**). :keyword callback: If provided, this callback will be called for every reply received (**only relevant in blocking mode**). :keyword \*\*props: Additional message properties. See :meth:`kombu.Producer.publish`. """ return (nowait and self.cast or self.call)(method, args, **kwargs) def get_queues(self): return [self.type_to_queue[type]() for type in self.types] def get_direct_queue(self): """Returns a unique queue that can be used to listen for messages to this class.""" return Queue(self.id, self.exchange, routing_key=self.routing_key, auto_delete=True) def get_scatter_queue(self): return Queue('%s.%s.scatter' % (self.name, self.id), self.exchange, routing_key=self.type_to_rkey['scatter'], auto_delete=True) def get_rr_queue(self): return Queue(self.exchange.name + '.rr', self.exchange, routing_key=self.type_to_rkey['round-robin'], auto_delete=True) def get_reply_queue(self, ticket): return Queue( ticket, self.reply_exchange, ticket, auto_delete=True, queue_arguments={'x-expires': int(self.reply_expires * 1000)}) def Consumer(self, channel, **kwargs): """Returns a :class:`kombu.Consumer` instance for this Actor.""" kwargs.setdefault('no_ack', self.no_ack) return Consumer(channel, self.get_queues(), callbacks=[self.on_message], **kwargs) def _publish(self, body, producer, before=None, **props): if before is not None: before(producer.connection, producer.channel) maybe_declare(props['exchange'], producer.channel) return producer.publish(body, **props) def cast(self, method, args={}, before=None, retry=None, retry_policy=None, type=None, **props): """Send message to actor. Discarding replies.""" retry = self.retry if retry is None else retry body = {'class': self.name, 'method': method, 'args': args} exchange = self.exchange _retry_policy = self.retry_policy if retry_policy: # merge default and custom policies. _retry_policy = dict(_retry_policy, **retry_policy) if type: props.setdefault('routing_key', self.type_to_rkey[type]) props.setdefault('routing_key', self.default_routing_key) props.setdefault('serializer', self.serializer) props = dict(props, exchange=exchange, before=before) ipublish(producers[self._connection], self._publish, (body, ), dict(props, exchange=exchange, before=before), **(retry_policy or {})) def call(self, method, args={}, retry=False, retry_policy=None, **props): """Send message to actor and return :class:`AsyncResult`.""" ticket = uuid() reply_q = self.get_reply_queue(ticket) def before(connection, channel): reply_q(channel).declare() self.cast(method, args, before, **dict(props, reply_to=ticket)) return self.AsyncResult(ticket, self) def handle_cast(self, body, message): """Handle cast message.""" self._DISPATCH(body) def handle_call(self, body, message): """Handle call message.""" try: r = self._DISPATCH(body, ticket=message.properties['reply_to']) except self.Next: # don't reply, delegate to other agent. pass else: self.reply(message, r) def reply(self, req, body, **props): return isend_reply(producers[self._connection], self.reply_exchange, req, body, props) def on_message(self, body, message): """What to do when a message is received. This is a kombu consumer callback taking the standard ``body`` and ``message`` arguments. Note that if the properties of the message contains a value for ``reply_to`` then a proper implementation is expected to send a reply. """ if message.properties.get('reply_to'): handler = self.handle_call else: handler = self.handle_cast def handle(): # Do not ack the message if an exceptional error occurs, # but do ack the message if SystemExit or KeyboardInterrupt # is raised, as this is probably intended. try: handler(body, message) except Exception: raise except BaseException: message.ack() raise else: message.ack() handle() def _collect_replies(self, conn, channel, ticket, *args, **kwargs): kwargs.setdefault('timeout', self.default_timeout) if 'limit' not in kwargs: kwargs['limit'] = self.get_default_scatter_limit() return collect_replies(conn, channel, self.get_reply_queue(ticket), *args, **kwargs) def lookup_action(self, name): try: method = getattr(self.state, name) except AttributeError: raise KeyError(name) if not callable(method) or name.startswith('_'): raise KeyError(method) return method def _DISPATCH(self, body, ticket=None): """Dispatch message to the appropriate method in :attr:`state`, handle possible exceptions, and return a response suitable to be used in a reply. To protect from calling special methods it does not dispatch method names starting with underscore (``_``). This returns the return value or exception error with defaults fields in a suitable format to be used as a reply. The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt` will not be handled, and will propagate. In the case of a successful call the return value will be:: {'ok': return_value, **default_fields} If the method raised an exception the return value will be:: {'nok': [repr exc, str traceback], **default_fields} :raises KeyError: if the method specified is unknown or is a special method (name starting with underscore). """ if ticket: sticket = '%s' % (shortuuid(ticket), ) else: ticket = sticket = str(self.next_anon_ticket()) try: method, args = itemgetter('method', 'args')(body) self.log.info('#%s --> %s', sticket, self._reprcall(method, args)) act = self.lookup_action(method) r = {'ok': act(**kwdict(args or {}))} self.log.info('#%s <-- %s', sticket, reprkwargs(r)) except self.Next: raise except Exception as exc: einfo = sys.exc_info() r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]} self.log.error('#%s <-- nok=%r', sticket, exc) return dict(self._default_fields, **r) def _get_traceback(self, exc_info): return ''.join(traceback.format_exception(*exc_info)) def _reprcall(self, method, args): return '%s.%s' % (self.name, reprcall(method, (), args)) def bind(self, connection, agent=None): return self.__class__(connection, self.id, self.name, self.exchange, agent=agent) def is_bound(self): return self.connection is not None def __copy__(self): cls, args = self.__reduce__() return cls(*args) def __reduce__(self): return (self.__class__, (self.connection, self.id, self.name, self.exchange)) @property def _connection(self): if not self.is_bound(): raise self.NotBoundError('Actor is not bound to any connection.') return self.connection @cached_property def _default_fields(self): return dict(builtin_fields, **self.default_fields) @property def routing_key(self): if self.default_routing_key: return self.default_routing_key return self.agent.id
class Actor: AsyncResult = AsyncResult Error = exceptions.CellError Next = exceptions.Next NoReplyError = exceptions.NoReplyError NoRouteError = exceptions.NoRouteError NotBoundError = exceptions.NotBoundError #: Actor name. #: Defaults to the defined class name. name = None #: Default exchange(direct) used for messages to this actor. exchange = None #: Default routing key used if no ``to`` argument passed. default_routing_key = None #: Delivery mode: persistent or transient. Default is persistent. delivery_mode = 'persistent' #: Set to True to disable acks. no_ack = False #: List of calling types this actor should handle. #: Valid types are: #: #: * direct #: Send the message directly to an agent by exact routing key. #: * round-robin #: Send the message to an agent by round-robin. #: * scatter #: Send the message to all of the agents (broadcast). types = (ACTOR_TYPE.DIRECT, ACTOR_TYPE.SCATTER, ACTOR_TYPE.RR) #: Default serializer used to send messages and reply messages. serializer = 'json' #: Default timeout in seconds as a float which after #: we give up waiting for replies. default_timeout = 5.0 #: Time in seconds as a float which after replies expires. reply_expires = 100.0 #: Exchange used for replies. reply_exchange = Exchange('cl.reply', 'direct') #: Exchange used for forwarding/binding with other actors. outbox_exchange = None #: Exchange used for receiving broadcast commands for this actor type. _scatter_exchange = None #: Exchange used for round-robin commands for this actor type. _rr_exchange = None #: Should we retry publishing messages by default? #: Default: NO retry = None #: time-to-live for the actor before becoming Idle ttl = 20 idle = 40 #: Default policy used when retrying publishing messages. #: see :meth:`kombu.BrokerConnection.ensure` for a list #: of supported keys. retry_policy = {'max_retries': 100, 'interval_start': 0, 'interval_max': 1, 'interval_step': 0.2} #: returns the next anonymous ticket number #: used fo+r identifying related logs. ticket_count = count(1) #: Additional fields added to reply messages by default. default_fields = {} #: Map of calling types and their special routing keys. type_to_rkey = {'rr': '__rr__', ACTOR_TYPE.RR: '__rr__', ACTOR_TYPE.SCATTER: '__scatter__'} meta = {} consumer = None class state: """Placeholder class for actor's supported methods.""" pass def __init__(self, connection=None, id=None, name=None, exchange=None, logger=None, agent=None, outbox_exchange=None, group_exchange=None, **kwargs): self.connection = connection self.id = id or uuid() self.name = name or self.name or self.__class__.__name__ self.outbox_exchange = outbox_exchange or self.outbox_exchange self.agent = agent if self.default_fields is None: self.default_fields = {} # - setup exchanges and queues self.exchange = exchange or self.get_direct_exchange() if group_exchange: self._scatter_exchange = Exchange( group_exchange, 'fanout', auto_delete=True) typemap = { ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct], ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr], ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter] } self.type_to_queue = {k: v[0] for k, v in items(typemap)} self.type_to_exchange = {k: v[1] for k, v in items(typemap)} if not self.outbox_exchange: self.outbox_exchange = Exchange( 'cl.%s.output' % self.name, type='topic', ) # - setup logging logger_name = self.name if self.agent: logger_name = '%s#%s' % (self.name, shortuuid(self.id)) self.log = Log('!<%s>' % logger_name, logger=logger) self.state = self.contribute_to_state(self.construct_state()) # actor specific initialization. self.construct() def _add_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): source_exchange = Exchange(**source) binder = self.get_binder(inbox_type) maybe_declare(source_exchange, self.connection.default_channel) binder(exchange=source_exchange, routing_key=routing_key) def _remove_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): source_exchange = Exchange(**source) unbinder = self.get_unbinder(inbox_type) unbinder(exchange=source_exchange, routing_key=routing_key) def get_binder(self, type): if type == ACTOR_TYPE.DIRECT: entity = self.type_to_queue[type]() elif type in self.types: entity = self.type_to_exchange[type]() else: raise ValueError('Unsupported type: {0}'.format(type)) binder = entity.bind_to # @TODO: Declare probably should not happened here entity.maybe_bind(self.connection.default_channel) maybe_declare(entity, entity.channel) return binder def get_unbinder(self, type): if type == ACTOR_TYPE.DIRECT: entity = self.type_to_queue[type]() unbinder = entity.unbind_from else: entity = self.type_to_exchange[type]() unbinder = entity.exchange_unbind entity = entity.maybe_bind(self.connection.default_channel) # @TODO: Declare probably should not happened here return unbinder def add_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): self.call('add_binding', { 'source': source.as_dict(), 'routing_key': routing_key, 'inbox_type': inbox_type, }, type=ACTOR_TYPE.DIRECT) def remove_binding(self, source, routing_key='', inbox_type=ACTOR_TYPE.DIRECT): self.call('remove_binding', { 'source': source.as_dict(), 'routing_key': routing_key, 'inbox_type': inbox_type, }, type=ACTOR_TYPE.DIRECT) def construct(self): """Actor specific initialization.""" pass def construct_state(self): """Instantiates the state class of this actor.""" return self.state() def on_agent_ready(self): pass def contribute_to_object(self, obj, map): for attr, value in items(map): setattr_default(obj, attr, value) return obj def contribute_to_state(self, state): try: contribute = state.contribute_to_state except AttributeError: # set default state attributes. return self.contribute_to_object(state, { 'actor': self, 'agent': self.agent, 'connection': self.connection, 'log': self.log, 'Next': self.Next, 'NoRouteError': self.NoRouteError, 'NoReplyError': self.NoReplyError, 'add_binding': self._add_binding, 'remove_binding': self._remove_binding, }) else: return contribute(self) def send(self, method, args={}, to=None, nowait=False, **kwargs): """Call method on agent listening to ``routing_key``. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. j """ if to is None: to = self.routing_key r = self.call_or_cast(method, args, routing_key=to, nowait=nowait, **kwargs) if not nowait: return r.get() def throw(self, method, args={}, nowait=False, **kwargs): """Call method on one of the agents in round robin. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. """ r = self.call_or_cast(method, args, type=ACTOR_TYPE.RR, nowait=nowait, **kwargs) if not nowait: return r def scatter(self, method, args={}, nowait=False, timeout=None, **kwargs): """Broadcast method to all agents. if nowait is False, returns generator to iterate over the results. :keyword limit: Limit number of reads from the queue. Unlimited by default. :keyword timeout: the timeout (in float seconds) waiting for replies. Default is :attr:`default_timeout`. **Examples** ``scatter`` is a generator (if nowait is False):: >>> res = scatter() >>> res.next() # one event consumed, or timed out. >>> res = scatter(limit=2): >>> for i in res: # two events consumed or timeout >>> pass See :meth:`call_or_cast` for a full list of supported arguments. """ timeout = timeout if timeout is not None else self.default_timeout r = self.call_or_cast(method, args, type=ACTOR_TYPE.SCATTER, nowait=nowait, timeout=timeout, **kwargs) if not nowait: return r.gather(timeout=timeout, **kwargs) def call_or_cast(self, method, args={}, nowait=False, **kwargs): """Apply remote `method` asynchronously or synchronously depending on the value of `nowait`. :param method: The name of the remote method to perform. :param args: Dictionary of arguments for the method. :keyword nowait: If false the call will block until the result is available and return it (default), if true the call will be non-blocking and no result will be returned. :keyword retry: If set to true then message sending will be retried in the event of connection failures. Default is decided by the :attr:`retry` attributed. :keyword retry_policy: Override retry policies. See :attr:`retry_policy`. This must be a dictionary, and keys will be merged with the default retry policy. :keyword timeout: Timeout to wait for replies in seconds as a float (**only relevant in blocking mode**). :keyword limit: Limit number of replies to wait for (**only relevant in blocking mode**). :keyword callback: If provided, this callback will be called for every reply received (**only relevant in blocking mode**). :keyword \*\*props: Additional message properties. See :meth:`kombu.Producer.publish`. """ return (nowait and self.cast or self.call)(method, args, **kwargs) def get_scatter_exchange(self): """Returns a :class:'kombu.Exchange' for type fanout""" return Exchange('cl.scatter.%s' % self.name, 'fanout', auto_delete=True) def get_rr_exchange(self): """Returns a :class:'kombu.Exchange' instance with type set to fanout. The exchange is used for sending in a round-robin style""" return Exchange('cl.rr.%s' % self.name, 'fanout', auto_delete=True) def get_direct_exchange(self): """Returns a :class:'kombu.Exchange' with type direct""" return Exchange('cl.%s' % self.name, 'direct', auto_delete=True) def get_queues(self): return [self.type_to_queue[type]() for type in self.types] def get_direct_queue(self): """Returns a :class: `kombu.Queue` instance to be used to listen for messages send to this specific Actor instance""" return Queue(self.id, self.inbox_direct, routing_key=self.routing_key, auto_delete=True) def get_scatter_queue(self): """Returns a :class: `kombu.Queue` instance for receiving broadcast commands for this actor type.""" return Queue('%s.%s.scatter' % (self.name, self.id), self.inbox_scatter, auto_delete=True) def get_rr_queue(self): """Returns a :class: `kombu.Queue` instance for receiving round-robin commands for this actor type.""" return Queue(self.inbox_rr.name + '.rr', self.inbox_rr, auto_delete=True) def get_reply_queue(self, ticket): return Queue(ticket, self.reply_exchange, ticket, auto_delete=True, queue_arguments={ 'x-expires': int(self.reply_expires * 1000)}) def Consumer(self, channel, **kwargs): """Returns a :class:`kombu.Consumer` instance for this Actor""" kwargs.setdefault('no_ack', self.no_ack) return Consumer(channel, self.get_queues(), callbacks=[self.on_message], **kwargs) def emit(self, method, args={}, retry=None): return self.cast(method, args, retry=retry, exchange=self.outbox) def cast(self, method, args={}, declare=None, retry=None, retry_policy=None, type=None, exchange=None, **props): """Send message to actor. Discarding replies.""" retry = self.retry if retry is None else retry body = {'class': self.name, 'method': method, 'args': args} _retry_policy = self.retry_policy if retry_policy: # merge default and custom policies. _retry_policy = dict(_retry_policy, **retry_policy) if type and type not in self.types: raise ValueError('Unsupported type: {0}'.format(type)) elif not type: type = ACTOR_TYPE.DIRECT props.setdefault('routing_key', self.routing_key) props.setdefault('serializer', self.serializer) exchange = exchange or self.type_to_exchange[type]() declare = (maybe_list(declare) or []) + [exchange] with producers[self._connection].acquire(block=True) as producer: return producer.publish(body, exchange=exchange, declare=declare, retry=retry, retry_policy=retry_policy, **props) def call(self, method, args={}, retry=False, retry_policy=None, ticket=None, **props): """Send message to the same actor and return :class:`AsyncResult`.""" ticket = ticket or uuid() reply_q = self.get_reply_queue(ticket) self.cast(method, args, declare=[reply_q], reply_to=ticket, **props) return self.AsyncResult(ticket, self) def handle_cast(self, body, message): """Handle cast message.""" self._DISPATCH(body) def handle_call(self, body, message): """Handle call message.""" try: r = self._DISPATCH(body, ticket=message.properties['reply_to']) except self.Next: # don't reply, delegate to other agents. pass else: self.reply(message, r) def reply(self, req, body, **props): with producers[self._connection].acquire(block=True) as producer: content_type = req.content_type serializer = serialization.registry.type_to_name[content_type] return producer.publish( body, declare=[self.reply_exchange], routing_key=req.properties['reply_to'], correlation_id=req.properties.get('correlation_id'), serializer=serializer, **props ) def on_message(self, body, message): self.agent.process_message(self, body, message) def _on_message(self, body, message): """What to do when a message is received. This is a kombu consumer callback taking the standard ``body`` and ``message`` arguments. Note that if the properties of the message contains a value for ``reply_to`` then a proper implementation is expected to send a reply. """ if message.properties.get('reply_to'): handler = self.handle_call else: handler = self.handle_cast def handle(): # Do not ack the message if an exceptional error occurs, # but do ack the message if SystemExit or KeyboardInterrupt # is raised, as this is probably intended. try: handler(body, message) except Exception: raise except BaseException: message.ack() raise else: message.ack() handle() def _collect_replies(self, conn, channel, ticket, *args, **kwargs): kwargs.setdefault('timeout', self.default_timeout) if 'limit' not in kwargs and self.agent: kwargs['limit'] = self.agent.get_default_scatter_limit() if 'ignore_timeout' not in kwargs and not kwargs.get('limit', None): kwargs.setdefault('ignore_timeout', False) return collect_replies(conn, channel, self.get_reply_queue(ticket), *args, **kwargs) def lookup_action(self, name): try: if not name: method = self.default_receive else: method = getattr(self.state, name) except AttributeError: raise KeyError(name) if not callable(method) or name.startswith('_'): raise KeyError(method) return method def default_receive(self, msg_body): """Override in the derived classes.""" pass def _DISPATCH(self, body, ticket=None): """Dispatch message to the appropriate method in :attr:`state`, handle possible exceptions, and return a response suitable to be used in a reply. To protect from calling special methods it does not dispatch method names starting with underscore (``_``). This returns the return value or exception error with defaults fields in a suitable format to be used as a reply. The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt` will not be handled, and will propagate. In the case of a successful call the return value will be:: {'ok': return_value, **default_fields} If the method raised an exception the return value will be:: {'nok': [repr exc, str traceback], **default_fields} :raises KeyError: if the method specified is unknown or is a special method (name starting with underscore). """ if ticket: sticket = '%s' % (shortuuid(ticket), ) else: ticket = sticket = str(next(self.ticket_counter)) try: method, args = itemgetter('method', 'args')(body) self.log.info('#%s --> %s', sticket, self._reprcall(method, args)) act = self.lookup_action(method) r = {'ok': act(args or {})} self.log.info('#%s <-- %s', sticket, reprkwargs(r)) except self.Next: raise except Exception as exc: einfo = sys.exc_info() r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]} self.log.error('#%s <-- nok=%r', sticket, exc) return dict(self._default_fields, **r) def _get_traceback(self, exc_info): return ''.join(traceback.format_exception(*exc_info)) def _reprcall(self, method, args): return '%s.%s' % (self.name, reprcall(method, (), args)) def bind(self, connection, agent=None): return self.__class__(connection, self.id, self.name, self.exchange, agent=agent) def is_bound(self): return self.connection is not None def __copy__(self): cls, args = self.__reduce__() return cls(*args) def __reduce__(self): return (self.__class__, (self.connection, self.id, self.name, self.exchange)) @property def outbox(self): return self.outbox_exchange def _inbox_rr(self): if not self._rr_exchange: self._rr_exchange = self.get_rr_exchange() return self._rr_exchange @property def inbox_rr(self): return self._inbox_rr() def _inbox_direct(self): return self.exchange @property def inbox_direct(self): return self._inbox_direct() def _inbox_scatter(self): if not self._scatter_exchange: self._scatter_exchange = self.get_scatter_exchange() return self._scatter_exchange @property def inbox_scatter(self): return self._inbox_scatter() @property def _connection(self): if not self.is_bound(): raise self.NotBoundError('Actor is not bound to any connection.') return self.connection @cached_property def _default_fields(self): return dict(BUILTIN_FIELDS, **self.default_fields) @property def routing_key(self): if self.default_routing_key: return self.default_routing_key else: return self.id
class Actor(object): __metaclass__ = ActorType AsyncResult = AsyncResult Error = exceptions.CellError Next = exceptions.Next NoReplyError = exceptions.NoReplyError NoRouteError = exceptions.NoRouteError NotBoundError = exceptions.NotBoundError #: Actor name. #: Defaults to the defined class name. name = None #: Default exchange used for messages to this actor. exchange = None #: Default routing key used if no ``to`` argument passed. default_routing_key = None #: Delivery mode: persistent or transient. Default is persistent. delivery_mode = 'persistent' #: Set to True to disable acks. no_ack = False #: List of calling types this actor should handle. #: Valid types are: #: #: * direct #: Send the message directly to an agent by exact routing key. #: * round-robin #: Send the message to an agent by round-robin. #: * scatter #: Send the message to all of the agents (broadcast). types = ('direct', ) #: Default serializer used to send messages and reply messages. serializer = 'json' #: Default timeout in seconds as a float which after #: we give up waiting for replies. default_timeout = 10.0 #: Time in seconds as a float which after replies expires. reply_expires = 100.0 #: Exchanged used for replies. reply_exchange = Exchange('cl.reply', 'direct') #: Should we retry publishing messages by default? #: Default: NO retry = None #: Default policy used when retrying publishing messages. #: see :meth:`kombu.BrokerConnection.ensure` for a list #: of supported keys. retry_policy = {'max_retries': 100, 'interval_start': 0, 'interval_max': 1, 'interval_step': 0.2} #: returns the next anonymous ticket number #: used for identifying related logs. next_anon_ticket = count(1).next #: Additional fields added to reply messages by default. default_fields = {} #: Map of calling types and their special routing keys. type_to_rkey = {'rr': '__rr__', 'round-robin': '__rr__', 'scatter': '__scatter__'} meta = {} class state: pass def __init__(self, connection=None, id=None, name=None, exchange=None, logger=None, agent=None, **kwargs): self.connection = connection self.id = id or uuid() self.name = name or self.name or self.__class__.__name__ self.exchange = exchange or self.exchange self.agent = agent self.type_to_queue = {'direct': self.get_direct_queue, 'round-robin': self.get_rr_queue, 'scatter': self.get_scatter_queue} if self.default_fields is None: self.default_fields = {} if not self.exchange: self.exchange = Exchange('cl.%s' % (self.name, ), 'direct', auto_delete=True) logger_name = self.name if self.agent: logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, )) self.log = Log('!<%s>' % (logger_name, ), logger=logger) self.state = self.contribute_to_state(self.construct_state()) self.setup() def setup(self): pass def construct_state(self): """Instantiates the state class of this actor.""" return self.state() def maybe_setattr(self, obj, attr, value): if not hasattr(obj, attr): setattr(obj, attr, value) def on_agent_ready(self): pass def contribute_to_object(self, obj, map): for attr, value in map.iteritems(): self.maybe_setattr(obj, attr, value) return obj def contribute_to_state(self, state): try: contribute = state.contribute_to_state except AttributeError: return self.contribute_to_object(state, { 'actor': self, 'agent': self.agent, 'log': self.log, 'Next': self.Next, 'NoRouteError': self.NoRouteError, 'NoReplyError': self.NoReplyError}) else: return contribute(self) def send(self, method, args={}, to=None, nowait=False, **kwargs): """Call method on agent listening to ``routing_key``. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. """ if to is None: to = self.routing_key r = self.call_or_cast(method, args, routing_key=to, nowait=nowait, **kwargs) if not nowait: return r.get() def throw(self, method, args={}, nowait=False, **kwargs): """Call method on one of the agents in round robin. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the reply. """ r = self.call_or_cast(method, args, type='round-robin', nowait=nowait, **kwargs) if not nowait: return r.get() def scatter(self, method, args={}, nowait=False, **kwargs): """Broadcast method to all agents. In this context the reply limit is disabled, and the timeout is set to 1 by default, which means we collect all the replies that managed to be sent within the requested timeout. See :meth:`call_or_cast` for a full list of supported arguments. If the keyword argument `nowait` is false (default) it will block and return the replies. """ kwargs.setdefault('timeout', 2) r = self.call_or_cast(method, args, type='scatter', nowait=nowait, **kwargs) if not nowait: return r.gather(**kwargs) def get_default_scatter_limit(self): if self.agent: return self.agent.get_default_scatter_limit(self.name) return None def call_or_cast(self, method, args={}, nowait=False, **kwargs): """Apply remote `method` asynchronously or synchronously depending on the value of `nowait`. :param method: The name of the remote method to perform. :keyword args: Dictionary of arguments for the method. :keyword nowait: If false the call will be block until the result is available and return it (default), if true the call will be non-blocking. :keyword retry: If set to true then message sending will be retried in the event of connection failures. Default is decided by the :attr:`retry` attributed. :keyword retry_policy: Override retry policies. See :attr:`retry_policy`. This must be a dictionary, and keys will be merged with the default retry policy. :keyword timeout: Timeout to wait for replies in seconds as a float (**only relevant in blocking mode**). :keyword limit: Limit number of replies to wait for (**only relevant in blocking mode**). :keyword callback: If provided, this callback will be called for every reply received (**only relevant in blocking mode**). :keyword \*\*props: Additional message properties. See :meth:`kombu.Producer.publish`. """ return (nowait and self.cast or self.call)(method, args, **kwargs) def get_queues(self): return [self.type_to_queue[type]() for type in self.types] def get_direct_queue(self): """Returns a unique queue that can be used to listen for messages to this class.""" return Queue(self.id, self.exchange, routing_key=self.routing_key, auto_delete=True) def get_scatter_queue(self): return Queue('%s.%s.scatter' % (self.name, self.id), self.exchange, routing_key=self.type_to_rkey['scatter'], auto_delete=True) def get_rr_queue(self): return Queue(self.exchange.name + '.rr', self.exchange, routing_key=self.type_to_rkey['round-robin'], auto_delete=True) def get_reply_queue(self, ticket): return Queue(ticket, self.reply_exchange, ticket, auto_delete=True, queue_arguments={ 'x-expires': int(self.reply_expires * 1000)}) def Consumer(self, channel, **kwargs): """Returns a :class:`kombu.Consumer` instance for this Actor.""" kwargs.setdefault('no_ack', self.no_ack) return Consumer(channel, self.get_queues(), callbacks=[self.on_message], **kwargs) def _publish(self, body, producer, before=None, **props): if before is not None: before(producer.connection, producer.channel) maybe_declare(props['exchange'], producer.channel) return producer.publish(body, **props) def cast(self, method, args={}, before=None, retry=None, retry_policy=None, type=None, **props): """Send message to actor. Discarding replies.""" retry = self.retry if retry is None else retry body = {'class': self.name, 'method': method, 'args': args} exchange = self.exchange _retry_policy = self.retry_policy if retry_policy: # merge default and custom policies. _retry_policy = dict(_retry_policy, **retry_policy) if type: props.setdefault('routing_key', self.type_to_rkey[type]) props.setdefault('routing_key', self.default_routing_key) props.setdefault('serializer', self.serializer) props = dict(props, exchange=exchange, before=before) ipublish(producers[self._connection], self._publish, (body, ), dict(props, exchange=exchange, before=before), **(retry_policy or {})) def call(self, method, args={}, retry=False, retry_policy=None, **props): """Send message to actor and return :class:`AsyncResult`.""" ticket = uuid() reply_q = self.get_reply_queue(ticket) def before(connection, channel): reply_q(channel).declare() self.cast(method, args, before, **dict(props, reply_to=ticket)) return self.AsyncResult(ticket, self) def handle_cast(self, body, message): """Handle cast message.""" self._DISPATCH(body) def handle_call(self, body, message): """Handle call message.""" try: r = self._DISPATCH(body, ticket=message.properties['reply_to']) except self.Next: # don't reply, delegate to other agent. pass else: self.reply(message, r) def reply(self, req, body, **props): return isend_reply(producers[self._connection], self.reply_exchange, req, body, props) def on_message(self, body, message): """What to do when a message is received. This is a kombu consumer callback taking the standard ``body`` and ``message`` arguments. Note that if the properties of the message contains a value for ``reply_to`` then a proper implementation is expected to send a reply. """ if message.properties.get('reply_to'): handler = self.handle_call else: handler = self.handle_cast def handle(): # Do not ack the message if an exceptional error occurs, # but do ack the message if SystemExit or KeyboardInterrupt # is raised, as this is probably intended. try: handler(body, message) except Exception: raise except BaseException: message.ack() raise else: message.ack() handle() def _collect_replies(self, conn, channel, ticket, *args, **kwargs): kwargs.setdefault('timeout', self.default_timeout) if 'limit' not in kwargs: kwargs['limit'] = self.get_default_scatter_limit() return collect_replies(conn, channel, self.get_reply_queue(ticket), *args, **kwargs) def lookup_action(self, name): try: method = getattr(self.state, name) except AttributeError: raise KeyError(name) if not callable(method) or name.startswith('_'): raise KeyError(method) return method def _DISPATCH(self, body, ticket=None): """Dispatch message to the appropriate method in :attr:`state`, handle possible exceptions, and return a response suitable to be used in a reply. To protect from calling special methods it does not dispatch method names starting with underscore (``_``). This returns the return value or exception error with defaults fields in a suitable format to be used as a reply. The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt` will not be handled, and will propagate. In the case of a successful call the return value will be:: {'ok': return_value, **default_fields} If the method raised an exception the return value will be:: {'nok': [repr exc, str traceback], **default_fields} :raises KeyError: if the method specified is unknown or is a special method (name starting with underscore). """ if ticket: sticket = '%s' % (shortuuid(ticket), ) else: ticket = sticket = str(self.next_anon_ticket()) try: method, args = itemgetter('method', 'args')(body) self.log.info('#%s --> %s', sticket, self._reprcall(method, args)) act = self.lookup_action(method) r = {'ok': act(**kwdict(args or {}))} self.log.info('#%s <-- %s', sticket, reprkwargs(r)) except self.Next: raise except Exception, exc: einfo = sys.exc_info() r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]} self.log.error('#%s <-- nok=%r', sticket, exc) return dict(self._default_fields, **r)