def __init__(self,
connection=None,
id=None,
name=None,
exchange=None,
logger=None,
agent=None,
**kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.exchange = exchange or self.exchange
self.agent = agent
self.type_to_queue = {
'direct': self.get_direct_queue,
'round-robin': self.get_rr_queue,
'scatter': self.get_scatter_queue
}
if self.default_fields is None:
self.default_fields = {}
if not self.exchange:
self.exchange = Exchange('cl.%s' % (self.name, ),
'direct',
auto_delete=True)
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, ))
self.log = Log('!<%s>' % (logger_name, ), logger=logger)
self.state = self.contribute_to_state(self.construct_state())
self.setup()
def __init__(self,
connection=None,
id=None,
name=None,
exchange=None,
logger=None,
agent=None,
outbox_exchange=None,
group_exchange=None,
**kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.outbox_exchange = outbox_exchange or self.outbox_exchange
self.agent = agent
if self.default_fields is None:
self.default_fields = {}
# - setup exchanges and queues
self.exchange = exchange or self.get_direct_exchange()
if group_exchange:
self._scatter_exchange = Exchange(group_exchange,
'fanout',
auto_delete=True)
typemap = {
ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct],
ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr],
ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter]
}
self.type_to_queue = {k: v[0] for k, v in items(typemap)}
self.type_to_exchange = {k: v[1] for k, v in items(typemap)}
if not self.outbox_exchange:
self.outbox_exchange = Exchange(
'cl.%s.output' % self.name,
type='topic',
)
# - setup logging
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.id))
self.log = Log('!<%s>' % logger_name, logger=logger)
self.state = self.contribute_to_state(self.construct_state())
# actor specific initialization.
self.construct()
def __init__(self, connection=None, id=None, name=None, exchange=None,
logger=None, agent=None, **kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.exchange = exchange or self.exchange
self.agent = agent
if not self.exchange:
self.exchange = Exchange('cl.%s' % (self.name, ), 'direct')
type_map = {ACT_TYPE.DIRECT:
[self.get_direct_queue, self._inbox_direct],
ACT_TYPE.RR:
[self.get_rr_queue, self._inbox_rr],
ACT_TYPE.SCATTER:
[self.get_scatter_queue, self._inbox_scatter]}
self.type_to_queue = {k: v[0] for k, v in type_map.iteritems()}
self.type_to_exchange = {k: v[1] for k, v in type_map.iteritems()}
if self.default_fields is None:
self.default_fields = {}
if not self.output_exchange:
self.output_exchange = Exchange('cl.%s.output' % (self.name), 'topic')
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, ))
self.log = Log('!<%s>' % (logger_name, ), logger=logger)
self.state = self.contribute_to_state(self.construct_state())
self.setup()
def __init__(self, connection=None, id=None, name=None, exchange=None,
logger=None, agent=None, **kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.exchange = exchange or self.exchange
self.agent = agent
self.type_to_queue = {'direct': self.get_direct_queue,
'round-robin': self.get_rr_queue,
'scatter': self.get_scatter_queue}
if self.default_fields is None:
self.default_fields = {}
if not self.exchange:
self.exchange = Exchange('cl.%s' % (self.name, ), 'direct',
auto_delete=True)
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, ))
self.log = Log('!<%s>' % (logger_name, ), logger=logger)
self.state = self.contribute_to_state(self.construct_state())
self.setup()
def __init__(self, connection=None, id=None, name=None, exchange=None,
logger=None, agent=None, outbox_exchange=None,
group_exchange=None, **kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.outbox_exchange = outbox_exchange or self.outbox_exchange
self.agent = agent
if self.default_fields is None:
self.default_fields = {}
# - setup exchanges and queues
self.exchange = exchange or self.get_direct_exchange()
if group_exchange:
self._scatter_exchange = Exchange(
group_exchange, 'fanout', auto_delete=True)
typemap = {
ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct],
ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr],
ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter]
}
self.type_to_queue = {k: v[0] for k, v in items(typemap)}
self.type_to_exchange = {k: v[1] for k, v in items(typemap)}
if not self.outbox_exchange:
self.outbox_exchange = Exchange(
'cl.%s.output' % self.name, type='topic',
)
# - setup logging
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.id))
self.log = Log('!<%s>' % logger_name, logger=logger)
self.state = self.contribute_to_state(self.construct_state())
# actor specific initialization.
self.construct()
def setup(self):
self.log = Log('Log', Mock())
self.logger = self.log.logger
class test_LogMixin(Case):
def setup(self):
self.log = Log('Log', Mock())
self.logger = self.log.logger
def test_debug(self):
self.log.debug('debug')
self.logger.log.assert_called_with(logging.DEBUG, 'Log - debug')
def test_info(self):
self.log.info('info')
self.logger.log.assert_called_with(logging.INFO, 'Log - info')
def test_warning(self):
self.log.warn('warning')
self.logger.log.assert_called_with(logging.WARN, 'Log - warning')
def test_error(self):
self.log.error('error', exc_info='exc')
self.logger.log.assert_called_with(
logging.ERROR, 'Log - error', exc_info='exc',
)
def test_critical(self):
self.log.critical('crit', exc_info='exc')
self.logger.log.assert_called_with(
logging.CRITICAL, 'Log - crit', exc_info='exc',
)
def test_error_when_DISABLE_TRACEBACKS(self):
from kombu import log
log.DISABLE_TRACEBACKS = True
try:
self.log.error('error')
self.logger.log.assert_called_with(logging.ERROR, 'Log - error')
finally:
log.DISABLE_TRACEBACKS = False
def test_get_loglevel(self):
self.assertEqual(self.log.get_loglevel('DEBUG'), logging.DEBUG)
self.assertEqual(self.log.get_loglevel('ERROR'), logging.ERROR)
self.assertEqual(self.log.get_loglevel(logging.INFO), logging.INFO)
def test_is_enabled_for(self):
self.logger.isEnabledFor.return_value = True
self.assertTrue(self.log.is_enabled_for('DEBUG'))
self.logger.isEnabledFor.assert_called_with(logging.DEBUG)
def test_LogMixin_get_logger(self):
self.assertIs(LogMixin().get_logger(),
logging.getLogger('LogMixin'))
def test_Log_get_logger(self):
self.assertIs(Log('test_Log').get_logger(),
logging.getLogger('test_Log'))
def test_log_when_not_enabled(self):
self.logger.isEnabledFor.return_value = False
self.log.debug('debug')
self.assertFalse(self.logger.log.called)
def test_log_with_format(self):
self.log.debug('Host %r removed', 'example.com')
self.logger.log.assert_called_with(
logging.DEBUG, 'Log - Host %s removed', ANY,
)
self.assertEqual(
self.logger.log.call_args[0][2].strip('u'),
"'example.com'",
)
class Actor:
AsyncResult = AsyncResult
Error = exceptions.CellError
Next = exceptions.Next
NoReplyError = exceptions.NoReplyError
NoRouteError = exceptions.NoRouteError
NotBoundError = exceptions.NotBoundError
#: Actor name.
#: Defaults to the defined class name.
name = None
#: Default exchange(direct) used for messages to this actor.
exchange = None
#: Default routing key used if no ``to`` argument passed.
default_routing_key = None
#: Delivery mode: persistent or transient. Default is persistent.
delivery_mode = 'persistent'
#: Set to True to disable acks.
no_ack = False
#: List of calling types this actor should handle.
#: Valid types are:
#:
#: * direct
#: Send the message directly to an agent by exact routing key.
#: * round-robin
#: Send the message to an agent by round-robin.
#: * scatter
#: Send the message to all of the agents (broadcast).
types = (ACTOR_TYPE.DIRECT, ACTOR_TYPE.SCATTER, ACTOR_TYPE.RR)
#: Default serializer used to send messages and reply messages.
serializer = 'json'
#: Default timeout in seconds as a float which after
#: we give up waiting for replies.
default_timeout = 5.0
#: Time in seconds as a float which after replies expires.
reply_expires = 100.0
#: Exchange used for replies.
reply_exchange = Exchange('cl.reply', 'direct')
#: Exchange used for forwarding/binding with other actors.
outbox_exchange = None
#: Exchange used for receiving broadcast commands for this actor type.
_scatter_exchange = None
#: Exchange used for round-robin commands for this actor type.
_rr_exchange = None
#: Should we retry publishing messages by default?
#: Default: NO
retry = None
#: time-to-live for the actor before becoming Idle
ttl = 20
idle = 40
#: Default policy used when retrying publishing messages.
#: see :meth:`kombu.BrokerConnection.ensure` for a list
#: of supported keys.
retry_policy = {
'max_retries': 100,
'interval_start': 0,
'interval_max': 1,
'interval_step': 0.2
}
#: returns the next anonymous ticket number
#: used fo+r identifying related logs.
ticket_count = count(1)
#: Additional fields added to reply messages by default.
default_fields = {}
#: Map of calling types and their special routing keys.
type_to_rkey = {
'rr': '__rr__',
ACTOR_TYPE.RR: '__rr__',
ACTOR_TYPE.SCATTER: '__scatter__'
}
meta = {}
consumer = None
class state:
"""Placeholder class for actor's supported methods."""
pass
def __init__(self,
connection=None,
id=None,
name=None,
exchange=None,
logger=None,
agent=None,
outbox_exchange=None,
group_exchange=None,
**kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.outbox_exchange = outbox_exchange or self.outbox_exchange
self.agent = agent
if self.default_fields is None:
self.default_fields = {}
# - setup exchanges and queues
self.exchange = exchange or self.get_direct_exchange()
if group_exchange:
self._scatter_exchange = Exchange(group_exchange,
'fanout',
auto_delete=True)
typemap = {
ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct],
ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr],
ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter]
}
self.type_to_queue = {k: v[0] for k, v in items(typemap)}
self.type_to_exchange = {k: v[1] for k, v in items(typemap)}
if not self.outbox_exchange:
self.outbox_exchange = Exchange(
'cl.%s.output' % self.name,
type='topic',
)
# - setup logging
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.id))
self.log = Log('!<%s>' % logger_name, logger=logger)
self.state = self.contribute_to_state(self.construct_state())
# actor specific initialization.
self.construct()
def _add_binding(self,
source,
routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
source_exchange = Exchange(**source)
binder = self.get_binder(inbox_type)
maybe_declare(source_exchange, self.connection.default_channel)
binder(exchange=source_exchange, routing_key=routing_key)
def _remove_binding(self,
source,
routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
source_exchange = Exchange(**source)
unbinder = self.get_unbinder(inbox_type)
unbinder(exchange=source_exchange, routing_key=routing_key)
def get_binder(self, type):
if type == ACTOR_TYPE.DIRECT:
entity = self.type_to_queue[type]()
elif type in self.types:
entity = self.type_to_exchange[type]()
else:
raise ValueError('Unsupported type: {0}'.format(type))
binder = entity.bind_to
# @TODO: Declare probably should not happened here
entity.maybe_bind(self.connection.default_channel)
maybe_declare(entity, entity.channel)
return binder
def get_unbinder(self, type):
if type == ACTOR_TYPE.DIRECT:
entity = self.type_to_queue[type]()
unbinder = entity.unbind_from
else:
entity = self.type_to_exchange[type]()
unbinder = entity.exchange_unbind
entity = entity.maybe_bind(self.connection.default_channel)
# @TODO: Declare probably should not happened here
return unbinder
def add_binding(self,
source,
routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
self.call('add_binding', {
'source': source.as_dict(),
'routing_key': routing_key,
'inbox_type': inbox_type,
},
type=ACTOR_TYPE.DIRECT)
def remove_binding(self,
source,
routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
self.call('remove_binding', {
'source': source.as_dict(),
'routing_key': routing_key,
'inbox_type': inbox_type,
},
type=ACTOR_TYPE.DIRECT)
def construct(self):
"""Actor specific initialization."""
pass
def construct_state(self):
"""Instantiates the state class of this actor."""
return self.state()
def on_agent_ready(self):
pass
def contribute_to_object(self, obj, map):
for attr, value in items(map):
setattr_default(obj, attr, value)
return obj
def contribute_to_state(self, state):
try:
contribute = state.contribute_to_state
except AttributeError:
# set default state attributes.
return self.contribute_to_object(
state, {
'actor': self,
'agent': self.agent,
'connection': self.connection,
'log': self.log,
'Next': self.Next,
'NoRouteError': self.NoRouteError,
'NoReplyError': self.NoReplyError,
'add_binding': self._add_binding,
'remove_binding': self._remove_binding,
})
else:
return contribute(self)
def send(self, method, args={}, to=None, nowait=False, **kwargs):
"""Call method on agent listening to ``routing_key``.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
j
"""
if to is None:
to = self.routing_key
r = self.call_or_cast(method,
args,
routing_key=to,
nowait=nowait,
**kwargs)
if not nowait:
return r.get()
def throw(self, method, args={}, nowait=False, **kwargs):
"""Call method on one of the agents in round robin.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
"""
r = self.call_or_cast(method,
args,
type=ACTOR_TYPE.RR,
nowait=nowait,
**kwargs)
if not nowait:
return r
def scatter(self, method, args={}, nowait=False, timeout=None, **kwargs):
"""Broadcast method to all agents.
if nowait is False, returns generator to iterate over the results.
:keyword limit: Limit number of reads from the queue.
Unlimited by default.
:keyword timeout: the timeout (in float seconds) waiting for replies.
Default is :attr:`default_timeout`.
**Examples**
``scatter`` is a generator (if nowait is False)::
>>> res = scatter()
>>> res.next() # one event consumed, or timed out.
>>> res = scatter(limit=2):
>>> for i in res: # two events consumed or timeout
>>> pass
See :meth:`call_or_cast` for a full list of supported
arguments.
"""
timeout = timeout if timeout is not None else self.default_timeout
r = self.call_or_cast(method,
args,
type=ACTOR_TYPE.SCATTER,
nowait=nowait,
timeout=timeout,
**kwargs)
if not nowait:
return r.gather(timeout=timeout, **kwargs)
def call_or_cast(self, method, args={}, nowait=False, **kwargs):
"""Apply remote `method` asynchronously or synchronously depending
on the value of `nowait`.
:param method: The name of the remote method to perform.
:param args: Dictionary of arguments for the method.
:keyword nowait: If false the call will block until the result
is available and return it (default), if true the call will be
non-blocking and no result will be returned.
:keyword retry: If set to true then message sending will be retried
in the event of connection failures. Default is decided by the
:attr:`retry` attributed.
:keyword retry_policy: Override retry policies.
See :attr:`retry_policy`. This must be a dictionary, and keys will
be merged with the default retry policy.
:keyword timeout: Timeout to wait for replies in seconds as a float
(**only relevant in blocking mode**).
:keyword limit: Limit number of replies to wait for
(**only relevant in blocking mode**).
:keyword callback: If provided, this callback will be called for every
reply received (**only relevant in blocking mode**).
:keyword \*\*props: Additional message properties.
See :meth:`kombu.Producer.publish`.
"""
return (nowait and self.cast or self.call)(method, args, **kwargs)
def get_scatter_exchange(self):
"""Returns a :class:'kombu.Exchange' for type fanout"""
return Exchange('cl.scatter.%s' % self.name,
'fanout',
auto_delete=True)
def get_rr_exchange(self):
"""Returns a :class:'kombu.Exchange' instance with type set to fanout.
The exchange is used for sending in a round-robin style"""
return Exchange('cl.rr.%s' % self.name, 'fanout', auto_delete=True)
def get_direct_exchange(self):
"""Returns a :class:'kombu.Exchange' with type direct"""
return Exchange('cl.%s' % self.name, 'direct', auto_delete=True)
def get_queues(self):
return [self.type_to_queue[type]() for type in self.types]
def get_direct_queue(self):
"""Returns a :class: `kombu.Queue` instance to be used to listen
for messages send to this specific Actor instance"""
return Queue(self.id,
self.inbox_direct,
routing_key=self.routing_key,
auto_delete=True)
def get_scatter_queue(self):
"""Returns a :class: `kombu.Queue` instance for receiving broadcast
commands for this actor type."""
return Queue('%s.%s.scatter' % (self.name, self.id),
self.inbox_scatter,
auto_delete=True)
def get_rr_queue(self):
"""Returns a :class: `kombu.Queue` instance for receiving round-robin
commands for this actor type."""
return Queue(self.inbox_rr.name + '.rr',
self.inbox_rr,
auto_delete=True)
def get_reply_queue(self, ticket):
return Queue(
ticket,
self.reply_exchange,
ticket,
auto_delete=True,
queue_arguments={'x-expires': int(self.reply_expires * 1000)})
def Consumer(self, channel, **kwargs):
"""Returns a :class:`kombu.Consumer` instance for this Actor"""
kwargs.setdefault('no_ack', self.no_ack)
return Consumer(channel,
self.get_queues(),
callbacks=[self.on_message],
**kwargs)
def emit(self, method, args={}, retry=None):
return self.cast(method, args, retry=retry, exchange=self.outbox)
def cast(self,
method,
args={},
declare=None,
retry=None,
retry_policy=None,
type=None,
exchange=None,
**props):
"""Send message to actor. Discarding replies."""
retry = self.retry if retry is None else retry
body = {'class': self.name, 'method': method, 'args': args}
_retry_policy = self.retry_policy
if retry_policy: # merge default and custom policies.
_retry_policy = dict(_retry_policy, **retry_policy)
if type and type not in self.types:
raise ValueError('Unsupported type: {0}'.format(type))
elif not type:
type = ACTOR_TYPE.DIRECT
props.setdefault('routing_key', self.routing_key)
props.setdefault('serializer', self.serializer)
exchange = exchange or self.type_to_exchange[type]()
declare = (maybe_list(declare) or []) + [exchange]
with producers[self._connection].acquire(block=True) as producer:
return producer.publish(body,
exchange=exchange,
declare=declare,
retry=retry,
retry_policy=retry_policy,
**props)
def call(self,
method,
args={},
retry=False,
retry_policy=None,
ticket=None,
**props):
"""Send message to the same actor and return :class:`AsyncResult`."""
ticket = ticket or uuid()
reply_q = self.get_reply_queue(ticket)
self.cast(method, args, declare=[reply_q], reply_to=ticket, **props)
return self.AsyncResult(ticket, self)
def handle_cast(self, body, message):
"""Handle cast message."""
self._DISPATCH(body)
def handle_call(self, body, message):
"""Handle call message."""
try:
r = self._DISPATCH(body, ticket=message.properties['reply_to'])
except self.Next:
# don't reply, delegate to other agents.
pass
else:
self.reply(message, r)
def reply(self, req, body, **props):
with producers[self._connection].acquire(block=True) as producer:
content_type = req.content_type
serializer = serialization.registry.type_to_name[content_type]
return producer.publish(
body,
declare=[self.reply_exchange],
routing_key=req.properties['reply_to'],
correlation_id=req.properties.get('correlation_id'),
serializer=serializer,
**props)
def on_message(self, body, message):
self.agent.process_message(self, body, message)
def _on_message(self, body, message):
"""What to do when a message is received.
This is a kombu consumer callback taking the standard
``body`` and ``message`` arguments.
Note that if the properties of the message contains
a value for ``reply_to`` then a proper implementation
is expected to send a reply.
"""
if message.properties.get('reply_to'):
handler = self.handle_call
else:
handler = self.handle_cast
def handle():
# Do not ack the message if an exceptional error occurs,
# but do ack the message if SystemExit or KeyboardInterrupt
# is raised, as this is probably intended.
try:
handler(body, message)
except Exception:
raise
except BaseException:
message.ack()
raise
else:
message.ack()
handle()
def _collect_replies(self, conn, channel, ticket, *args, **kwargs):
kwargs.setdefault('timeout', self.default_timeout)
if 'limit' not in kwargs and self.agent:
kwargs['limit'] = self.agent.get_default_scatter_limit()
if 'ignore_timeout' not in kwargs and not kwargs.get('limit', None):
kwargs.setdefault('ignore_timeout', False)
return collect_replies(conn, channel, self.get_reply_queue(ticket),
*args, **kwargs)
def lookup_action(self, name):
try:
if not name:
method = self.default_receive
else:
method = getattr(self.state, name)
except AttributeError:
raise KeyError(name)
if not callable(method) or name.startswith('_'):
raise KeyError(method)
return method
def default_receive(self, msg_body):
"""Override in the derived classes."""
pass
def _DISPATCH(self, body, ticket=None):
"""Dispatch message to the appropriate method
in :attr:`state`, handle possible exceptions,
and return a response suitable to be used in a reply.
To protect from calling special methods it does not dispatch
method names starting with underscore (``_``).
This returns the return value or exception error
with defaults fields in a suitable format to be used
as a reply.
The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt`
will not be handled, and will propagate.
In the case of a successful call the return value will
be::
{'ok': return_value, **default_fields}
If the method raised an exception the return value
will be::
{'nok': [repr exc, str traceback], **default_fields}
:raises KeyError: if the method specified is unknown
or is a special method (name starting with underscore).
"""
if ticket:
sticket = '%s' % (shortuuid(ticket), )
else:
ticket = sticket = str(next(self.ticket_counter))
try:
method, args = itemgetter('method', 'args')(body)
self.log.info('#%s --> %s', sticket, self._reprcall(method, args))
act = self.lookup_action(method)
r = {'ok': act(args or {})}
self.log.info('#%s <-- %s', sticket, reprkwargs(r))
except self.Next:
raise
except Exception as exc:
einfo = sys.exc_info()
r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]}
self.log.error('#%s <-- nok=%r', sticket, exc)
return dict(self._default_fields, **r)
def _get_traceback(self, exc_info):
return ''.join(traceback.format_exception(*exc_info))
def _reprcall(self, method, args):
return '%s.%s' % (self.name, reprcall(method, (), args))
def bind(self, connection, agent=None):
return self.__class__(connection,
self.id,
self.name,
self.exchange,
agent=agent)
def is_bound(self):
return self.connection is not None
def __copy__(self):
cls, args = self.__reduce__()
return cls(*args)
def __reduce__(self):
return (self.__class__, (self.connection, self.id, self.name,
self.exchange))
@property
def outbox(self):
return self.outbox_exchange
def _inbox_rr(self):
if not self._rr_exchange:
self._rr_exchange = self.get_rr_exchange()
return self._rr_exchange
@property
def inbox_rr(self):
return self._inbox_rr()
def _inbox_direct(self):
return self.exchange
@property
def inbox_direct(self):
return self._inbox_direct()
def _inbox_scatter(self):
if not self._scatter_exchange:
self._scatter_exchange = self.get_scatter_exchange()
return self._scatter_exchange
@property
def inbox_scatter(self):
return self._inbox_scatter()
@property
def _connection(self):
if not self.is_bound():
raise self.NotBoundError('Actor is not bound to any connection.')
return self.connection
@cached_property
def _default_fields(self):
return dict(BUILTIN_FIELDS, **self.default_fields)
@property
def routing_key(self):
if self.default_routing_key:
return self.default_routing_key
else:
return self.id
def setUp(self):
self.log = Log('Log', Mock())
self.logger = self.log.logger
class test_LogMixin(Case):
def setUp(self):
self.log = Log('Log', Mock())
self.logger = self.log.logger
def test_debug(self):
self.log.debug('debug')
self.logger.log.assert_called_with(logging.DEBUG, 'Log - debug')
def test_info(self):
self.log.info('info')
self.logger.log.assert_called_with(logging.INFO, 'Log - info')
def test_warning(self):
self.log.warn('warning')
self.logger.log.assert_called_with(logging.WARN, 'Log - warning')
def test_error(self):
self.log.error('error', exc_info='exc')
self.logger.log.assert_called_with(
logging.ERROR, 'Log - error', exc_info='exc',
)
def test_critical(self):
self.log.critical('crit', exc_info='exc')
self.logger.log.assert_called_with(
logging.CRITICAL, 'Log - crit', exc_info='exc',
)
def test_error_when_DISABLE_TRACEBACKS(self):
from kombu import log
log.DISABLE_TRACEBACKS = True
try:
self.log.error('error')
self.logger.log.assert_called_with(logging.ERROR, 'Log - error')
finally:
log.DISABLE_TRACEBACKS = False
def test_get_loglevel(self):
self.assertEqual(self.log.get_loglevel('DEBUG'), logging.DEBUG)
self.assertEqual(self.log.get_loglevel('ERROR'), logging.ERROR)
self.assertEqual(self.log.get_loglevel(logging.INFO), logging.INFO)
def test_is_enabled_for(self):
self.logger.isEnabledFor.return_value = True
self.assertTrue(self.log.is_enabled_for('DEBUG'))
self.logger.isEnabledFor.assert_called_with(logging.DEBUG)
def test_LogMixin_get_logger(self):
self.assertIs(LogMixin().get_logger(),
logging.getLogger('LogMixin'))
def test_Log_get_logger(self):
self.assertIs(Log('test_Log').get_logger(),
logging.getLogger('test_Log'))
def test_log_when_not_enabled(self):
self.logger.isEnabledFor.return_value = False
self.log.debug('debug')
self.assertFalse(self.logger.log.called)
def test_log_with_format(self):
self.log.debug('Host %r removed', 'example.com')
self.logger.log.assert_called_with(
logging.DEBUG, 'Log - Host %s removed', "'example.com'",
)
def test_Log_get_logger(self):
self.assertIs(Log('test_Log').get_logger(),
logging.getLogger('test_Log'))
class Actor(object, metaclass=ActorType):
AsyncResult = AsyncResult
Error = exceptions.CellError
Next = exceptions.Next
NoReplyError = exceptions.NoReplyError
NoRouteError = exceptions.NoRouteError
NotBoundError = exceptions.NotBoundError
#: Actor name.
#: Defaults to the defined class name.
name = None
#: Default exchange used for messages to this actor.
exchange = None
#: Default routing key used if no ``to`` argument passed.
default_routing_key = None
#: Delivery mode: persistent or transient. Default is persistent.
delivery_mode = 'persistent'
#: Set to True to disable acks.
no_ack = False
#: List of calling types this actor should handle.
#: Valid types are:
#:
#: * direct
#: Send the message directly to an agent by exact routing key.
#: * round-robin
#: Send the message to an agent by round-robin.
#: * scatter
#: Send the message to all of the agents (broadcast).
types = ('direct', )
#: Default serializer used to send messages and reply messages.
serializer = 'json'
#: Default timeout in seconds as a float which after
#: we give up waiting for replies.
default_timeout = 10.0
#: Time in seconds as a float which after replies expires.
reply_expires = 100.0
#: Exchanged used for replies.
reply_exchange = Exchange('cl.reply', 'direct')
#: Should we retry publishing messages by default?
#: Default: NO
retry = None
#: Default policy used when retrying publishing messages.
#: see :meth:`kombu.BrokerConnection.ensure` for a list
#: of supported keys.
retry_policy = {
'max_retries': 100,
'interval_start': 0,
'interval_max': 1,
'interval_step': 0.2
}
#: returns the next anonymous ticket number
#: used for identifying related logs.
next_anon_ticket = count(1).__next__
#: Additional fields added to reply messages by default.
default_fields = {}
#: Map of calling types and their special routing keys.
type_to_rkey = {
'rr': '__rr__',
'round-robin': '__rr__',
'scatter': '__scatter__'
}
meta = {}
class state:
pass
def __init__(self,
connection=None,
id=None,
name=None,
exchange=None,
logger=None,
agent=None,
**kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.exchange = exchange or self.exchange
self.agent = agent
self.type_to_queue = {
'direct': self.get_direct_queue,
'round-robin': self.get_rr_queue,
'scatter': self.get_scatter_queue
}
if self.default_fields is None:
self.default_fields = {}
if not self.exchange:
self.exchange = Exchange('cl.%s' % (self.name, ),
'direct',
auto_delete=True)
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, ))
self.log = Log('!<%s>' % (logger_name, ), logger=logger)
self.state = self.contribute_to_state(self.construct_state())
self.setup()
def setup(self):
pass
def construct_state(self):
"""Instantiates the state class of this actor."""
return self.state()
def maybe_setattr(self, obj, attr, value):
if not hasattr(obj, attr):
setattr(obj, attr, value)
def on_agent_ready(self):
pass
def contribute_to_object(self, obj, map):
for attr, value in map.items():
self.maybe_setattr(obj, attr, value)
return obj
def contribute_to_state(self, state):
try:
contribute = state.contribute_to_state
except AttributeError:
return self.contribute_to_object(
state, {
'actor': self,
'agent': self.agent,
'log': self.log,
'Next': self.Next,
'NoRouteError': self.NoRouteError,
'NoReplyError': self.NoReplyError
})
else:
return contribute(self)
def send(self, method, args={}, to=None, nowait=False, **kwargs):
"""Call method on agent listening to ``routing_key``.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
"""
if to is None:
to = self.routing_key
r = self.call_or_cast(method,
args,
routing_key=to,
nowait=nowait,
**kwargs)
if not nowait:
return r.get()
def throw(self, method, args={}, nowait=False, **kwargs):
"""Call method on one of the agents in round robin.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
"""
r = self.call_or_cast(method,
args,
type='round-robin',
nowait=nowait,
**kwargs)
if not nowait:
return r.get()
def scatter(self, method, args={}, nowait=False, **kwargs):
"""Broadcast method to all agents.
In this context the reply limit is disabled, and the timeout
is set to 1 by default, which means we collect all the replies
that managed to be sent within the requested timeout.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the replies.
"""
kwargs.setdefault('timeout', 2)
r = self.call_or_cast(method,
args,
type='scatter',
nowait=nowait,
**kwargs)
if not nowait:
return r.gather(**kwargs)
def get_default_scatter_limit(self):
if self.agent:
return self.agent.get_default_scatter_limit(self.name)
return None
def call_or_cast(self, method, args={}, nowait=False, **kwargs):
"""Apply remote `method` asynchronously or synchronously depending
on the value of `nowait`.
:param method: The name of the remote method to perform.
:keyword args: Dictionary of arguments for the method.
:keyword nowait: If false the call will be block until the result
is available and return it (default), if true the call will be
non-blocking.
:keyword retry: If set to true then message sending will be retried
in the event of connection failures. Default is decided by the
:attr:`retry` attributed.
:keyword retry_policy: Override retry policies.
See :attr:`retry_policy`. This must be a dictionary, and keys will
be merged with the default retry policy.
:keyword timeout: Timeout to wait for replies in seconds as a float
(**only relevant in blocking mode**).
:keyword limit: Limit number of replies to wait for
(**only relevant in blocking mode**).
:keyword callback: If provided, this callback will be called for every
reply received (**only relevant in blocking mode**).
:keyword \*\*props: Additional message properties.
See :meth:`kombu.Producer.publish`.
"""
return (nowait and self.cast or self.call)(method, args, **kwargs)
def get_queues(self):
return [self.type_to_queue[type]() for type in self.types]
def get_direct_queue(self):
"""Returns a unique queue that can be used to listen for messages
to this class."""
return Queue(self.id,
self.exchange,
routing_key=self.routing_key,
auto_delete=True)
def get_scatter_queue(self):
return Queue('%s.%s.scatter' % (self.name, self.id),
self.exchange,
routing_key=self.type_to_rkey['scatter'],
auto_delete=True)
def get_rr_queue(self):
return Queue(self.exchange.name + '.rr',
self.exchange,
routing_key=self.type_to_rkey['round-robin'],
auto_delete=True)
def get_reply_queue(self, ticket):
return Queue(
ticket,
self.reply_exchange,
ticket,
auto_delete=True,
queue_arguments={'x-expires': int(self.reply_expires * 1000)})
def Consumer(self, channel, **kwargs):
"""Returns a :class:`kombu.Consumer` instance for this Actor."""
kwargs.setdefault('no_ack', self.no_ack)
return Consumer(channel,
self.get_queues(),
callbacks=[self.on_message],
**kwargs)
def _publish(self, body, producer, before=None, **props):
if before is not None:
before(producer.connection, producer.channel)
maybe_declare(props['exchange'], producer.channel)
return producer.publish(body, **props)
def cast(self,
method,
args={},
before=None,
retry=None,
retry_policy=None,
type=None,
**props):
"""Send message to actor. Discarding replies."""
retry = self.retry if retry is None else retry
body = {'class': self.name, 'method': method, 'args': args}
exchange = self.exchange
_retry_policy = self.retry_policy
if retry_policy: # merge default and custom policies.
_retry_policy = dict(_retry_policy, **retry_policy)
if type:
props.setdefault('routing_key', self.type_to_rkey[type])
props.setdefault('routing_key', self.default_routing_key)
props.setdefault('serializer', self.serializer)
props = dict(props, exchange=exchange, before=before)
ipublish(producers[self._connection], self._publish, (body, ),
dict(props, exchange=exchange, before=before),
**(retry_policy or {}))
def call(self, method, args={}, retry=False, retry_policy=None, **props):
"""Send message to actor and return :class:`AsyncResult`."""
ticket = uuid()
reply_q = self.get_reply_queue(ticket)
def before(connection, channel):
reply_q(channel).declare()
self.cast(method, args, before, **dict(props, reply_to=ticket))
return self.AsyncResult(ticket, self)
def handle_cast(self, body, message):
"""Handle cast message."""
self._DISPATCH(body)
def handle_call(self, body, message):
"""Handle call message."""
try:
r = self._DISPATCH(body, ticket=message.properties['reply_to'])
except self.Next:
# don't reply, delegate to other agent.
pass
else:
self.reply(message, r)
def reply(self, req, body, **props):
return isend_reply(producers[self._connection], self.reply_exchange,
req, body, props)
def on_message(self, body, message):
"""What to do when a message is received.
This is a kombu consumer callback taking the standard
``body`` and ``message`` arguments.
Note that if the properties of the message contains
a value for ``reply_to`` then a proper implementation
is expected to send a reply.
"""
if message.properties.get('reply_to'):
handler = self.handle_call
else:
handler = self.handle_cast
def handle():
# Do not ack the message if an exceptional error occurs,
# but do ack the message if SystemExit or KeyboardInterrupt
# is raised, as this is probably intended.
try:
handler(body, message)
except Exception:
raise
except BaseException:
message.ack()
raise
else:
message.ack()
handle()
def _collect_replies(self, conn, channel, ticket, *args, **kwargs):
kwargs.setdefault('timeout', self.default_timeout)
if 'limit' not in kwargs:
kwargs['limit'] = self.get_default_scatter_limit()
return collect_replies(conn, channel, self.get_reply_queue(ticket),
*args, **kwargs)
def lookup_action(self, name):
try:
method = getattr(self.state, name)
except AttributeError:
raise KeyError(name)
if not callable(method) or name.startswith('_'):
raise KeyError(method)
return method
def _DISPATCH(self, body, ticket=None):
"""Dispatch message to the appropriate method
in :attr:`state`, handle possible exceptions,
and return a response suitable to be used in a reply.
To protect from calling special methods it does not dispatch
method names starting with underscore (``_``).
This returns the return value or exception error
with defaults fields in a suitable format to be used
as a reply.
The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt`
will not be handled, and will propagate.
In the case of a successful call the return value will
be::
{'ok': return_value, **default_fields}
If the method raised an exception the return value
will be::
{'nok': [repr exc, str traceback], **default_fields}
:raises KeyError: if the method specified is unknown
or is a special method (name starting with underscore).
"""
if ticket:
sticket = '%s' % (shortuuid(ticket), )
else:
ticket = sticket = str(self.next_anon_ticket())
try:
method, args = itemgetter('method', 'args')(body)
self.log.info('#%s --> %s', sticket, self._reprcall(method, args))
act = self.lookup_action(method)
r = {'ok': act(**kwdict(args or {}))}
self.log.info('#%s <-- %s', sticket, reprkwargs(r))
except self.Next:
raise
except Exception as exc:
einfo = sys.exc_info()
r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]}
self.log.error('#%s <-- nok=%r', sticket, exc)
return dict(self._default_fields, **r)
def _get_traceback(self, exc_info):
return ''.join(traceback.format_exception(*exc_info))
def _reprcall(self, method, args):
return '%s.%s' % (self.name, reprcall(method, (), args))
def bind(self, connection, agent=None):
return self.__class__(connection,
self.id,
self.name,
self.exchange,
agent=agent)
def is_bound(self):
return self.connection is not None
def __copy__(self):
cls, args = self.__reduce__()
return cls(*args)
def __reduce__(self):
return (self.__class__, (self.connection, self.id, self.name,
self.exchange))
@property
def _connection(self):
if not self.is_bound():
raise self.NotBoundError('Actor is not bound to any connection.')
return self.connection
@cached_property
def _default_fields(self):
return dict(builtin_fields, **self.default_fields)
@property
def routing_key(self):
if self.default_routing_key:
return self.default_routing_key
return self.agent.id
def test_Log_get_logger(self):
assert Log('test_Log').get_logger() is logging.getLogger('test_Log')
class Actor:
AsyncResult = AsyncResult
Error = exceptions.CellError
Next = exceptions.Next
NoReplyError = exceptions.NoReplyError
NoRouteError = exceptions.NoRouteError
NotBoundError = exceptions.NotBoundError
#: Actor name.
#: Defaults to the defined class name.
name = None
#: Default exchange(direct) used for messages to this actor.
exchange = None
#: Default routing key used if no ``to`` argument passed.
default_routing_key = None
#: Delivery mode: persistent or transient. Default is persistent.
delivery_mode = 'persistent'
#: Set to True to disable acks.
no_ack = False
#: List of calling types this actor should handle.
#: Valid types are:
#:
#: * direct
#: Send the message directly to an agent by exact routing key.
#: * round-robin
#: Send the message to an agent by round-robin.
#: * scatter
#: Send the message to all of the agents (broadcast).
types = (ACTOR_TYPE.DIRECT, ACTOR_TYPE.SCATTER, ACTOR_TYPE.RR)
#: Default serializer used to send messages and reply messages.
serializer = 'json'
#: Default timeout in seconds as a float which after
#: we give up waiting for replies.
default_timeout = 5.0
#: Time in seconds as a float which after replies expires.
reply_expires = 100.0
#: Exchange used for replies.
reply_exchange = Exchange('cl.reply', 'direct')
#: Exchange used for forwarding/binding with other actors.
outbox_exchange = None
#: Exchange used for receiving broadcast commands for this actor type.
_scatter_exchange = None
#: Exchange used for round-robin commands for this actor type.
_rr_exchange = None
#: Should we retry publishing messages by default?
#: Default: NO
retry = None
#: time-to-live for the actor before becoming Idle
ttl = 20
idle = 40
#: Default policy used when retrying publishing messages.
#: see :meth:`kombu.BrokerConnection.ensure` for a list
#: of supported keys.
retry_policy = {'max_retries': 100,
'interval_start': 0,
'interval_max': 1,
'interval_step': 0.2}
#: returns the next anonymous ticket number
#: used fo+r identifying related logs.
ticket_count = count(1)
#: Additional fields added to reply messages by default.
default_fields = {}
#: Map of calling types and their special routing keys.
type_to_rkey = {'rr': '__rr__',
ACTOR_TYPE.RR: '__rr__',
ACTOR_TYPE.SCATTER: '__scatter__'}
meta = {}
consumer = None
class state:
"""Placeholder class for actor's supported methods."""
pass
def __init__(self, connection=None, id=None, name=None, exchange=None,
logger=None, agent=None, outbox_exchange=None,
group_exchange=None, **kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.outbox_exchange = outbox_exchange or self.outbox_exchange
self.agent = agent
if self.default_fields is None:
self.default_fields = {}
# - setup exchanges and queues
self.exchange = exchange or self.get_direct_exchange()
if group_exchange:
self._scatter_exchange = Exchange(
group_exchange, 'fanout', auto_delete=True)
typemap = {
ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct],
ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr],
ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter]
}
self.type_to_queue = {k: v[0] for k, v in items(typemap)}
self.type_to_exchange = {k: v[1] for k, v in items(typemap)}
if not self.outbox_exchange:
self.outbox_exchange = Exchange(
'cl.%s.output' % self.name, type='topic',
)
# - setup logging
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.id))
self.log = Log('!<%s>' % logger_name, logger=logger)
self.state = self.contribute_to_state(self.construct_state())
# actor specific initialization.
self.construct()
def _add_binding(self, source, routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
source_exchange = Exchange(**source)
binder = self.get_binder(inbox_type)
maybe_declare(source_exchange, self.connection.default_channel)
binder(exchange=source_exchange, routing_key=routing_key)
def _remove_binding(self, source, routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
source_exchange = Exchange(**source)
unbinder = self.get_unbinder(inbox_type)
unbinder(exchange=source_exchange, routing_key=routing_key)
def get_binder(self, type):
if type == ACTOR_TYPE.DIRECT:
entity = self.type_to_queue[type]()
elif type in self.types:
entity = self.type_to_exchange[type]()
else:
raise ValueError('Unsupported type: {0}'.format(type))
binder = entity.bind_to
# @TODO: Declare probably should not happened here
entity.maybe_bind(self.connection.default_channel)
maybe_declare(entity, entity.channel)
return binder
def get_unbinder(self, type):
if type == ACTOR_TYPE.DIRECT:
entity = self.type_to_queue[type]()
unbinder = entity.unbind_from
else:
entity = self.type_to_exchange[type]()
unbinder = entity.exchange_unbind
entity = entity.maybe_bind(self.connection.default_channel)
# @TODO: Declare probably should not happened here
return unbinder
def add_binding(self, source, routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
self.call('add_binding', {
'source': source.as_dict(),
'routing_key': routing_key,
'inbox_type': inbox_type,
}, type=ACTOR_TYPE.DIRECT)
def remove_binding(self, source, routing_key='',
inbox_type=ACTOR_TYPE.DIRECT):
self.call('remove_binding', {
'source': source.as_dict(),
'routing_key': routing_key,
'inbox_type': inbox_type,
}, type=ACTOR_TYPE.DIRECT)
def construct(self):
"""Actor specific initialization."""
pass
def construct_state(self):
"""Instantiates the state class of this actor."""
return self.state()
def on_agent_ready(self):
pass
def contribute_to_object(self, obj, map):
for attr, value in items(map):
setattr_default(obj, attr, value)
return obj
def contribute_to_state(self, state):
try:
contribute = state.contribute_to_state
except AttributeError:
# set default state attributes.
return self.contribute_to_object(state, {
'actor': self,
'agent': self.agent,
'connection': self.connection,
'log': self.log,
'Next': self.Next,
'NoRouteError': self.NoRouteError,
'NoReplyError': self.NoReplyError,
'add_binding': self._add_binding,
'remove_binding': self._remove_binding,
})
else:
return contribute(self)
def send(self, method, args={}, to=None, nowait=False, **kwargs):
"""Call method on agent listening to ``routing_key``.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
j
"""
if to is None:
to = self.routing_key
r = self.call_or_cast(method, args, routing_key=to,
nowait=nowait, **kwargs)
if not nowait:
return r.get()
def throw(self, method, args={}, nowait=False, **kwargs):
"""Call method on one of the agents in round robin.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
"""
r = self.call_or_cast(method, args, type=ACTOR_TYPE.RR,
nowait=nowait, **kwargs)
if not nowait:
return r
def scatter(self, method, args={}, nowait=False, timeout=None, **kwargs):
"""Broadcast method to all agents.
if nowait is False, returns generator to iterate over the results.
:keyword limit: Limit number of reads from the queue.
Unlimited by default.
:keyword timeout: the timeout (in float seconds) waiting for replies.
Default is :attr:`default_timeout`.
**Examples**
``scatter`` is a generator (if nowait is False)::
>>> res = scatter()
>>> res.next() # one event consumed, or timed out.
>>> res = scatter(limit=2):
>>> for i in res: # two events consumed or timeout
>>> pass
See :meth:`call_or_cast` for a full list of supported
arguments.
"""
timeout = timeout if timeout is not None else self.default_timeout
r = self.call_or_cast(method, args, type=ACTOR_TYPE.SCATTER,
nowait=nowait, timeout=timeout, **kwargs)
if not nowait:
return r.gather(timeout=timeout, **kwargs)
def call_or_cast(self, method, args={}, nowait=False, **kwargs):
"""Apply remote `method` asynchronously or synchronously depending
on the value of `nowait`.
:param method: The name of the remote method to perform.
:param args: Dictionary of arguments for the method.
:keyword nowait: If false the call will block until the result
is available and return it (default), if true the call will be
non-blocking and no result will be returned.
:keyword retry: If set to true then message sending will be retried
in the event of connection failures. Default is decided by the
:attr:`retry` attributed.
:keyword retry_policy: Override retry policies.
See :attr:`retry_policy`. This must be a dictionary, and keys will
be merged with the default retry policy.
:keyword timeout: Timeout to wait for replies in seconds as a float
(**only relevant in blocking mode**).
:keyword limit: Limit number of replies to wait for
(**only relevant in blocking mode**).
:keyword callback: If provided, this callback will be called for every
reply received (**only relevant in blocking mode**).
:keyword \*\*props: Additional message properties.
See :meth:`kombu.Producer.publish`.
"""
return (nowait and self.cast or self.call)(method, args, **kwargs)
def get_scatter_exchange(self):
"""Returns a :class:'kombu.Exchange' for type fanout"""
return Exchange('cl.scatter.%s' % self.name, 'fanout',
auto_delete=True)
def get_rr_exchange(self):
"""Returns a :class:'kombu.Exchange' instance with type set to fanout.
The exchange is used for sending in a round-robin style"""
return Exchange('cl.rr.%s' % self.name, 'fanout', auto_delete=True)
def get_direct_exchange(self):
"""Returns a :class:'kombu.Exchange' with type direct"""
return Exchange('cl.%s' % self.name, 'direct', auto_delete=True)
def get_queues(self):
return [self.type_to_queue[type]() for type in self.types]
def get_direct_queue(self):
"""Returns a :class: `kombu.Queue` instance to be used to listen
for messages send to this specific Actor instance"""
return Queue(self.id, self.inbox_direct, routing_key=self.routing_key,
auto_delete=True)
def get_scatter_queue(self):
"""Returns a :class: `kombu.Queue` instance for receiving broadcast
commands for this actor type."""
return Queue('%s.%s.scatter' % (self.name, self.id),
self.inbox_scatter, auto_delete=True)
def get_rr_queue(self):
"""Returns a :class: `kombu.Queue` instance for receiving round-robin
commands for this actor type."""
return Queue(self.inbox_rr.name + '.rr', self.inbox_rr,
auto_delete=True)
def get_reply_queue(self, ticket):
return Queue(ticket, self.reply_exchange, ticket, auto_delete=True,
queue_arguments={
'x-expires': int(self.reply_expires * 1000)})
def Consumer(self, channel, **kwargs):
"""Returns a :class:`kombu.Consumer` instance for this Actor"""
kwargs.setdefault('no_ack', self.no_ack)
return Consumer(channel, self.get_queues(),
callbacks=[self.on_message], **kwargs)
def emit(self, method, args={}, retry=None):
return self.cast(method, args, retry=retry, exchange=self.outbox)
def cast(self, method, args={}, declare=None, retry=None,
retry_policy=None, type=None, exchange=None, **props):
"""Send message to actor. Discarding replies."""
retry = self.retry if retry is None else retry
body = {'class': self.name, 'method': method, 'args': args}
_retry_policy = self.retry_policy
if retry_policy: # merge default and custom policies.
_retry_policy = dict(_retry_policy, **retry_policy)
if type and type not in self.types:
raise ValueError('Unsupported type: {0}'.format(type))
elif not type:
type = ACTOR_TYPE.DIRECT
props.setdefault('routing_key', self.routing_key)
props.setdefault('serializer', self.serializer)
exchange = exchange or self.type_to_exchange[type]()
declare = (maybe_list(declare) or []) + [exchange]
with producers[self._connection].acquire(block=True) as producer:
return producer.publish(body, exchange=exchange, declare=declare,
retry=retry, retry_policy=retry_policy,
**props)
def call(self, method, args={}, retry=False, retry_policy=None,
ticket=None, **props):
"""Send message to the same actor and return :class:`AsyncResult`."""
ticket = ticket or uuid()
reply_q = self.get_reply_queue(ticket)
self.cast(method, args, declare=[reply_q], reply_to=ticket, **props)
return self.AsyncResult(ticket, self)
def handle_cast(self, body, message):
"""Handle cast message."""
self._DISPATCH(body)
def handle_call(self, body, message):
"""Handle call message."""
try:
r = self._DISPATCH(body, ticket=message.properties['reply_to'])
except self.Next:
# don't reply, delegate to other agents.
pass
else:
self.reply(message, r)
def reply(self, req, body, **props):
with producers[self._connection].acquire(block=True) as producer:
content_type = req.content_type
serializer = serialization.registry.type_to_name[content_type]
return producer.publish(
body,
declare=[self.reply_exchange],
routing_key=req.properties['reply_to'],
correlation_id=req.properties.get('correlation_id'),
serializer=serializer,
**props
)
def on_message(self, body, message):
self.agent.process_message(self, body, message)
def _on_message(self, body, message):
"""What to do when a message is received.
This is a kombu consumer callback taking the standard
``body`` and ``message`` arguments.
Note that if the properties of the message contains
a value for ``reply_to`` then a proper implementation
is expected to send a reply.
"""
if message.properties.get('reply_to'):
handler = self.handle_call
else:
handler = self.handle_cast
def handle():
# Do not ack the message if an exceptional error occurs,
# but do ack the message if SystemExit or KeyboardInterrupt
# is raised, as this is probably intended.
try:
handler(body, message)
except Exception:
raise
except BaseException:
message.ack()
raise
else:
message.ack()
handle()
def _collect_replies(self, conn, channel, ticket, *args, **kwargs):
kwargs.setdefault('timeout', self.default_timeout)
if 'limit' not in kwargs and self.agent:
kwargs['limit'] = self.agent.get_default_scatter_limit()
if 'ignore_timeout' not in kwargs and not kwargs.get('limit', None):
kwargs.setdefault('ignore_timeout', False)
return collect_replies(conn, channel, self.get_reply_queue(ticket),
*args, **kwargs)
def lookup_action(self, name):
try:
if not name:
method = self.default_receive
else:
method = getattr(self.state, name)
except AttributeError:
raise KeyError(name)
if not callable(method) or name.startswith('_'):
raise KeyError(method)
return method
def default_receive(self, msg_body):
"""Override in the derived classes."""
pass
def _DISPATCH(self, body, ticket=None):
"""Dispatch message to the appropriate method
in :attr:`state`, handle possible exceptions,
and return a response suitable to be used in a reply.
To protect from calling special methods it does not dispatch
method names starting with underscore (``_``).
This returns the return value or exception error
with defaults fields in a suitable format to be used
as a reply.
The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt`
will not be handled, and will propagate.
In the case of a successful call the return value will
be::
{'ok': return_value, **default_fields}
If the method raised an exception the return value
will be::
{'nok': [repr exc, str traceback], **default_fields}
:raises KeyError: if the method specified is unknown
or is a special method (name starting with underscore).
"""
if ticket:
sticket = '%s' % (shortuuid(ticket), )
else:
ticket = sticket = str(next(self.ticket_counter))
try:
method, args = itemgetter('method', 'args')(body)
self.log.info('#%s --> %s',
sticket, self._reprcall(method, args))
act = self.lookup_action(method)
r = {'ok': act(args or {})}
self.log.info('#%s <-- %s', sticket, reprkwargs(r))
except self.Next:
raise
except Exception as exc:
einfo = sys.exc_info()
r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]}
self.log.error('#%s <-- nok=%r', sticket, exc)
return dict(self._default_fields, **r)
def _get_traceback(self, exc_info):
return ''.join(traceback.format_exception(*exc_info))
def _reprcall(self, method, args):
return '%s.%s' % (self.name, reprcall(method, (), args))
def bind(self, connection, agent=None):
return self.__class__(connection, self.id,
self.name, self.exchange, agent=agent)
def is_bound(self):
return self.connection is not None
def __copy__(self):
cls, args = self.__reduce__()
return cls(*args)
def __reduce__(self):
return (self.__class__, (self.connection, self.id,
self.name, self.exchange))
@property
def outbox(self):
return self.outbox_exchange
def _inbox_rr(self):
if not self._rr_exchange:
self._rr_exchange = self.get_rr_exchange()
return self._rr_exchange
@property
def inbox_rr(self):
return self._inbox_rr()
def _inbox_direct(self):
return self.exchange
@property
def inbox_direct(self):
return self._inbox_direct()
def _inbox_scatter(self):
if not self._scatter_exchange:
self._scatter_exchange = self.get_scatter_exchange()
return self._scatter_exchange
@property
def inbox_scatter(self):
return self._inbox_scatter()
@property
def _connection(self):
if not self.is_bound():
raise self.NotBoundError('Actor is not bound to any connection.')
return self.connection
@cached_property
def _default_fields(self):
return dict(BUILTIN_FIELDS, **self.default_fields)
@property
def routing_key(self):
if self.default_routing_key:
return self.default_routing_key
else:
return self.id
class Actor(object):
__metaclass__ = ActorType
AsyncResult = AsyncResult
Error = exceptions.CellError
Next = exceptions.Next
NoReplyError = exceptions.NoReplyError
NoRouteError = exceptions.NoRouteError
NotBoundError = exceptions.NotBoundError
#: Actor name.
#: Defaults to the defined class name.
name = None
#: Default exchange used for messages to this actor.
exchange = None
#: Default routing key used if no ``to`` argument passed.
default_routing_key = None
#: Delivery mode: persistent or transient. Default is persistent.
delivery_mode = 'persistent'
#: Set to True to disable acks.
no_ack = False
#: List of calling types this actor should handle.
#: Valid types are:
#:
#: * direct
#: Send the message directly to an agent by exact routing key.
#: * round-robin
#: Send the message to an agent by round-robin.
#: * scatter
#: Send the message to all of the agents (broadcast).
types = ('direct', )
#: Default serializer used to send messages and reply messages.
serializer = 'json'
#: Default timeout in seconds as a float which after
#: we give up waiting for replies.
default_timeout = 10.0
#: Time in seconds as a float which after replies expires.
reply_expires = 100.0
#: Exchanged used for replies.
reply_exchange = Exchange('cl.reply', 'direct')
#: Should we retry publishing messages by default?
#: Default: NO
retry = None
#: Default policy used when retrying publishing messages.
#: see :meth:`kombu.BrokerConnection.ensure` for a list
#: of supported keys.
retry_policy = {'max_retries': 100,
'interval_start': 0,
'interval_max': 1,
'interval_step': 0.2}
#: returns the next anonymous ticket number
#: used for identifying related logs.
next_anon_ticket = count(1).next
#: Additional fields added to reply messages by default.
default_fields = {}
#: Map of calling types and their special routing keys.
type_to_rkey = {'rr': '__rr__',
'round-robin': '__rr__',
'scatter': '__scatter__'}
meta = {}
class state:
pass
def __init__(self, connection=None, id=None, name=None, exchange=None,
logger=None, agent=None, **kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.exchange = exchange or self.exchange
self.agent = agent
self.type_to_queue = {'direct': self.get_direct_queue,
'round-robin': self.get_rr_queue,
'scatter': self.get_scatter_queue}
if self.default_fields is None:
self.default_fields = {}
if not self.exchange:
self.exchange = Exchange('cl.%s' % (self.name, ), 'direct',
auto_delete=True)
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.agent.id, ))
self.log = Log('!<%s>' % (logger_name, ), logger=logger)
self.state = self.contribute_to_state(self.construct_state())
self.setup()
def setup(self):
pass
def construct_state(self):
"""Instantiates the state class of this actor."""
return self.state()
def maybe_setattr(self, obj, attr, value):
if not hasattr(obj, attr):
setattr(obj, attr, value)
def on_agent_ready(self):
pass
def contribute_to_object(self, obj, map):
for attr, value in map.iteritems():
self.maybe_setattr(obj, attr, value)
return obj
def contribute_to_state(self, state):
try:
contribute = state.contribute_to_state
except AttributeError:
return self.contribute_to_object(state, {
'actor': self,
'agent': self.agent,
'log': self.log,
'Next': self.Next,
'NoRouteError': self.NoRouteError,
'NoReplyError': self.NoReplyError})
else:
return contribute(self)
def send(self, method, args={}, to=None, nowait=False, **kwargs):
"""Call method on agent listening to ``routing_key``.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
"""
if to is None:
to = self.routing_key
r = self.call_or_cast(method, args, routing_key=to,
nowait=nowait, **kwargs)
if not nowait:
return r.get()
def throw(self, method, args={}, nowait=False, **kwargs):
"""Call method on one of the agents in round robin.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the reply.
"""
r = self.call_or_cast(method, args, type='round-robin',
nowait=nowait, **kwargs)
if not nowait:
return r.get()
def scatter(self, method, args={}, nowait=False, **kwargs):
"""Broadcast method to all agents.
In this context the reply limit is disabled, and the timeout
is set to 1 by default, which means we collect all the replies
that managed to be sent within the requested timeout.
See :meth:`call_or_cast` for a full list of supported
arguments.
If the keyword argument `nowait` is false (default) it
will block and return the replies.
"""
kwargs.setdefault('timeout', 2)
r = self.call_or_cast(method, args, type='scatter',
nowait=nowait, **kwargs)
if not nowait:
return r.gather(**kwargs)
def get_default_scatter_limit(self):
if self.agent:
return self.agent.get_default_scatter_limit(self.name)
return None
def call_or_cast(self, method, args={}, nowait=False, **kwargs):
"""Apply remote `method` asynchronously or synchronously depending
on the value of `nowait`.
:param method: The name of the remote method to perform.
:keyword args: Dictionary of arguments for the method.
:keyword nowait: If false the call will be block until the result
is available and return it (default), if true the call will be
non-blocking.
:keyword retry: If set to true then message sending will be retried
in the event of connection failures. Default is decided by the
:attr:`retry` attributed.
:keyword retry_policy: Override retry policies.
See :attr:`retry_policy`. This must be a dictionary, and keys will
be merged with the default retry policy.
:keyword timeout: Timeout to wait for replies in seconds as a float
(**only relevant in blocking mode**).
:keyword limit: Limit number of replies to wait for
(**only relevant in blocking mode**).
:keyword callback: If provided, this callback will be called for every
reply received (**only relevant in blocking mode**).
:keyword \*\*props: Additional message properties.
See :meth:`kombu.Producer.publish`.
"""
return (nowait and self.cast or self.call)(method, args, **kwargs)
def get_queues(self):
return [self.type_to_queue[type]() for type in self.types]
def get_direct_queue(self):
"""Returns a unique queue that can be used to listen for messages
to this class."""
return Queue(self.id, self.exchange, routing_key=self.routing_key,
auto_delete=True)
def get_scatter_queue(self):
return Queue('%s.%s.scatter' % (self.name, self.id), self.exchange,
routing_key=self.type_to_rkey['scatter'],
auto_delete=True)
def get_rr_queue(self):
return Queue(self.exchange.name + '.rr', self.exchange,
routing_key=self.type_to_rkey['round-robin'],
auto_delete=True)
def get_reply_queue(self, ticket):
return Queue(ticket, self.reply_exchange, ticket, auto_delete=True,
queue_arguments={
'x-expires': int(self.reply_expires * 1000)})
def Consumer(self, channel, **kwargs):
"""Returns a :class:`kombu.Consumer` instance for this Actor."""
kwargs.setdefault('no_ack', self.no_ack)
return Consumer(channel, self.get_queues(),
callbacks=[self.on_message], **kwargs)
def _publish(self, body, producer, before=None, **props):
if before is not None:
before(producer.connection, producer.channel)
maybe_declare(props['exchange'], producer.channel)
return producer.publish(body, **props)
def cast(self, method, args={}, before=None, retry=None,
retry_policy=None, type=None, **props):
"""Send message to actor. Discarding replies."""
retry = self.retry if retry is None else retry
body = {'class': self.name, 'method': method, 'args': args}
exchange = self.exchange
_retry_policy = self.retry_policy
if retry_policy: # merge default and custom policies.
_retry_policy = dict(_retry_policy, **retry_policy)
if type:
props.setdefault('routing_key', self.type_to_rkey[type])
props.setdefault('routing_key', self.default_routing_key)
props.setdefault('serializer', self.serializer)
props = dict(props, exchange=exchange, before=before)
ipublish(producers[self._connection], self._publish,
(body, ), dict(props, exchange=exchange, before=before),
**(retry_policy or {}))
def call(self, method, args={}, retry=False, retry_policy=None, **props):
"""Send message to actor and return :class:`AsyncResult`."""
ticket = uuid()
reply_q = self.get_reply_queue(ticket)
def before(connection, channel):
reply_q(channel).declare()
self.cast(method, args, before,
**dict(props, reply_to=ticket))
return self.AsyncResult(ticket, self)
def handle_cast(self, body, message):
"""Handle cast message."""
self._DISPATCH(body)
def handle_call(self, body, message):
"""Handle call message."""
try:
r = self._DISPATCH(body, ticket=message.properties['reply_to'])
except self.Next:
# don't reply, delegate to other agent.
pass
else:
self.reply(message, r)
def reply(self, req, body, **props):
return isend_reply(producers[self._connection],
self.reply_exchange, req, body, props)
def on_message(self, body, message):
"""What to do when a message is received.
This is a kombu consumer callback taking the standard
``body`` and ``message`` arguments.
Note that if the properties of the message contains
a value for ``reply_to`` then a proper implementation
is expected to send a reply.
"""
if message.properties.get('reply_to'):
handler = self.handle_call
else:
handler = self.handle_cast
def handle():
# Do not ack the message if an exceptional error occurs,
# but do ack the message if SystemExit or KeyboardInterrupt
# is raised, as this is probably intended.
try:
handler(body, message)
except Exception:
raise
except BaseException:
message.ack()
raise
else:
message.ack()
handle()
def _collect_replies(self, conn, channel, ticket, *args, **kwargs):
kwargs.setdefault('timeout', self.default_timeout)
if 'limit' not in kwargs:
kwargs['limit'] = self.get_default_scatter_limit()
return collect_replies(conn, channel, self.get_reply_queue(ticket),
*args, **kwargs)
def lookup_action(self, name):
try:
method = getattr(self.state, name)
except AttributeError:
raise KeyError(name)
if not callable(method) or name.startswith('_'):
raise KeyError(method)
return method
def _DISPATCH(self, body, ticket=None):
"""Dispatch message to the appropriate method
in :attr:`state`, handle possible exceptions,
and return a response suitable to be used in a reply.
To protect from calling special methods it does not dispatch
method names starting with underscore (``_``).
This returns the return value or exception error
with defaults fields in a suitable format to be used
as a reply.
The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt`
will not be handled, and will propagate.
In the case of a successful call the return value will
be::
{'ok': return_value, **default_fields}
If the method raised an exception the return value
will be::
{'nok': [repr exc, str traceback], **default_fields}
:raises KeyError: if the method specified is unknown
or is a special method (name starting with underscore).
"""
if ticket:
sticket = '%s' % (shortuuid(ticket), )
else:
ticket = sticket = str(self.next_anon_ticket())
try:
method, args = itemgetter('method', 'args')(body)
self.log.info('#%s --> %s',
sticket, self._reprcall(method, args))
act = self.lookup_action(method)
r = {'ok': act(**kwdict(args or {}))}
self.log.info('#%s <-- %s', sticket, reprkwargs(r))
except self.Next:
raise
except Exception, exc:
einfo = sys.exc_info()
r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]}
self.log.error('#%s <-- nok=%r', sticket, exc)
return dict(self._default_fields, **r)
