def test_reply__collect(self):
mailbox = pidbox.Mailbox('test_reply__collect')(self.connection)
exchange = mailbox.reply_exchange.name
channel = self.connection.channel()
mailbox.reply_queue(channel).declare()
ticket = uuid()
mailbox._publish_reply({'foo': 'bar'}, exchange, mailbox.oid, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
reply = mailbox._collect(ticket, limit=1, callback=callback,
channel=channel)
self.assertEqual(reply, [{'foo': 'bar'}])
self.assertTrue(_callback_called[0])
ticket = uuid()
mailbox._publish_reply({'biz': 'boz'}, exchange, mailbox.oid, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{'biz': 'boz'}])
de = mailbox.connection.drain_events = Mock()
de.side_effect = socket.timeout
mailbox._collect(ticket, limit=1, channel=channel)
def test_reply__collect(self):
mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
exchange = mailbox.reply_exchange.name
ticket = uuid()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
channel = self.connection.channel()
reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel)
self.assertEqual(reply, [{"foo": "bar"}])
self.assertTrue(_callback_called[0])
ticket = uuid()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{"biz": "boz"}])
de = mailbox.connection.drain_events = Mock()
de.side_effect = socket.timeout
mailbox._collect(ticket, limit=1, channel=channel)
def test_stop(self, conn):
ag, a, id1, id2 = dA(conn), A(), uuid(), uuid()
ag.state.spawn(qualname(a), id1)
ag.state.spawn(qualname(a), id2)
self.assertEquals(len(ag.state.registry), 2)
actor1, actor2 = ag.state.registry[id1], ag.state.registry[id2]
ag.stop()
self.assertEquals(len(ag.state.registry), 2)
self.assertEquals(actor1.consumer.channel.queues, {})
self.assertEquals(actor2.consumer.channel.queues, {})
def test_gather_kwargs(self, conn, collect):
actor = Actor(conn)
ares = AsyncResult(uuid(), actor)
prev_to_python = ares.to_python
new_to_python = lambda x, propagate = True: x
ares.to_python = new_to_python
# Test default kwargs,
# nothing is passed, the actor does not have agent assigned
self.assert_gather_kwargs(
ares, collect, {},
timeout=actor.default_timeout, ignore_timeout=False)
# limit - set the default agent limit if NONE is set
# Test default kwargs, nothing is passed,
# the actor does have default agent assigned
actor.agent = dAgent(conn)
self.assert_gather_kwargs(
ares, collect, {},
timeout=actor.default_timeout, limit=None, ignore_timeout=False)
# limit - set the default agent limit if NONE is set
# Test default kwargs, nothing is passed,
# the actor does have agent with custom scatter limit assigned
ag = Ag(conn)
actor.agent = ag
self.assert_gather_kwargs(
ares, collect, {}, timeout=actor.default_timeout,
limit=ag.get_default_scatter_limit())
# pass all args
actor.agent = Ag(conn)
timeout, ignore_timeout, limit = 200.0, False, uuid()
self.assert_gather_kwargs(
ares, collect,
{'timeout': timeout, 'ignore_timeout': ignore_timeout,
'limit': limit},
timeout=timeout, limit=limit, ignore_timeout=ignore_timeout)
# ig ignore_tiemout is passed,
# the custom logic for limit is not applies
actor.agent = None
timeout, ignore_timeout = 200.0, True
self.assert_gather_kwargs(
ares, collect,
{'timeout': timeout, 'ignore_timeout': ignore_timeout},
timeout=timeout, ignore_timeout=ignore_timeout)
ares.to_python = prev_to_python
def MockMessage(self, id=None, receipt_handle=None, body=None):
m = Mock(name='message')
m.id = id or uuid()
m.receipt_handle = receipt_handle or uuid()
m._body = body
def _get_body():
return m._body
m.get_body.side_effect = _get_body
def _set_body(value):
m._body = value
m.set_body.side_effect = _set_body
return m
def test_basic_publish__get__consume__restore(self, n="test_basic_publish"):
c = memory_client().channel()
c.exchange_declare(n)
c.queue_declare(n)
c.queue_bind(n, n, n)
c.queue_declare(n + "2")
c.queue_bind(n + "2", n, n)
m = c.prepare_message("nthex quick brown fox...")
c.basic_publish(m, n, n)
r1 = c.message_to_python(c.basic_get(n))
self.assertTrue(r1)
self.assertEqual(r1.body, "nthex quick brown fox...".encode("utf-8"))
self.assertIsNone(c.basic_get(n))
consumer_tag = uuid()
c.basic_consume(n + "2", False, consumer_tag=consumer_tag, callback=lambda *a: None)
self.assertIn(n + "2", c._active_queues)
r2, _ = c.drain_events()
r2 = c.message_to_python(r2)
self.assertEqual(r2.body, "nthex quick brown fox...".encode("utf-8"))
self.assertEqual(r2.delivery_info["exchange"], n)
self.assertEqual(r2.delivery_info["routing_key"], n)
with self.assertRaises(virtual.Empty):
c.drain_events()
c.basic_cancel(consumer_tag)
c._restore(r2)
r3 = c.message_to_python(c.basic_get(n))
self.assertTrue(r3)
self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8"))
self.assertIsNone(c.basic_get(n))
def count_to(self, target):
token = uuid()
init = 0
self.targets[token] = (target, init)
self.adder.throw('add_one', {'i': init, 'token': token},
nowait=True, callback='count',
ckwargs={'token': token})
def handle(self, *args, **options):
headers = {
'correlation_id': uuid(),
'sequence': 0,
'date': dpn_strftime(datetime.now()),
}
body = {
"message_name": "registry-item-create",
"dpn_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
"local_id": "APTRUST-282dcbdd-c16b-42f1-8c21-0dd7875fb94e",
"first_node_name": "aptrust",
"replicating_node_names": ["hathi", "chron", "sdr"],
"version_number": 1,
"previous_version_object_id": "null",
"forward_version_object_id": "null",
"first_version_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
"fixity_algorithm": "sha256",
"fixity_value": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
"lastfixity_date": "2013-01-18T09:49:28-0800",
"creation_date": "2013-01-05T09:49:28-0800",
"last_modified_date": "2013-01-05T09:49:28-0800",
"bag_size": 65536,
"brightening_object_id": ["a02de3cd-a74b-4cc6-adec-16f1dc65f726",
"C92de3cd-a789-4cc6-adec-16a40c65f726", ],
"rights_object_id": ["0df688d4-8dfb-4768-bee9-639558f40488", ],
"object_type": "data",
}
msg = RegistryItemCreate(headers, body)
msg.send(DPN_BROADCAST_KEY)
def _freeze_gid(self, options):
# remove task_id and use that as the group_id,
# if we don't remove it then every task will have the same id...
options = dict(self.options, **options)
options['group_id'] = group_id = (
options.pop('task_id', uuid()))
return options, group_id, options.get('root_id')
def CallServer(self, method, args = None):
try:
LOG.debug(_("strBroker : %s "), self._strBroker)
connection = BrokerConnection(self._strBroker)
# create the response channel
respQueueName = self._respQueueName + str(uuid())
respconnection = BrokerConnection(self._strBroker)
respQueue = respconnection.SimpleQueue(respQueueName,
queue_opts = {'durable': False, 'auto_delete': True},
exchange_opts = {'delivery_mode' : 1,
'auto_delete' : True,
'durable' : False})
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
payload = {"RespQueue": respQueueName, "Source": self._strBroker, 'Method': method, 'args': args}
producer.publish(payload, exchange = self._exchange, serializer="json", routing_key = self._routing_key)
# wait for the response
resp_message = respQueue.get(block=True, timeout=1)
resp_message.ack()
respQueue.close()
#respQueue.delete()
except:
LOG.debug(_("Exception caught : %s"), sys.exc_info()[0])
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
if type(resp_message.payload['Result']).__name__ in ('list', 'tuple'):
nElems = len(resp_message.payload['Result'])
if resp_message.payload['Result'][ nElems - 1 ] == -128:
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
elif type(resp_message.payload['Result']).__name__ == 'int':
if resp_message.payload['Result'] == -128:
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
else:
raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception")
return resp_message.payload['Result']
def run(self, header, body, partial_args, app=None, interval=None,
countdown=1, max_retries=None, propagate=None, eager=False,
task_id=None, **options):
app = app or self._get_app(body)
propagate = (app.conf.chord_propagates
if propagate is None else propagate)
group_id = uuid()
root_id = body.options.get('root_id')
body.chord_size = self.__length_hint__()
options = dict(self.options, **options) if options else self.options
if options:
options.pop('task_id', None)
body.options.update(options)
results = header.freeze(
group_id=group_id, chord=body, root_id=root_id).results
bodyres = body.freeze(task_id, root_id=root_id)
parent = app.backend.apply_chord(
header, partial_args, group_id, body,
interval=interval, countdown=countdown,
options=options, max_retries=max_retries,
propagate=propagate, result=results)
bodyres.parent = parent
return bodyres
def with_ctypes_masked():
from kombu.utils import ctypes, uuid
self.assertIsNone(ctypes)
tid = uuid()
self.assertTrue(tid)
self.assertIsInstance(tid, string_t)
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
self.assertIn(tid, revoked)
self.assertTrue(request.terminate.call_count)
self.assertIn('terminate:', r['ok'])
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
self.assertIn('tasks unknown', r['ok'])
finally:
worker_state.task_ready(request)
def send_task(self, name, args=None, kwargs=None, countdown=None,
eta=None, task_id=None, producer=None, connection=None,
router=None, result_cls=None, expires=None,
publisher=None, link=None, link_error=None,
add_to_parent=True, reply_to=None, **options):
task_id = task_id or uuid()
producer = producer or publisher # XXX compat
router = router or self.amqp.router
conf = self.conf
if conf.CELERY_ALWAYS_EAGER: # pragma: no cover
warnings.warn(AlwaysEagerIgnored(
'CELERY_ALWAYS_EAGER has no effect on send_task',
), stacklevel=2)
options = router.route(options, name, args, kwargs)
if connection:
producer = self.amqp.TaskProducer(connection)
with self.producer_or_acquire(producer) as P:
self.backend.on_task_call(P, task_id)
task_id = P.publish_task(
name, args, kwargs, countdown=countdown, eta=eta,
task_id=task_id, expires=expires,
callbacks=maybe_list(link), errbacks=maybe_list(link_error),
reply_to=reply_to or self.oid, **options
)
result = (result_cls or self.AsyncResult)(task_id)
if add_to_parent:
parent = get_current_worker_task()
if parent:
parent.add_trail(result)
return result
def __init__(self, name=None, queue=None, **kwargs):
kwargs.setdefault("exchange", entity.Exchange(name, type="fanout",
auto_delete=True))
kwargs.setdefault("auto_delete", True)
kwargs.setdefault("alias", name)
return super(Broadcast, self).__init__(
name=queue or "bcast.%s" % (uuid(), ), **kwargs)
def freeze(self, _id=None, group_id=None, chord=None,
root_id=None, parent_id=None):
"""Finalize the signature by adding a concrete task id.
The task will not be called and you should not call the signature
twice after freezing it as that will result in two task messages
using the same task id.
Returns:
~@AsyncResult: promise of future evaluation.
"""
opts = self.options
try:
tid = opts['task_id']
except KeyError:
tid = opts['task_id'] = _id or uuid()
if root_id:
opts['root_id'] = root_id
if parent_id:
opts['parent_id'] = parent_id
if 'reply_to' not in opts:
opts['reply_to'] = self.app.oid
if group_id:
opts['group_id'] = group_id
if chord:
opts['chord'] = chord
return self.AsyncResult(tid)
def handle(self, *args, **options):
msg = ReplicationInitQuery()
headers = {
'correlation_id': uuid(),
'sequence': 0,
'date': dpn_strftime(datetime.now())
}
msg.set_headers(**headers)
body = {
'message_name': 'replication-init-query',
'replication_size': 4502,
'protocol': ['https', 'rsync'],
'dpn_object_id': uuid()
}
msg.set_body(**body)
msg.send(DPN_BROADCAST_KEY)
def _freeze(self, _id=None):
opts = self.options
try:
tid = opts["task_id"]
except KeyError:
tid = opts["task_id"] = _id or uuid()
return self.AsyncResult(tid)
def __init__(self, channel, handlers=None, routing_key='#',
node_id=None, app=None, queue_prefix=None,
accept=None, queue_ttl=None, queue_expires=None):
self.app = app_or_default(app or self.app)
self.channel = maybe_channel(channel)
self.handlers = {} if handlers is None else handlers
self.routing_key = routing_key
self.node_id = node_id or uuid()
self.queue_prefix = queue_prefix or self.app.conf.event_queue_prefix
self.exchange = get_exchange(
self.connection or self.app.connection_for_write())
self.queue = Queue(
'.'.join([self.queue_prefix, self.node_id]),
exchange=self.exchange,
routing_key=self.routing_key,
auto_delete=True, durable=False,
queue_arguments=self._get_queue_arguments(
ttl=queue_ttl, expires=queue_expires,
),
)
self.clock = self.app.clock
self.adjust_clock = self.clock.adjust
self.forward_clock = self.clock.forward
if accept is None:
accept = {self.app.conf.event_serializer, 'json'}
self.accept = accept
def send_task(self, name, args=None, kwargs=None, countdown=None,
eta=None, task_id=None, producer=None, connection=None,
router=None, result_cls=None, expires=None,
publisher=None, link=None, link_error=None,
add_to_parent=True, group_id=None, retries=0, chord=None,
reply_to=None, time_limit=None, soft_time_limit=None,
root_id=None, parent_id=None, route_name=None,
shadow=None, chain=None, **options):
"""Send task by name.
:param name: Name of task to call (e.g. `"tasks.add"`).
:keyword result_cls: Specify custom result class. Default is
using :meth:`AsyncResult`.
Otherwise supports the same arguments as :meth:`@-Task.apply_async`.
"""
parent = have_parent = None
amqp = self.amqp
task_id = task_id or uuid()
producer = producer or publisher # XXX compat
router = router or amqp.router
conf = self.conf
if conf.task_always_eager: # pragma: no cover
warnings.warn(AlwaysEagerIgnored(
'task_always_eager has no effect on send_task',
), stacklevel=2)
options = router.route(options, route_name or name, args, kwargs)
if root_id is None:
parent, have_parent = self.current_worker_task, True
if parent:
root_id = parent.request.root_id or parent.request.id
if parent_id is None:
if not have_parent:
parent, have_parent = self.current_worker_task, True
if parent:
parent_id = parent.request.id
message = amqp.create_task_message(
task_id, name, args, kwargs, countdown, eta, group_id,
expires, retries, chord,
maybe_list(link), maybe_list(link_error),
reply_to or self.oid, time_limit, soft_time_limit,
self.conf.task_send_sent_event,
root_id, parent_id, shadow, chain,
)
if connection:
producer = amqp.Producer(connection)
with self.producer_or_acquire(producer) as P:
self.backend.on_task_call(P, task_id)
amqp.send_task_message(P, name, message, **options)
result = (result_cls or self.AsyncResult)(task_id)
if add_to_parent:
if not have_parent:
parent, have_parent = self.current_worker_task, True
if parent:
parent.add_trail(result)
return result
def _broadcast(self, command, arguments=None, destination=None,
reply=False, timeout=1, limit=None, callback=None, channel=None):
arguments = arguments or {}
reply_ticket = reply and uuid() or None
if destination is not None and \
not isinstance(destination, (list, tuple)):
raise ValueError("destination must be a list/tuple not %s" % (
type(destination)))
# Set reply limit to number of destinations (if specificed)
if limit is None and destination:
limit = destination and len(destination) or None
chan = channel or self.connection.channel()
try:
if reply_ticket:
self.get_reply_queue(reply_ticket)(chan).declare()
self._publish(command, arguments, destination=destination,
reply_ticket=reply_ticket,
channel=chan)
if reply_ticket:
return self._collect(reply_ticket, limit=limit,
timeout=timeout,
callback=callback,
channel=chan)
finally:
channel or chan.close()
def test_fast_trace_task(self):
from celery.app import trace
setup_worker_optimizations(self.app)
self.assertIs(trace.trace_task_ret, trace._fast_trace_task)
tid = uuid()
message = TaskMessage(self.mytask.name, tid, args=[4])
assert len(message.payload) == 3
try:
self.mytask.__trace__ = build_tracer(
self.mytask.name, self.mytask, self.app.loader, 'test',
app=self.app,
)
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
finally:
reset_worker_optimizations()
self.assertIs(trace.trace_task_ret, trace._trace_task_ret)
delattr(self.mytask, '__trace__')
failed, res, runtime = trace.trace_task_ret(
self.mytask.name, tid, message.headers, message.body,
message.content_type, message.content_encoding, app=self.app,
)
self.assertFalse(failed)
self.assertEqual(res, repr(4 ** 4))
self.assertIsNotNone(runtime)
self.assertIsInstance(runtime, numbers.Real)
def test_execute_using_pool__revoked(self):
tid = uuid()
job = self.zRequest(id=tid, revoked_tasks={tid})
job.revoked = Mock()
job.revoked.return_value = True
with self.assertRaises(TaskRevokedError):
job.execute_using_pool(self.pool)
def test_on_success__with_events(self):
job = self.zRequest(id=uuid())
job.send_event = Mock(name='send_event')
job.on_success((False, 'foo', 1.0))
job.send_event.assert_called_with(
'task-succeeded', result='foo', runtime=1.0,
)
def count_to(self, target):
token = uuid()
init = 0
self.targets[token] = (target, init)
self.adder.throw(
"add_one", {"i": init, "token": token}, nowait=True, callback="count", ckwargs={"token": token}
)
def test_gather(self, conn):
def collect_replies():
yield 1
yield 2
yield 3
ticket = uuid()
actor = Actor(conn)
actor._collect_replies = Mock(return_value=collect_replies())
ares = AsyncResult(ticket, actor)
ares.to_python = Mock()
all = ares.gather()
list(all)
actor._collect_replies.assert_caleld_once_with(conn, ANY, ticket)
self.assertEqual(ares.to_python.call_count,
len(list(collect_replies())))
# test that the to_python is applied to all results
actor._collect_replies.reset_mock()
actor._collect_replies = Mock(return_value=collect_replies())
prev_to_python = ares.to_python
new_to_python = lambda x, propagate = True: 'called_%s' % x
ares.to_python = new_to_python
all = ares.gather()
vals = list(all)
expected_vals = [new_to_python(i) for i in collect_replies()]
actor._collect_replies.assert_caleld_once_with(conn, ANY, ticket)
self.assertEqual(vals, expected_vals)
ares.to_python = prev_to_python
def test_execute(self):
tid = uuid()
job = self.xRequest(id=tid, args=[4], kwargs={})
self.assertEqual(job.execute(), 256)
meta = self.mytask.backend.get_task_meta(tid)
self.assertEqual(meta['status'], states.SUCCESS)
self.assertEqual(meta['result'], 256)
def test_get_set_forget(self):
b = self.Backend(app=self.app)
tid = uuid()
b.store_result(tid, 42, states.SUCCESS)
self.assertEqual(b.get_status(tid), states.SUCCESS)
self.assertEqual(b.get_result(tid), 42)
b.forget(tid)
self.assertEqual(b.get_status(tid), states.PENDING)
def test_revoke_with_name_not_in_registry(self):
tid = uuid()
m = {'method': 'revoke',
'destination': hostname,
'arguments': {'task_id': tid,
'task_name': 'xxxxxxxxx33333333388888'}}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
def test_process_cleanup_fails(self, _logger):
self.mytask.backend = Mock()
self.mytask.backend.process_cleanup = Mock(side_effect=KeyError())
tid = uuid()
ret = jail(self.app, tid, self.mytask.name, [2], {})
self.assertEqual(ret, 4)
self.mytask.backend.mark_as_done.assert_called()
self.assertIn('Process cleanup failed', _logger.error.call_args[0][0])
def __call__(self, body=None, **kwargs):
_chord = self.Chord
body = (body or self.kwargs['body']).clone()
kwargs = dict(self.kwargs, body=body, **kwargs)
if _chord.app.conf.CELERY_ALWAYS_EAGER:
return self.apply((), kwargs)
callback_id = body.options.setdefault('task_id', uuid())
_chord(**kwargs)
return _chord.AsyncResult(callback_id)
def apply_async(self, args, kwargs, **options):
args, kw = self.serialize_args(args, kwargs)
# Let's see if this is a retry. An existing task means yes.
# If it is one, we'll call _apply_async directly later on.
task = getattr(self.request, 'task', None)
task_id = options.get('task_id', None)
# if task is not None we are in a retry and site_path and
# authorized_userid are already in kw
if task is None:
kw['site_path'] = '/'.join(api.portal.get().getPhysicalPath())
kw['authorized_userid'] = api.user.get_current().getId()
without_transaction = options.pop('without_transaction', False)
celery = getCelery()
if task_id is None:
# Here we cheat a little: since we will not start the task
# up until the transaction is done,
# we cannot give back to whoever called apply_async
# its much beloved AsyncResult.
# But we can actually pass the task a specific task_id
# (although it's not very documented)
# and an AsyncResult at this point is just that id, basically.
task_id = uuid()
else:
# If this is a retry, task_id will be in the options.
# Get rid of it to avoid an error.
del options['task_id']
# Construct a fake result
if celery.conf.task_always_eager:
result_ = EagerResult(task_id, None, states.PENDING, None)
else:
result_ = result.AsyncResult(task_id)
# Note: one might be tempted to turn this into a datamanager.
# This would result in two wrong things happening:
# * A "commit within a commit" triggered by the function runner
# when CELERY_TASK_ALWAYS_EAGER is set,
# leading to the first invoked commit cleanup failing
# because the inner commit already cleaned up.
# * An async task failing in eager mode would also rollback
# the whole transaction, which is not desiderable.
# Consider the case where the syncronous code constructs an object
# and the async task updates it, if we roll back everything
# then also the original content construction goes away
# (even if, in and by itself, worked)
if without_transaction or celery.conf.task_always_eager or task:
return self._apply_async(args, kw, result_, celery, task_id,
options)
else:
queue_task_after_commit(args, kw, self, task_id, options)
# Return the "fake" result ID
return result_
def test_execute_jail_failure(self):
ret = jail(
self.app,
uuid(),
self.mytask_raising.name,
[4],
{},
)
self.assertIsInstance(ret, ExceptionInfo)
self.assertTupleEqual(ret.exception.args, (4, ))
def test_state_stop_actor_by_id(self, conn, consumer):
ag, a, id = dA(conn), A(), uuid()
ag.state.spawn(qualname(a), id)
self.assertEquals(len(ag.state.registry), 1)
actor = ag.state.registry[id]
ag.state.kill(id)
self.assertEquals(ag.state.registry, {})
actor.consumer.cancel.assert_called_once_with()
def test_on_success__SystemExit(self,
errors=(SystemExit, KeyboardInterrupt)):
for exc in errors:
einfo = None
try:
raise exc()
except exc:
einfo = ExceptionInfo()
with self.assertRaises(exc):
self.zRequest(id=uuid()).on_success((True, einfo, 1.0))
def election(self):
type = self.type
app = type.app
tid = self.options.get('task_id') or uuid()
with app.producer_or_acquire(None) as P:
props = type.backend.on_task_call(P, tid)
app.control.election(tid, 'task', self.clone(task_id=tid, **props),
connection=P.connection)
return type.AsyncResult(tid)
def test_marked_as_started(self):
_started = []
def store_result(tid, meta, state, **kwargs):
if state == states.STARTED:
_started.append(tid)
self.mytask.backend.store_result = Mock(name='store_result')
self.mytask.backend.store_result.side_effect = store_result
self.mytask.track_started = True
tid = uuid()
jail(self.app, tid, self.mytask.name, [2], {})
self.assertIn(tid, _started)
self.mytask.ignore_result = True
tid = uuid()
jail(self.app, tid, self.mytask.name, [2], {})
self.assertNotIn(tid, _started)
def test_get(self):
id1, id2 = uuid(), uuid()
def gather():
yield id1
yield id2
# test that it calls gather with limit = 1 and kwargs
ares = self.get_async_result()
ares.gather = Mock(return_value=['1'])
ares.get()
ares.gather.assert_called_once_with(limit=1)
ares.gather.reset_mock()
kwargs = {'timeout': 100, 'ignore_timeout': False,
'foo': 'bar', 'propaget': True}
ares.get(**kwargs)
ares.gather.assert_called_once_with(**dict(kwargs, limit=1))
ares.gather.reset_mock()
kwargs = {'timeout': 100, 'ignore_timeout': False, 'limit': 10}
ares.get(**kwargs)
ares.gather.assert_called_once_with(**kwargs)
ares.gather.reset_mock()
# it returns the first value of whatever gather returns
ares.gather = Mock(return_value=gather())
res = ares.get()
self.assertEqual(res, id1)
# if gather does not return result:
# self.NoReplyError('No reply received within time constraint')
ares.gather = Mock(return_value=None)
with self.assertRaises(ares.NoReplyError):
ares.get()
ares.gather.reset_mock()
ares.gather = Mock(return_value={})
with self.assertRaises(ares.NoReplyError):
ares.get()
def test_can_consume(self, stdout, stderr):
_restored = []
class RestoreChannel(virtual.Channel):
do_restore = True
def _restore(self, message):
_restored.append(message)
self.assertTrue(self.q.can_consume())
for i in range(self.q.prefetch_count - 1):
self.q.append(i, uuid())
self.assertTrue(self.q.can_consume())
self.q.append(i + 1, uuid())
self.assertFalse(self.q.can_consume())
tag1 = self.q._delivered.keys()[0]
self.q.ack(tag1)
self.assertTrue(self.q.can_consume())
tag2 = uuid()
self.q.append(i + 2, tag2)
self.assertFalse(self.q.can_consume())
self.q.reject(tag2)
self.assertTrue(self.q.can_consume())
self.q.channel = RestoreChannel(self.q.channel.connection)
tag3 = uuid()
self.q.append(i + 3, tag3)
self.q.reject(tag3, requeue=True)
self.q._flush()
self.q.restore_unacked_once()
self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1])
self.assertTrue(self.q._delivered.restored)
self.assertFalse(self.q._delivered)
self.q.restore_unacked_once()
self.q._delivered.restored = False
self.q.restore_unacked_once()
self.assertTrue(stderr.getvalue())
self.assertFalse(stdout.getvalue())
def test_state_select_returns_from_registry(self, conn):
class B(Actor):
pass
ag = dAgent(conn)
id1, id2 = uuid(), uuid()
with self.assertRaises(Actor.Next):
ag.state.select(qualname(A))
ag.state.registry[id1] = A()
key = ag.state.select(qualname(A))
self.assertEqual(key, id1)
ag.state.registry[id2] = B(conn)
keyA = ag.state.select(qualname(A))
keyB = ag.state.select(qualname(B))
self.assertEqual(keyA, id1)
self.assertEqual(keyB, id2)
def create_binding(self, queue):
"""Get binding item for queue.
Creates the item if it doesn't exist.
"""
item = self.get_queue(queue)
if item:
return item, item['id']
id = uuid()
return self.new_item(id), id
def test_create(self):
c = client().channel()
data = c.prepare_message("the quick brown fox...")
tag = data["properties"]["delivery_tag"] = uuid()
message = c.message_to_python(data)
self.assertIsInstance(message, virtual.Message)
self.assertIs(message, c.message_to_python(message))
self.assertEqual(message.body,
"the quick brown fox...".encode("utf-8"))
self.assertTrue(message.delivery_tag, tag)
def queue_declare(self, queue=None, passive=False, **kwargs):
"""Declare queue."""
queue = queue or 'amq.gen-%s' % uuid()
if passive and not self._has_queue(queue, **kwargs):
raise StdChannelError(
'404', 'NOT_FOUND - no queue {0!r} in vhost {1!r}'.format(
queue, self.connection.client.virtual_host or '/'),
(50, 10), 'Channel.queue_declare')
else:
self._new_queue(queue, **kwargs)
return queue, self._size(queue), 0
def test_create(self):
c = client().channel()
data = c.prepare_message('the quick brown fox...')
tag = data['properties']['delivery_tag'] = uuid()
message = c.message_to_python(data)
self.assertIsInstance(message, virtual.Message)
self.assertIs(message, c.message_to_python(message))
self.assertEqual(message.body,
'the quick brown fox...'.encode('utf-8'))
self.assertTrue(message.delivery_tag, tag)
def test_state_spawn_when_id_not_in_registry(self, conn, consumer):
ag, a, id = dA(conn), A(), uuid()
self.assertEquals(ag.state.registry, {})
ag.state.spawn(qualname(a), id)
self.assertEquals(len(ag.state.registry), 1)
actor = ag.state.registry[id]
self.assertIs(type(actor), A)
self.assertIsNotNone(actor.consumer)
actor.consumer.consume.assert_called_once_with()
def count_to(self, target):
token = uuid()
init = 0
self.targets[token] = (target, init)
self.adder.throw('add_one', {
'i': init,
'token': token
},
nowait=True,
callback='count',
ckwargs={'token': token})
def test_serializable(self):
c = client().channel()
body, content_type = compress('the quick brown fox...', 'gzip')
data = c.prepare_message(body, headers={'compression': content_type})
tag = data['properties']['delivery_tag'] = uuid()
message = c.message_to_python(data)
dict_ = message.serializable()
self.assertEqual(dict_['body'],
'the quick brown fox...'.encode('utf-8'))
self.assertEqual(dict_['properties']['delivery_tag'], tag)
self.assertNotIn('compression', dict_['headers'])
def test_revoke_with_name_not_in_registry(self):
tid = uuid()
m = {
'method': 'revoke',
'destination': hostname,
'arguments': {
'task_id': tid,
'task_name': 'xxxxxxxxx33333333388888'
}
}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
def test_revoke_with_name(self):
tid = uuid()
m = {
'method': 'revoke',
'destination': hostname,
'arguments': {
'task_id': tid,
'task_name': self.mytask.name
}
}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
def _test_on_failure(self, exception, **kwargs):
tid = uuid()
job = self.xRequest(id=tid, args=[4])
job.send_event = Mock(name='send_event')
job.task.backend.mark_as_failure = Mock(name='mark_as_failure')
try:
raise exception
except type(exception):
exc_info = ExceptionInfo()
job.on_failure(exc_info, **kwargs)
job.send_event.assert_called()
return job
def test_init(self):
ticket = uuid()
actor = Mock()
ares = AsyncResult(ticket, actor)
self.assertEquals(ares.ticket, ticket)
self.assertEqual(ares.actor, actor)
self.assertIsNone(ares._result)
self.assertEqual(ares.Error, CellError)
self.assertEqual(ares.NoReplyError, NoReplyError)
with self.assertRaises(TypeError):
AsyncResult(ticket)
def test_reject(self):
job = self.xRequest(id=uuid())
job.on_reject = Mock(name='on_reject')
job.reject(requeue=True)
job.on_reject.assert_called_with(
req_logger,
job.connection_errors,
True,
)
self.assertTrue(job.acknowledged)
job.on_reject.reset_mock()
job.reject(requeue=True)
job.on_reject.assert_not_called()
def test_execute_fail(self):
tid = uuid()
job = self.xRequest(
name=self.mytask_raising.name,
id=tid,
args=[4],
kwargs={},
)
self.assertIsInstance(job.execute(), ExceptionInfo)
assert self.mytask_raising.backend.serializer == 'pickle'
meta = self.mytask_raising.backend.get_task_meta(tid)
self.assertEqual(meta['status'], states.FAILURE)
self.assertIsInstance(meta['result'], KeyError)
def freeze(self, _id=None, group_id=None, chord=None):
opts = self.options
try:
tid = opts['task_id']
except KeyError:
tid = opts['task_id'] = _id or uuid()
if 'reply_to' not in opts:
opts['reply_to'] = self.app.oid
if group_id:
opts['group_id'] = group_id
if chord:
opts['chord'] = chord
return self.AsyncResult(tid)
def _freeze(self, _id=None):
opts = self.options
try:
gid = opts['group']
except KeyError:
gid = opts['group'] = uuid()
new_tasks, results = [], []
for task in self.tasks:
task = maybe_subtask(task).clone()
results.append(task._freeze())
new_tasks.append(task)
self.tasks = self.kwargs['tasks'] = new_tasks
return GroupResult(gid, results)
def freeze(self, _id=None):
opts = self.options
try:
gid = opts['task_id']
except KeyError:
gid = opts['task_id'] = uuid()
new_tasks, results = [], []
for task in self.tasks:
task = maybe_signature(task, app=self._app).clone()
results.append(task._freeze())
new_tasks.append(task)
self.tasks = self.kwargs['tasks'] = new_tasks
return self.app.GroupResult(gid, results)
def test_execute_ack(self):
scratch = {'ACK': False}
def on_ack(*args, **kwargs):
scratch['ACK'] = True
tid = uuid()
job = self.xRequest(id=tid, args=[4], on_ack=on_ack)
self.assertEqual(job.execute(), 256)
meta = self.mytask.backend.get_task_meta(tid)
self.assertTrue(scratch['ACK'])
self.assertEqual(meta['result'], 256)
self.assertEqual(meta['status'], states.SUCCESS)
def test_revoke(self):
tid = uuid()
m = {'method': 'revoke',
'destination': hostname,
'arguments': {'task_id': tid}}
self.panel.handle_message(m, None)
self.assertIn(tid, revoked)
m = {'method': 'revoke',
'destination': 'does.not.exist',
'arguments': {'task_id': tid + 'xxx'}}
self.panel.handle_message(m, None)
self.assertNotIn(tid + 'xxx', revoked)
def CallServer(self, method, args=None):
try:
LOG.debug(_("strBroker : %s "), self._strBroker)
connection = BrokerConnection(self._strBroker)
# create the response channel
respQueueName = self._respQueueName + str(uuid())
respconnection = BrokerConnection(self._strBroker)
respQueue = respconnection.SimpleQueue(respQueueName,
queue_opts={
'durable': False,
'auto_delete': True
},
exchange_opts={
'delivery_mode': 1,
'auto_delete': True,
'durable': False
})
with producers[connection].acquire(block=True) as producer:
maybe_declare(task_exchange, producer.channel)
payload = {
"RespQueue": respQueueName,
"Source": self._strBroker,
'Method': method,
'args': args
}
producer.publish(payload,
exchange=self._exchange,
serializer="json",
routing_key=self._routing_key)
# wait for the response
resp_message = respQueue.get(block=True, timeout=1)
resp_message.ack()
respQueue.close()
#respQueue.delete()
except:
LOG.debug(_("Exception caught : %s"), sys.exc_info()[0])
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
if type(resp_message.payload['Result']).__name__ in ('list', 'tuple'):
nElems = len(resp_message.payload['Result'])
if resp_message.payload['Result'][nElems - 1] == -128:
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
elif type(resp_message.payload['Result']).__name__ == 'int':
if resp_message.payload['Result'] == -128:
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
else:
raise OpenCLClientException.OpenCLClientException(
"OpenCL Interface Exception")
return resp_message.payload['Result']
def test_reply__collect(self):
mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
exchange = mailbox.reply_exchange.name
ticket = uuid()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
channel = self.connection.channel()
reply = mailbox._collect(ticket, limit=1, callback=callback,
channel=channel)
self.assertEqual(reply, [{"foo": "bar"}])
self.assertTrue(_callback_called[0])
ticket = uuid()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{"biz": "boz"}])
