def test_reply__collect(self): mailbox = pidbox.Mailbox('test_reply__collect')(self.connection) exchange = mailbox.reply_exchange.name channel = self.connection.channel() mailbox.reply_queue(channel).declare() ticket = uuid() mailbox._publish_reply({'foo': 'bar'}, exchange, mailbox.oid, ticket) _callback_called = [False] def callback(body): _callback_called[0] = True reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel) self.assertEqual(reply, [{'foo': 'bar'}]) self.assertTrue(_callback_called[0]) ticket = uuid() mailbox._publish_reply({'biz': 'boz'}, exchange, mailbox.oid, ticket) reply = mailbox._collect(ticket, limit=1, channel=channel) self.assertEqual(reply, [{'biz': 'boz'}]) de = mailbox.connection.drain_events = Mock() de.side_effect = socket.timeout mailbox._collect(ticket, limit=1, channel=channel)
def test_reply__collect(self): mailbox = pidbox.Mailbox("test_reply__collect")(self.connection) exchange = mailbox.reply_exchange.name ticket = uuid() mailbox.get_reply_queue(ticket)(self.connection.channel()).declare() mailbox._publish_reply({"foo": "bar"}, exchange, ticket) _callback_called = [False] def callback(body): _callback_called[0] = True channel = self.connection.channel() reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel) self.assertEqual(reply, [{"foo": "bar"}]) self.assertTrue(_callback_called[0]) ticket = uuid() mailbox.get_reply_queue(ticket)(self.connection.channel()).declare() mailbox._publish_reply({"biz": "boz"}, exchange, ticket) reply = mailbox._collect(ticket, limit=1, channel=channel) self.assertEqual(reply, [{"biz": "boz"}]) de = mailbox.connection.drain_events = Mock() de.side_effect = socket.timeout mailbox._collect(ticket, limit=1, channel=channel)
def test_stop(self, conn): ag, a, id1, id2 = dA(conn), A(), uuid(), uuid() ag.state.spawn(qualname(a), id1) ag.state.spawn(qualname(a), id2) self.assertEquals(len(ag.state.registry), 2) actor1, actor2 = ag.state.registry[id1], ag.state.registry[id2] ag.stop() self.assertEquals(len(ag.state.registry), 2) self.assertEquals(actor1.consumer.channel.queues, {}) self.assertEquals(actor2.consumer.channel.queues, {})
def test_gather_kwargs(self, conn, collect): actor = Actor(conn) ares = AsyncResult(uuid(), actor) prev_to_python = ares.to_python new_to_python = lambda x, propagate = True: x ares.to_python = new_to_python # Test default kwargs, # nothing is passed, the actor does not have agent assigned self.assert_gather_kwargs( ares, collect, {}, timeout=actor.default_timeout, ignore_timeout=False) # limit - set the default agent limit if NONE is set # Test default kwargs, nothing is passed, # the actor does have default agent assigned actor.agent = dAgent(conn) self.assert_gather_kwargs( ares, collect, {}, timeout=actor.default_timeout, limit=None, ignore_timeout=False) # limit - set the default agent limit if NONE is set # Test default kwargs, nothing is passed, # the actor does have agent with custom scatter limit assigned ag = Ag(conn) actor.agent = ag self.assert_gather_kwargs( ares, collect, {}, timeout=actor.default_timeout, limit=ag.get_default_scatter_limit()) # pass all args actor.agent = Ag(conn) timeout, ignore_timeout, limit = 200.0, False, uuid() self.assert_gather_kwargs( ares, collect, {'timeout': timeout, 'ignore_timeout': ignore_timeout, 'limit': limit}, timeout=timeout, limit=limit, ignore_timeout=ignore_timeout) # ig ignore_tiemout is passed, # the custom logic for limit is not applies actor.agent = None timeout, ignore_timeout = 200.0, True self.assert_gather_kwargs( ares, collect, {'timeout': timeout, 'ignore_timeout': ignore_timeout}, timeout=timeout, ignore_timeout=ignore_timeout) ares.to_python = prev_to_python
def MockMessage(self, id=None, receipt_handle=None, body=None): m = Mock(name='message') m.id = id or uuid() m.receipt_handle = receipt_handle or uuid() m._body = body def _get_body(): return m._body m.get_body.side_effect = _get_body def _set_body(value): m._body = value m.set_body.side_effect = _set_body return m
def test_basic_publish__get__consume__restore(self, n="test_basic_publish"): c = memory_client().channel() c.exchange_declare(n) c.queue_declare(n) c.queue_bind(n, n, n) c.queue_declare(n + "2") c.queue_bind(n + "2", n, n) m = c.prepare_message("nthex quick brown fox...") c.basic_publish(m, n, n) r1 = c.message_to_python(c.basic_get(n)) self.assertTrue(r1) self.assertEqual(r1.body, "nthex quick brown fox...".encode("utf-8")) self.assertIsNone(c.basic_get(n)) consumer_tag = uuid() c.basic_consume(n + "2", False, consumer_tag=consumer_tag, callback=lambda *a: None) self.assertIn(n + "2", c._active_queues) r2, _ = c.drain_events() r2 = c.message_to_python(r2) self.assertEqual(r2.body, "nthex quick brown fox...".encode("utf-8")) self.assertEqual(r2.delivery_info["exchange"], n) self.assertEqual(r2.delivery_info["routing_key"], n) with self.assertRaises(virtual.Empty): c.drain_events() c.basic_cancel(consumer_tag) c._restore(r2) r3 = c.message_to_python(c.basic_get(n)) self.assertTrue(r3) self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8")) self.assertIsNone(c.basic_get(n))
def count_to(self, target): token = uuid() init = 0 self.targets[token] = (target, init) self.adder.throw('add_one', {'i': init, 'token': token}, nowait=True, callback='count', ckwargs={'token': token})
def handle(self, *args, **options): headers = { 'correlation_id': uuid(), 'sequence': 0, 'date': dpn_strftime(datetime.now()), } body = { "message_name": "registry-item-create", "dpn_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "local_id": "APTRUST-282dcbdd-c16b-42f1-8c21-0dd7875fb94e", "first_node_name": "aptrust", "replicating_node_names": ["hathi", "chron", "sdr"], "version_number": 1, "previous_version_object_id": "null", "forward_version_object_id": "null", "first_version_object_id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", "fixity_algorithm": "sha256", "fixity_value": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", "lastfixity_date": "2013-01-18T09:49:28-0800", "creation_date": "2013-01-05T09:49:28-0800", "last_modified_date": "2013-01-05T09:49:28-0800", "bag_size": 65536, "brightening_object_id": ["a02de3cd-a74b-4cc6-adec-16f1dc65f726", "C92de3cd-a789-4cc6-adec-16a40c65f726", ], "rights_object_id": ["0df688d4-8dfb-4768-bee9-639558f40488", ], "object_type": "data", } msg = RegistryItemCreate(headers, body) msg.send(DPN_BROADCAST_KEY)
def _freeze_gid(self, options): # remove task_id and use that as the group_id, # if we don't remove it then every task will have the same id... options = dict(self.options, **options) options['group_id'] = group_id = ( options.pop('task_id', uuid())) return options, group_id, options.get('root_id')
def CallServer(self, method, args = None): try: LOG.debug(_("strBroker : %s "), self._strBroker) connection = BrokerConnection(self._strBroker) # create the response channel respQueueName = self._respQueueName + str(uuid()) respconnection = BrokerConnection(self._strBroker) respQueue = respconnection.SimpleQueue(respQueueName, queue_opts = {'durable': False, 'auto_delete': True}, exchange_opts = {'delivery_mode' : 1, 'auto_delete' : True, 'durable' : False}) with producers[connection].acquire(block=True) as producer: maybe_declare(task_exchange, producer.channel) payload = {"RespQueue": respQueueName, "Source": self._strBroker, 'Method': method, 'args': args} producer.publish(payload, exchange = self._exchange, serializer="json", routing_key = self._routing_key) # wait for the response resp_message = respQueue.get(block=True, timeout=1) resp_message.ack() respQueue.close() #respQueue.delete() except: LOG.debug(_("Exception caught : %s"), sys.exc_info()[0]) raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception") if type(resp_message.payload['Result']).__name__ in ('list', 'tuple'): nElems = len(resp_message.payload['Result']) if resp_message.payload['Result'][ nElems - 1 ] == -128: raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception") elif type(resp_message.payload['Result']).__name__ == 'int': if resp_message.payload['Result'] == -128: raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception") else: raise OpenCLClientException.OpenCLClientException("OpenCL Interface Exception") return resp_message.payload['Result']
def run(self, header, body, partial_args, app=None, interval=None, countdown=1, max_retries=None, propagate=None, eager=False, task_id=None, **options): app = app or self._get_app(body) propagate = (app.conf.chord_propagates if propagate is None else propagate) group_id = uuid() root_id = body.options.get('root_id') body.chord_size = self.__length_hint__() options = dict(self.options, **options) if options else self.options if options: options.pop('task_id', None) body.options.update(options) results = header.freeze( group_id=group_id, chord=body, root_id=root_id).results bodyres = body.freeze(task_id, root_id=root_id) parent = app.backend.apply_chord( header, partial_args, group_id, body, interval=interval, countdown=countdown, options=options, max_retries=max_retries, propagate=propagate, result=results) bodyres.parent = parent return bodyres
def with_ctypes_masked(): from kombu.utils import ctypes, uuid self.assertIsNone(ctypes) tid = uuid() self.assertTrue(tid) self.assertIsInstance(tid, string_t)
def test_revoke_terminate(self): request = Mock() request.id = tid = uuid() state = self.create_state() state.consumer = Mock() worker_state.task_reserved(request) try: r = control.revoke(state, tid, terminate=True) self.assertIn(tid, revoked) self.assertTrue(request.terminate.call_count) self.assertIn('terminate:', r['ok']) # unknown task id only revokes r = control.revoke(state, uuid(), terminate=True) self.assertIn('tasks unknown', r['ok']) finally: worker_state.task_ready(request)
def send_task(self, name, args=None, kwargs=None, countdown=None, eta=None, task_id=None, producer=None, connection=None, router=None, result_cls=None, expires=None, publisher=None, link=None, link_error=None, add_to_parent=True, reply_to=None, **options): task_id = task_id or uuid() producer = producer or publisher # XXX compat router = router or self.amqp.router conf = self.conf if conf.CELERY_ALWAYS_EAGER: # pragma: no cover warnings.warn(AlwaysEagerIgnored( 'CELERY_ALWAYS_EAGER has no effect on send_task', ), stacklevel=2) options = router.route(options, name, args, kwargs) if connection: producer = self.amqp.TaskProducer(connection) with self.producer_or_acquire(producer) as P: self.backend.on_task_call(P, task_id) task_id = P.publish_task( name, args, kwargs, countdown=countdown, eta=eta, task_id=task_id, expires=expires, callbacks=maybe_list(link), errbacks=maybe_list(link_error), reply_to=reply_to or self.oid, **options ) result = (result_cls or self.AsyncResult)(task_id) if add_to_parent: parent = get_current_worker_task() if parent: parent.add_trail(result) return result
def __init__(self, name=None, queue=None, **kwargs): kwargs.setdefault("exchange", entity.Exchange(name, type="fanout", auto_delete=True)) kwargs.setdefault("auto_delete", True) kwargs.setdefault("alias", name) return super(Broadcast, self).__init__( name=queue or "bcast.%s" % (uuid(), ), **kwargs)
def freeze(self, _id=None, group_id=None, chord=None, root_id=None, parent_id=None): """Finalize the signature by adding a concrete task id. The task will not be called and you should not call the signature twice after freezing it as that will result in two task messages using the same task id. Returns: ~@AsyncResult: promise of future evaluation. """ opts = self.options try: tid = opts['task_id'] except KeyError: tid = opts['task_id'] = _id or uuid() if root_id: opts['root_id'] = root_id if parent_id: opts['parent_id'] = parent_id if 'reply_to' not in opts: opts['reply_to'] = self.app.oid if group_id: opts['group_id'] = group_id if chord: opts['chord'] = chord return self.AsyncResult(tid)
def handle(self, *args, **options): msg = ReplicationInitQuery() headers = { 'correlation_id': uuid(), 'sequence': 0, 'date': dpn_strftime(datetime.now()) } msg.set_headers(**headers) body = { 'message_name': 'replication-init-query', 'replication_size': 4502, 'protocol': ['https', 'rsync'], 'dpn_object_id': uuid() } msg.set_body(**body) msg.send(DPN_BROADCAST_KEY)
def _freeze(self, _id=None): opts = self.options try: tid = opts["task_id"] except KeyError: tid = opts["task_id"] = _id or uuid() return self.AsyncResult(tid)
def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix=None, accept=None, queue_ttl=None, queue_expires=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers self.routing_key = routing_key self.node_id = node_id or uuid() self.queue_prefix = queue_prefix or self.app.conf.event_queue_prefix self.exchange = get_exchange( self.connection or self.app.connection_for_write()) self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, routing_key=self.routing_key, auto_delete=True, durable=False, queue_arguments=self._get_queue_arguments( ttl=queue_ttl, expires=queue_expires, ), ) self.clock = self.app.clock self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward if accept is None: accept = {self.app.conf.event_serializer, 'json'} self.accept = accept
def send_task(self, name, args=None, kwargs=None, countdown=None, eta=None, task_id=None, producer=None, connection=None, router=None, result_cls=None, expires=None, publisher=None, link=None, link_error=None, add_to_parent=True, group_id=None, retries=0, chord=None, reply_to=None, time_limit=None, soft_time_limit=None, root_id=None, parent_id=None, route_name=None, shadow=None, chain=None, **options): """Send task by name. :param name: Name of task to call (e.g. `"tasks.add"`). :keyword result_cls: Specify custom result class. Default is using :meth:`AsyncResult`. Otherwise supports the same arguments as :meth:`@-Task.apply_async`. """ parent = have_parent = None amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat router = router or amqp.router conf = self.conf if conf.task_always_eager: # pragma: no cover warnings.warn(AlwaysEagerIgnored( 'task_always_eager has no effect on send_task', ), stacklevel=2) options = router.route(options, route_name or name, args, kwargs) if root_id is None: parent, have_parent = self.current_worker_task, True if parent: root_id = parent.request.root_id or parent.request.id if parent_id is None: if not have_parent: parent, have_parent = self.current_worker_task, True if parent: parent_id = parent.request.id message = amqp.create_task_message( task_id, name, args, kwargs, countdown, eta, group_id, expires, retries, chord, maybe_list(link), maybe_list(link_error), reply_to or self.oid, time_limit, soft_time_limit, self.conf.task_send_sent_event, root_id, parent_id, shadow, chain, ) if connection: producer = amqp.Producer(connection) with self.producer_or_acquire(producer) as P: self.backend.on_task_call(P, task_id) amqp.send_task_message(P, name, message, **options) result = (result_cls or self.AsyncResult)(task_id) if add_to_parent: if not have_parent: parent, have_parent = self.current_worker_task, True if parent: parent.add_trail(result) return result
def _broadcast(self, command, arguments=None, destination=None, reply=False, timeout=1, limit=None, callback=None, channel=None): arguments = arguments or {} reply_ticket = reply and uuid() or None if destination is not None and \ not isinstance(destination, (list, tuple)): raise ValueError("destination must be a list/tuple not %s" % ( type(destination))) # Set reply limit to number of destinations (if specificed) if limit is None and destination: limit = destination and len(destination) or None chan = channel or self.connection.channel() try: if reply_ticket: self.get_reply_queue(reply_ticket)(chan).declare() self._publish(command, arguments, destination=destination, reply_ticket=reply_ticket, channel=chan) if reply_ticket: return self._collect(reply_ticket, limit=limit, timeout=timeout, callback=callback, channel=chan) finally: channel or chan.close()
def test_fast_trace_task(self): from celery.app import trace setup_worker_optimizations(self.app) self.assertIs(trace.trace_task_ret, trace._fast_trace_task) tid = uuid() message = TaskMessage(self.mytask.name, tid, args=[4]) assert len(message.payload) == 3 try: self.mytask.__trace__ = build_tracer( self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) self.assertIsNotNone(runtime) self.assertIsInstance(runtime, numbers.Real) finally: reset_worker_optimizations() self.assertIs(trace.trace_task_ret, trace._trace_task_ret) delattr(self.mytask, '__trace__') failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, ) self.assertFalse(failed) self.assertEqual(res, repr(4 ** 4)) self.assertIsNotNone(runtime) self.assertIsInstance(runtime, numbers.Real)
def test_execute_using_pool__revoked(self): tid = uuid() job = self.zRequest(id=tid, revoked_tasks={tid}) job.revoked = Mock() job.revoked.return_value = True with self.assertRaises(TaskRevokedError): job.execute_using_pool(self.pool)
def test_on_success__with_events(self): job = self.zRequest(id=uuid()) job.send_event = Mock(name='send_event') job.on_success((False, 'foo', 1.0)) job.send_event.assert_called_with( 'task-succeeded', result='foo', runtime=1.0, )
def count_to(self, target): token = uuid() init = 0 self.targets[token] = (target, init) self.adder.throw( "add_one", {"i": init, "token": token}, nowait=True, callback="count", ckwargs={"token": token} )
def test_gather(self, conn): def collect_replies(): yield 1 yield 2 yield 3 ticket = uuid() actor = Actor(conn) actor._collect_replies = Mock(return_value=collect_replies()) ares = AsyncResult(ticket, actor) ares.to_python = Mock() all = ares.gather() list(all) actor._collect_replies.assert_caleld_once_with(conn, ANY, ticket) self.assertEqual(ares.to_python.call_count, len(list(collect_replies()))) # test that the to_python is applied to all results actor._collect_replies.reset_mock() actor._collect_replies = Mock(return_value=collect_replies()) prev_to_python = ares.to_python new_to_python = lambda x, propagate = True: 'called_%s' % x ares.to_python = new_to_python all = ares.gather() vals = list(all) expected_vals = [new_to_python(i) for i in collect_replies()] actor._collect_replies.assert_caleld_once_with(conn, ANY, ticket) self.assertEqual(vals, expected_vals) ares.to_python = prev_to_python
def test_execute(self): tid = uuid() job = self.xRequest(id=tid, args=[4], kwargs={}) self.assertEqual(job.execute(), 256) meta = self.mytask.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.SUCCESS) self.assertEqual(meta['result'], 256)
def test_get_set_forget(self): b = self.Backend(app=self.app) tid = uuid() b.store_result(tid, 42, states.SUCCESS) self.assertEqual(b.get_status(tid), states.SUCCESS) self.assertEqual(b.get_result(tid), 42) b.forget(tid) self.assertEqual(b.get_status(tid), states.PENDING)
def test_revoke_with_name_not_in_registry(self): tid = uuid() m = {'method': 'revoke', 'destination': hostname, 'arguments': {'task_id': tid, 'task_name': 'xxxxxxxxx33333333388888'}} self.panel.handle_message(m, None) self.assertIn(tid, revoked)
def test_process_cleanup_fails(self, _logger): self.mytask.backend = Mock() self.mytask.backend.process_cleanup = Mock(side_effect=KeyError()) tid = uuid() ret = jail(self.app, tid, self.mytask.name, [2], {}) self.assertEqual(ret, 4) self.mytask.backend.mark_as_done.assert_called() self.assertIn('Process cleanup failed', _logger.error.call_args[0][0])
def __call__(self, body=None, **kwargs): _chord = self.Chord body = (body or self.kwargs['body']).clone() kwargs = dict(self.kwargs, body=body, **kwargs) if _chord.app.conf.CELERY_ALWAYS_EAGER: return self.apply((), kwargs) callback_id = body.options.setdefault('task_id', uuid()) _chord(**kwargs) return _chord.AsyncResult(callback_id)
def apply_async(self, args, kwargs, **options): args, kw = self.serialize_args(args, kwargs) # Let's see if this is a retry. An existing task means yes. # If it is one, we'll call _apply_async directly later on. task = getattr(self.request, 'task', None) task_id = options.get('task_id', None) # if task is not None we are in a retry and site_path and # authorized_userid are already in kw if task is None: kw['site_path'] = '/'.join(api.portal.get().getPhysicalPath()) kw['authorized_userid'] = api.user.get_current().getId() without_transaction = options.pop('without_transaction', False) celery = getCelery() if task_id is None: # Here we cheat a little: since we will not start the task # up until the transaction is done, # we cannot give back to whoever called apply_async # its much beloved AsyncResult. # But we can actually pass the task a specific task_id # (although it's not very documented) # and an AsyncResult at this point is just that id, basically. task_id = uuid() else: # If this is a retry, task_id will be in the options. # Get rid of it to avoid an error. del options['task_id'] # Construct a fake result if celery.conf.task_always_eager: result_ = EagerResult(task_id, None, states.PENDING, None) else: result_ = result.AsyncResult(task_id) # Note: one might be tempted to turn this into a datamanager. # This would result in two wrong things happening: # * A "commit within a commit" triggered by the function runner # when CELERY_TASK_ALWAYS_EAGER is set, # leading to the first invoked commit cleanup failing # because the inner commit already cleaned up. # * An async task failing in eager mode would also rollback # the whole transaction, which is not desiderable. # Consider the case where the syncronous code constructs an object # and the async task updates it, if we roll back everything # then also the original content construction goes away # (even if, in and by itself, worked) if without_transaction or celery.conf.task_always_eager or task: return self._apply_async(args, kw, result_, celery, task_id, options) else: queue_task_after_commit(args, kw, self, task_id, options) # Return the "fake" result ID return result_
def test_execute_jail_failure(self): ret = jail( self.app, uuid(), self.mytask_raising.name, [4], {}, ) self.assertIsInstance(ret, ExceptionInfo) self.assertTupleEqual(ret.exception.args, (4, ))
def test_state_stop_actor_by_id(self, conn, consumer): ag, a, id = dA(conn), A(), uuid() ag.state.spawn(qualname(a), id) self.assertEquals(len(ag.state.registry), 1) actor = ag.state.registry[id] ag.state.kill(id) self.assertEquals(ag.state.registry, {}) actor.consumer.cancel.assert_called_once_with()
def test_on_success__SystemExit(self, errors=(SystemExit, KeyboardInterrupt)): for exc in errors: einfo = None try: raise exc() except exc: einfo = ExceptionInfo() with self.assertRaises(exc): self.zRequest(id=uuid()).on_success((True, einfo, 1.0))
def election(self): type = self.type app = type.app tid = self.options.get('task_id') or uuid() with app.producer_or_acquire(None) as P: props = type.backend.on_task_call(P, tid) app.control.election(tid, 'task', self.clone(task_id=tid, **props), connection=P.connection) return type.AsyncResult(tid)
def test_marked_as_started(self): _started = [] def store_result(tid, meta, state, **kwargs): if state == states.STARTED: _started.append(tid) self.mytask.backend.store_result = Mock(name='store_result') self.mytask.backend.store_result.side_effect = store_result self.mytask.track_started = True tid = uuid() jail(self.app, tid, self.mytask.name, [2], {}) self.assertIn(tid, _started) self.mytask.ignore_result = True tid = uuid() jail(self.app, tid, self.mytask.name, [2], {}) self.assertNotIn(tid, _started)
def test_get(self): id1, id2 = uuid(), uuid() def gather(): yield id1 yield id2 # test that it calls gather with limit = 1 and kwargs ares = self.get_async_result() ares.gather = Mock(return_value=['1']) ares.get() ares.gather.assert_called_once_with(limit=1) ares.gather.reset_mock() kwargs = {'timeout': 100, 'ignore_timeout': False, 'foo': 'bar', 'propaget': True} ares.get(**kwargs) ares.gather.assert_called_once_with(**dict(kwargs, limit=1)) ares.gather.reset_mock() kwargs = {'timeout': 100, 'ignore_timeout': False, 'limit': 10} ares.get(**kwargs) ares.gather.assert_called_once_with(**kwargs) ares.gather.reset_mock() # it returns the first value of whatever gather returns ares.gather = Mock(return_value=gather()) res = ares.get() self.assertEqual(res, id1) # if gather does not return result: # self.NoReplyError('No reply received within time constraint') ares.gather = Mock(return_value=None) with self.assertRaises(ares.NoReplyError): ares.get() ares.gather.reset_mock() ares.gather = Mock(return_value={}) with self.assertRaises(ares.NoReplyError): ares.get()
def test_can_consume(self, stdout, stderr): _restored = [] class RestoreChannel(virtual.Channel): do_restore = True def _restore(self, message): _restored.append(message) self.assertTrue(self.q.can_consume()) for i in range(self.q.prefetch_count - 1): self.q.append(i, uuid()) self.assertTrue(self.q.can_consume()) self.q.append(i + 1, uuid()) self.assertFalse(self.q.can_consume()) tag1 = self.q._delivered.keys()[0] self.q.ack(tag1) self.assertTrue(self.q.can_consume()) tag2 = uuid() self.q.append(i + 2, tag2) self.assertFalse(self.q.can_consume()) self.q.reject(tag2) self.assertTrue(self.q.can_consume()) self.q.channel = RestoreChannel(self.q.channel.connection) tag3 = uuid() self.q.append(i + 3, tag3) self.q.reject(tag3, requeue=True) self.q._flush() self.q.restore_unacked_once() self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1]) self.assertTrue(self.q._delivered.restored) self.assertFalse(self.q._delivered) self.q.restore_unacked_once() self.q._delivered.restored = False self.q.restore_unacked_once() self.assertTrue(stderr.getvalue()) self.assertFalse(stdout.getvalue())
def test_state_select_returns_from_registry(self, conn): class B(Actor): pass ag = dAgent(conn) id1, id2 = uuid(), uuid() with self.assertRaises(Actor.Next): ag.state.select(qualname(A)) ag.state.registry[id1] = A() key = ag.state.select(qualname(A)) self.assertEqual(key, id1) ag.state.registry[id2] = B(conn) keyA = ag.state.select(qualname(A)) keyB = ag.state.select(qualname(B)) self.assertEqual(keyA, id1) self.assertEqual(keyB, id2)
def create_binding(self, queue): """Get binding item for queue. Creates the item if it doesn't exist. """ item = self.get_queue(queue) if item: return item, item['id'] id = uuid() return self.new_item(id), id
def test_create(self): c = client().channel() data = c.prepare_message("the quick brown fox...") tag = data["properties"]["delivery_tag"] = uuid() message = c.message_to_python(data) self.assertIsInstance(message, virtual.Message) self.assertIs(message, c.message_to_python(message)) self.assertEqual(message.body, "the quick brown fox...".encode("utf-8")) self.assertTrue(message.delivery_tag, tag)
def queue_declare(self, queue=None, passive=False, **kwargs): """Declare queue.""" queue = queue or 'amq.gen-%s' % uuid() if passive and not self._has_queue(queue, **kwargs): raise StdChannelError( '404', 'NOT_FOUND - no queue {0!r} in vhost {1!r}'.format( queue, self.connection.client.virtual_host or '/'), (50, 10), 'Channel.queue_declare') else: self._new_queue(queue, **kwargs) return queue, self._size(queue), 0
def test_create(self): c = client().channel() data = c.prepare_message('the quick brown fox...') tag = data['properties']['delivery_tag'] = uuid() message = c.message_to_python(data) self.assertIsInstance(message, virtual.Message) self.assertIs(message, c.message_to_python(message)) self.assertEqual(message.body, 'the quick brown fox...'.encode('utf-8')) self.assertTrue(message.delivery_tag, tag)
def test_state_spawn_when_id_not_in_registry(self, conn, consumer): ag, a, id = dA(conn), A(), uuid() self.assertEquals(ag.state.registry, {}) ag.state.spawn(qualname(a), id) self.assertEquals(len(ag.state.registry), 1) actor = ag.state.registry[id] self.assertIs(type(actor), A) self.assertIsNotNone(actor.consumer) actor.consumer.consume.assert_called_once_with()
def count_to(self, target): token = uuid() init = 0 self.targets[token] = (target, init) self.adder.throw('add_one', { 'i': init, 'token': token }, nowait=True, callback='count', ckwargs={'token': token})
def test_serializable(self): c = client().channel() body, content_type = compress('the quick brown fox...', 'gzip') data = c.prepare_message(body, headers={'compression': content_type}) tag = data['properties']['delivery_tag'] = uuid() message = c.message_to_python(data) dict_ = message.serializable() self.assertEqual(dict_['body'], 'the quick brown fox...'.encode('utf-8')) self.assertEqual(dict_['properties']['delivery_tag'], tag) self.assertNotIn('compression', dict_['headers'])
def test_revoke_with_name_not_in_registry(self): tid = uuid() m = { 'method': 'revoke', 'destination': hostname, 'arguments': { 'task_id': tid, 'task_name': 'xxxxxxxxx33333333388888' } } self.panel.handle_message(m, None) self.assertIn(tid, revoked)
def test_revoke_with_name(self): tid = uuid() m = { 'method': 'revoke', 'destination': hostname, 'arguments': { 'task_id': tid, 'task_name': self.mytask.name } } self.panel.handle_message(m, None) self.assertIn(tid, revoked)
def _test_on_failure(self, exception, **kwargs): tid = uuid() job = self.xRequest(id=tid, args=[4]) job.send_event = Mock(name='send_event') job.task.backend.mark_as_failure = Mock(name='mark_as_failure') try: raise exception except type(exception): exc_info = ExceptionInfo() job.on_failure(exc_info, **kwargs) job.send_event.assert_called() return job
def test_init(self): ticket = uuid() actor = Mock() ares = AsyncResult(ticket, actor) self.assertEquals(ares.ticket, ticket) self.assertEqual(ares.actor, actor) self.assertIsNone(ares._result) self.assertEqual(ares.Error, CellError) self.assertEqual(ares.NoReplyError, NoReplyError) with self.assertRaises(TypeError): AsyncResult(ticket)
def test_reject(self): job = self.xRequest(id=uuid()) job.on_reject = Mock(name='on_reject') job.reject(requeue=True) job.on_reject.assert_called_with( req_logger, job.connection_errors, True, ) self.assertTrue(job.acknowledged) job.on_reject.reset_mock() job.reject(requeue=True) job.on_reject.assert_not_called()
def test_execute_fail(self): tid = uuid() job = self.xRequest( name=self.mytask_raising.name, id=tid, args=[4], kwargs={}, ) self.assertIsInstance(job.execute(), ExceptionInfo) assert self.mytask_raising.backend.serializer == 'pickle' meta = self.mytask_raising.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.FAILURE) self.assertIsInstance(meta['result'], KeyError)
def freeze(self, _id=None, group_id=None, chord=None): opts = self.options try: tid = opts['task_id'] except KeyError: tid = opts['task_id'] = _id or uuid() if 'reply_to' not in opts: opts['reply_to'] = self.app.oid if group_id: opts['group_id'] = group_id if chord: opts['chord'] = chord return self.AsyncResult(tid)
def _freeze(self, _id=None): opts = self.options try: gid = opts['group'] except KeyError: gid = opts['group'] = uuid() new_tasks, results = [], [] for task in self.tasks: task = maybe_subtask(task).clone() results.append(task._freeze()) new_tasks.append(task) self.tasks = self.kwargs['tasks'] = new_tasks return GroupResult(gid, results)
def freeze(self, _id=None): opts = self.options try: gid = opts['task_id'] except KeyError: gid = opts['task_id'] = uuid() new_tasks, results = [], [] for task in self.tasks: task = maybe_signature(task, app=self._app).clone() results.append(task._freeze()) new_tasks.append(task) self.tasks = self.kwargs['tasks'] = new_tasks return self.app.GroupResult(gid, results)
def test_execute_ack(self): scratch = {'ACK': False} def on_ack(*args, **kwargs): scratch['ACK'] = True tid = uuid() job = self.xRequest(id=tid, args=[4], on_ack=on_ack) self.assertEqual(job.execute(), 256) meta = self.mytask.backend.get_task_meta(tid) self.assertTrue(scratch['ACK']) self.assertEqual(meta['result'], 256) self.assertEqual(meta['status'], states.SUCCESS)
def test_revoke(self): tid = uuid() m = {'method': 'revoke', 'destination': hostname, 'arguments': {'task_id': tid}} self.panel.handle_message(m, None) self.assertIn(tid, revoked) m = {'method': 'revoke', 'destination': 'does.not.exist', 'arguments': {'task_id': tid + 'xxx'}} self.panel.handle_message(m, None) self.assertNotIn(tid + 'xxx', revoked)
def CallServer(self, method, args=None): try: LOG.debug(_("strBroker : %s "), self._strBroker) connection = BrokerConnection(self._strBroker) # create the response channel respQueueName = self._respQueueName + str(uuid()) respconnection = BrokerConnection(self._strBroker) respQueue = respconnection.SimpleQueue(respQueueName, queue_opts={ 'durable': False, 'auto_delete': True }, exchange_opts={ 'delivery_mode': 1, 'auto_delete': True, 'durable': False }) with producers[connection].acquire(block=True) as producer: maybe_declare(task_exchange, producer.channel) payload = { "RespQueue": respQueueName, "Source": self._strBroker, 'Method': method, 'args': args } producer.publish(payload, exchange=self._exchange, serializer="json", routing_key=self._routing_key) # wait for the response resp_message = respQueue.get(block=True, timeout=1) resp_message.ack() respQueue.close() #respQueue.delete() except: LOG.debug(_("Exception caught : %s"), sys.exc_info()[0]) raise OpenCLClientException.OpenCLClientException( "OpenCL Interface Exception") if type(resp_message.payload['Result']).__name__ in ('list', 'tuple'): nElems = len(resp_message.payload['Result']) if resp_message.payload['Result'][nElems - 1] == -128: raise OpenCLClientException.OpenCLClientException( "OpenCL Interface Exception") elif type(resp_message.payload['Result']).__name__ == 'int': if resp_message.payload['Result'] == -128: raise OpenCLClientException.OpenCLClientException( "OpenCL Interface Exception") else: raise OpenCLClientException.OpenCLClientException( "OpenCL Interface Exception") return resp_message.payload['Result']
def test_reply__collect(self): mailbox = pidbox.Mailbox("test_reply__collect")(self.connection) exchange = mailbox.reply_exchange.name ticket = uuid() mailbox.get_reply_queue(ticket)(self.connection.channel()).declare() mailbox._publish_reply({"foo": "bar"}, exchange, ticket) _callback_called = [False] def callback(body): _callback_called[0] = True channel = self.connection.channel() reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel) self.assertEqual(reply, [{"foo": "bar"}]) self.assertTrue(_callback_called[0]) ticket = uuid() mailbox.get_reply_queue(ticket)(self.connection.channel()).declare() mailbox._publish_reply({"biz": "boz"}, exchange, ticket) reply = mailbox._collect(ticket, limit=1, channel=channel) self.assertEqual(reply, [{"biz": "boz"}])