def run(self, set, body, interval=1, max_retries=None, **kwargs): if not isinstance(set, TaskSet): set = TaskSet(set) r = [] setid = gen_unique_id() for task in set.tasks: uuid = gen_unique_id() task.options.update(task_id=uuid, chord=body) r.append(current_app.AsyncResult(uuid)) current_app.TaskSetResult(setid, r).save() self.backend.on_chord_apply(setid, body, interval, max_retries) return set.apply_async(taskset_id=setid)
def with_ctypes_masked(): from kombu.utils import ctypes, gen_unique_id self.assertIsNone(ctypes) uuid = gen_unique_id() self.assertTrue(uuid) self.assertIsInstance(uuid, basestring)
def test_basic_publish__get__consume__restore(self, n="test_basic_publish"): c = memory_client().channel() c.exchange_declare(n) c.queue_declare(n) c.queue_bind(n, n, n) c.queue_declare(n + "2") c.queue_bind(n + "2", n, n) m = c.prepare_message("nthex quick brown fox...") c.basic_publish(m, n, n) r1 = c.message_to_python(c.basic_get(n)) self.assertTrue(r1) self.assertEqual(r1.body, "nthex quick brown fox...".encode("utf-8")) self.assertIsNone(c.basic_get(n)) consumer_tag = gen_unique_id() c.basic_consume(n + "2", False, consumer_tag=consumer_tag, callback=lambda *a: None) self.assertIn(n + "2", c._active_queues) r2, _ = c.drain_events() r2 = c.message_to_python(r2) self.assertEqual(r2.body, "nthex quick brown fox...".encode("utf-8")) self.assertEqual(r2.delivery_info["exchange"], n) self.assertEqual(r2.delivery_info["routing_key"], n) self.assertRaises(virtual.Empty, c.drain_events) c.basic_cancel(consumer_tag) c._restore(r2) r3 = c.message_to_python(c.basic_get(n)) self.assertTrue(r3) self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8")) self.assertIsNone(c.basic_get(n))
def _broadcast(self, command, arguments=None, destination=None, reply=False, timeout=1, limit=None, callback=None, channel=None): arguments = arguments or {} reply_ticket = reply and gen_unique_id() or None if destination is not None and \ not isinstance(destination, (list, tuple)): raise ValueError("destination must be a list/tuple not %s" % ( type(destination))) # Set reply limit to number of destinations (if specificed) if limit is None and destination: limit = destination and len(destination) or None chan = channel or self.connection.channel() try: if reply_ticket: self.get_reply_queue(reply_ticket)(chan).declare() self._publish(command, arguments, destination=destination, reply_ticket=reply_ticket, channel=chan) if reply_ticket: return self._collect(reply_ticket, limit=limit, timeout=timeout, callback=callback, channel=chan) finally: channel or chan.close()
def test_serializable(self): c = client().channel() data = c.prepare_message("the quick brown fox...") tag = data["properties"]["delivery_tag"] = gen_unique_id() message = c.message_to_python(data) dict_ = message.serializable() self.assertEqual(dict_["body"], "the quick brown fox...") self.assertEqual(dict_["properties"]["delivery_tag"], tag)
def test_error_mail_sender(self): x = ErrorMail.subject % { "name": "task_name", "id": gen_unique_id(), "exc": "FOOBARBAZ", "hostname": "lana" } self.assertTrue(x)
def test_serializable(self): c = client().channel() data = c.prepare_message("the quick brown fox...") tag = data["properties"]["delivery_tag"] = gen_unique_id() message = c.message_to_python(data) dict_ = message.serializable() self.assertEqual(dict_["body"], "the quick brown fox...".encode("utf-8")) self.assertEqual(dict_["properties"]["delivery_tag"], tag)
def test_create(self): c = client().channel() data = c.prepare_message("the quick brown fox...") tag = data["properties"]["delivery_tag"] = gen_unique_id() message = c.message_to_python(data) self.assertIsInstance(message, virtual.Message) self.assertIs(message, c.message_to_python(message)) self.assertEqual(message.body, "the quick brown fox...") self.assertTrue(message.delivery_tag, tag)
def test_can_consume(self, stdout, stderr): _restored = [] class RestoreChannel(virtual.Channel): do_restore = True def _restore(self, message): _restored.append(message) self.assertTrue(self.q.can_consume()) for i in range(self.q.prefetch_count - 1): self.q.append(i, gen_unique_id()) self.assertTrue(self.q.can_consume()) self.q.append(i + 1, gen_unique_id()) self.assertFalse(self.q.can_consume()) tag1 = self.q._delivered.keys()[0] self.q.ack(tag1) self.assertTrue(self.q.can_consume()) tag2 = gen_unique_id() self.q.append(i + 2, tag2) self.assertFalse(self.q.can_consume()) self.q.reject(tag2) self.assertTrue(self.q.can_consume()) self.q.channel = RestoreChannel(self.q.channel.connection) tag3 = gen_unique_id() self.q.append(i + 3, tag3) self.q.reject(tag3, requeue=True) self.q._flush() self.q.restore_unacked_once() self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1]) self.assertTrue(self.q._delivered.restored) self.assertFalse(self.q._delivered) self.q.restore_unacked_once() self.q._delivered.restored = False self.q.restore_unacked_once() self.assertTrue(stderr.getvalue()) self.assertFalse(stdout.getvalue())
def create_binding(self, queue): """Get binding item for queue. Creates the item if it doesn't exist. """ item = self.get_queue(queue) if item: return item, item["id"] id = gen_unique_id() return self.new_item(id), id
def test_create(self): c = client().channel() data = c.prepare_message("the quick brown fox...") tag = data["properties"]["delivery_tag"] = gen_unique_id() message = c.message_to_python(data) self.assertIsInstance(message, virtual.Message) self.assertIs(message, c.message_to_python(message)) self.assertEqual(message.body, "the quick brown fox...".encode("utf-8")) self.assertTrue(message.delivery_tag, tag)
def test_reply__collect(self): mailbox = pidbox.Mailbox("test_reply__collect")(self.connection) exchange = mailbox.reply_exchange.name ticket = gen_unique_id() mailbox.get_reply_queue(ticket)(self.connection.channel()).declare() mailbox._publish_reply({"foo": "bar"}, exchange, ticket) _callback_called = [False] def callback(body): _callback_called[0] = True channel = self.connection.channel() reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel) self.assertEqual(reply, [{"foo": "bar"}]) self.assertTrue(_callback_called[0]) ticket = gen_unique_id() mailbox.get_reply_queue(ticket)(self.connection.channel()).declare() mailbox._publish_reply({"biz": "boz"}, exchange, ticket) reply = mailbox._collect(ticket, limit=1, channel=channel) self.assertEqual(reply, [{"biz": "boz"}])
def __init__(self, addrport='', id=None, loglevel=logging.INFO, logfile=None, without_httpd=False, numc=2, sup_interval=None, ready_event=None, colored=None, **kwargs): self.id = id or gen_unique_id() if isinstance(addrport, basestring): addr, _, port = addrport.partition(':') addrport = (addr, int(port) if port else 8000) self.addrport = addrport self.connection = celery.broker_connection() self.without_httpd = without_httpd self.logfile = logfile self.loglevel = loglevel self.numc = numc self.ready_event = ready_event self.exit_request = Event() self.colored = colored or term.colored(enabled=False) self.httpd = None gSup = find_symbol(self, self.intsup_cls) if not self.without_httpd: self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport), signals.httpd_ready) self.supervisor = gSup( instantiate(self, self.supervisor_cls, sup_interval), signals.supervisor_ready) self.controllers = [ gSup( instantiate(self, self.controller_cls, id='%s.%s' % (self.id, i), connection=self.connection, branch=self), signals.controller_ready) for i in xrange(1, numc + 1) ] c = [self.supervisor] + self.controllers + [self.httpd] c = self.components = list(filter(None, c)) self._components_ready = dict( zip([z.thread for z in c], [False] * len(c))) for controller in self.controllers: if hasattr(controller.thread, 'presence'): self._components_ready[controller.thread.presence] = False self._components_shutdown = dict(self._components_ready) super(Branch, self).__init__()
def __init__(self, addrport='', id=None, loglevel=logging.INFO, logfile=None, without_httpd=False, numc=2, sup_interval=None, ready_event=None, colored=None, **kwargs): self.id = id or gen_unique_id() if isinstance(addrport, basestring): addr, _, port = addrport.partition(':') addrport = (addr, int(port) if port else 8000) self.addrport = addrport self.connection = celery.broker_connection() self.without_httpd = without_httpd self.logfile = logfile self.loglevel = loglevel self.numc = numc self.ready_event = ready_event self.exit_request = Event() self.colored = colored or term.colored(enabled=False) self.httpd = None gSup = find_symbol(self, self.intsup_cls) if not self.without_httpd: self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport), signals.httpd_ready) self.supervisor = gSup(instantiate(self, self.supervisor_cls, sup_interval), signals.supervisor_ready) self.controllers = [gSup(instantiate(self, self.controller_cls, id='%s.%s' % (self.id, i), connection=self.connection, branch=self), signals.controller_ready) for i in xrange(1, numc + 1)] c = [self.supervisor] + self.controllers + [self.httpd] c = self.components = list(filter(None, c)) self._components_ready = dict(zip([z.thread for z in c], [False] * len(c))) for controller in self.controllers: if hasattr(controller.thread, 'presence'): self._components_ready[controller.thread.presence] = False self._components_shutdown = dict(self._components_ready) super(Branch, self).__init__()
def __init__(self, server_id = None, amqp_host='localhost', amqp_user ='******', amqp_password='******', amqp_vhost='/', amqp_port=5672, ssl=False, timeout=0): self.logger = logging.getLogger('callme.proxy') self.timeout = 0 self.is_received = False self.connection = BrokerConnection(hostname=amqp_host, userid=amqp_user, password=amqp_password, virtual_host=amqp_vhost, port=amqp_port, ssl=ssl) self.channel = self.connection.channel() self.timeout = timeout my_uuid = gen_unique_id() self.reply_id = "client_"+amqp_user+"_ex_" + my_uuid self.logger.debug("Queue ID: %s" %self.reply_id) src_exchange = Exchange(self.reply_id, "direct", durable=False ,auto_delete=True) src_queue = Queue("client_"+amqp_user+"_queue_"+my_uuid, exchange=src_exchange, auto_delete=True, durable=False) # must declare in advance so reply message isn't # published before. src_queue(self.channel).declare() consumer = Consumer(channel=self.channel, queues=src_queue, callbacks=[self._on_response]) consumer.consume()
def test_gen_unique_id(self): i1 = utils.gen_unique_id() i2 = utils.gen_unique_id() self.assertIsInstance(i1, str) self.assertNotEqual(i1, i2)
def __call__(self, body, **options): uuid = body.options.setdefault("task_id", gen_unique_id()) self.Chord.apply_async((list(self.tasks), body), self.options, **options) return body.type.app.AsyncResult(uuid)
def test_error_mail_sender(self): x = ErrorMail.subject % {"name": "task_name", "id": gen_unique_id(), "exc": "FOOBARBAZ", "hostname": "lana"} self.assertTrue(x)
def create_queue(self, field_value): return Queue(gen_unique_id(), self.exchange, field_value, auto_delete=self.auto_delete)