Beispiel #1
0
 def run(self, set, body, interval=1, max_retries=None, **kwargs):
     if not isinstance(set, TaskSet):
         set = TaskSet(set)
     r = []
     setid = gen_unique_id()
     for task in set.tasks:
         uuid = gen_unique_id()
         task.options.update(task_id=uuid, chord=body)
         r.append(current_app.AsyncResult(uuid))
     current_app.TaskSetResult(setid, r).save()
     self.backend.on_chord_apply(setid, body, interval, max_retries)
     return set.apply_async(taskset_id=setid)
Beispiel #2
0
 def run(self, set, body, interval=1, max_retries=None, **kwargs):
     if not isinstance(set, TaskSet):
         set = TaskSet(set)
     r = []
     setid = gen_unique_id()
     for task in set.tasks:
         uuid = gen_unique_id()
         task.options.update(task_id=uuid, chord=body)
         r.append(current_app.AsyncResult(uuid))
     current_app.TaskSetResult(setid, r).save()
     self.backend.on_chord_apply(setid, body, interval, max_retries)
     return set.apply_async(taskset_id=setid)
Beispiel #3
0
        def with_ctypes_masked():
            from kombu.utils import ctypes, gen_unique_id

            self.assertIsNone(ctypes)
            uuid = gen_unique_id()
            self.assertTrue(uuid)
            self.assertIsInstance(uuid, basestring)
Beispiel #4
0
        def with_ctypes_masked():
            from kombu.utils import ctypes, gen_unique_id

            self.assertIsNone(ctypes)
            uuid = gen_unique_id()
            self.assertTrue(uuid)
            self.assertIsInstance(uuid, basestring)
Beispiel #5
0
    def test_basic_publish__get__consume__restore(self, n="test_basic_publish"):
        c = memory_client().channel()

        c.exchange_declare(n)
        c.queue_declare(n)
        c.queue_bind(n, n, n)
        c.queue_declare(n + "2")
        c.queue_bind(n + "2", n, n)

        m = c.prepare_message("nthex quick brown fox...")
        c.basic_publish(m, n, n)

        r1 = c.message_to_python(c.basic_get(n))
        self.assertTrue(r1)
        self.assertEqual(r1.body, "nthex quick brown fox...".encode("utf-8"))
        self.assertIsNone(c.basic_get(n))

        consumer_tag = gen_unique_id()

        c.basic_consume(n + "2", False, consumer_tag=consumer_tag, callback=lambda *a: None)
        self.assertIn(n + "2", c._active_queues)
        r2, _ = c.drain_events()
        r2 = c.message_to_python(r2)
        self.assertEqual(r2.body, "nthex quick brown fox...".encode("utf-8"))
        self.assertEqual(r2.delivery_info["exchange"], n)
        self.assertEqual(r2.delivery_info["routing_key"], n)
        self.assertRaises(virtual.Empty, c.drain_events)
        c.basic_cancel(consumer_tag)

        c._restore(r2)
        r3 = c.message_to_python(c.basic_get(n))
        self.assertTrue(r3)
        self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8"))
        self.assertIsNone(c.basic_get(n))
Beispiel #6
0
    def _broadcast(self, command, arguments=None, destination=None,
            reply=False, timeout=1, limit=None, callback=None, channel=None):
        arguments = arguments or {}
        reply_ticket = reply and gen_unique_id() or None

        if destination is not None and \
                not isinstance(destination, (list, tuple)):
            raise ValueError("destination must be a list/tuple not %s" % (
                    type(destination)))

        # Set reply limit to number of destinations (if specificed)
        if limit is None and destination:
            limit = destination and len(destination) or None

        chan = channel or self.connection.channel()
        try:
            if reply_ticket:
                self.get_reply_queue(reply_ticket)(chan).declare()

            self._publish(command, arguments, destination=destination,
                                              reply_ticket=reply_ticket,
                                              channel=chan)

            if reply_ticket:
                return self._collect(reply_ticket, limit=limit,
                                                   timeout=timeout,
                                                   callback=callback,
                                                   channel=chan)
        finally:
            channel or chan.close()
 def test_serializable(self):
     c = client().channel()
     data = c.prepare_message("the quick brown fox...")
     tag = data["properties"]["delivery_tag"] = gen_unique_id()
     message = c.message_to_python(data)
     dict_ = message.serializable()
     self.assertEqual(dict_["body"], "the quick brown fox...")
     self.assertEqual(dict_["properties"]["delivery_tag"], tag)
Beispiel #8
0
 def test_error_mail_sender(self):
     x = ErrorMail.subject % {
         "name": "task_name",
         "id": gen_unique_id(),
         "exc": "FOOBARBAZ",
         "hostname": "lana"
     }
     self.assertTrue(x)
Beispiel #9
0
 def test_serializable(self):
     c = client().channel()
     data = c.prepare_message("the quick brown fox...")
     tag = data["properties"]["delivery_tag"] = gen_unique_id()
     message = c.message_to_python(data)
     dict_ = message.serializable()
     self.assertEqual(dict_["body"],
                      "the quick brown fox...".encode("utf-8"))
     self.assertEqual(dict_["properties"]["delivery_tag"], tag)
    def test_create(self):
        c = client().channel()
        data = c.prepare_message("the quick brown fox...")
        tag = data["properties"]["delivery_tag"] = gen_unique_id()
        message = c.message_to_python(data)
        self.assertIsInstance(message, virtual.Message)
        self.assertIs(message, c.message_to_python(message))

        self.assertEqual(message.body, "the quick brown fox...")
        self.assertTrue(message.delivery_tag, tag)
Beispiel #11
0
    def test_can_consume(self, stdout, stderr):
        _restored = []

        class RestoreChannel(virtual.Channel):
            do_restore = True

            def _restore(self, message):
                _restored.append(message)

        self.assertTrue(self.q.can_consume())
        for i in range(self.q.prefetch_count - 1):
            self.q.append(i, gen_unique_id())
            self.assertTrue(self.q.can_consume())
        self.q.append(i + 1, gen_unique_id())
        self.assertFalse(self.q.can_consume())

        tag1 = self.q._delivered.keys()[0]
        self.q.ack(tag1)
        self.assertTrue(self.q.can_consume())

        tag2 = gen_unique_id()
        self.q.append(i + 2, tag2)
        self.assertFalse(self.q.can_consume())
        self.q.reject(tag2)
        self.assertTrue(self.q.can_consume())

        self.q.channel = RestoreChannel(self.q.channel.connection)
        tag3 = gen_unique_id()
        self.q.append(i + 3, tag3)
        self.q.reject(tag3, requeue=True)
        self.q._flush()
        self.q.restore_unacked_once()
        self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1])
        self.assertTrue(self.q._delivered.restored)
        self.assertFalse(self.q._delivered)

        self.q.restore_unacked_once()
        self.q._delivered.restored = False
        self.q.restore_unacked_once()

        self.assertTrue(stderr.getvalue())
        self.assertFalse(stdout.getvalue())
Beispiel #12
0
    def test_can_consume(self, stdout, stderr):
        _restored = []

        class RestoreChannel(virtual.Channel):
            do_restore = True

            def _restore(self, message):
                _restored.append(message)

        self.assertTrue(self.q.can_consume())
        for i in range(self.q.prefetch_count - 1):
            self.q.append(i, gen_unique_id())
            self.assertTrue(self.q.can_consume())
        self.q.append(i + 1, gen_unique_id())
        self.assertFalse(self.q.can_consume())

        tag1 = self.q._delivered.keys()[0]
        self.q.ack(tag1)
        self.assertTrue(self.q.can_consume())

        tag2 = gen_unique_id()
        self.q.append(i + 2, tag2)
        self.assertFalse(self.q.can_consume())
        self.q.reject(tag2)
        self.assertTrue(self.q.can_consume())

        self.q.channel = RestoreChannel(self.q.channel.connection)
        tag3 = gen_unique_id()
        self.q.append(i + 3, tag3)
        self.q.reject(tag3, requeue=True)
        self.q._flush()
        self.q.restore_unacked_once()
        self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1])
        self.assertTrue(self.q._delivered.restored)
        self.assertFalse(self.q._delivered)

        self.q.restore_unacked_once()
        self.q._delivered.restored = False
        self.q.restore_unacked_once()

        self.assertTrue(stderr.getvalue())
        self.assertFalse(stdout.getvalue())
Beispiel #13
0
    def create_binding(self, queue):
        """Get binding item for queue.

        Creates the item if it doesn't exist.

        """
        item = self.get_queue(queue)
        if item:
            return item, item["id"]
        id = gen_unique_id()
        return self.new_item(id), id
Beispiel #14
0
    def test_create(self):
        c = client().channel()
        data = c.prepare_message("the quick brown fox...")
        tag = data["properties"]["delivery_tag"] = gen_unique_id()
        message = c.message_to_python(data)
        self.assertIsInstance(message, virtual.Message)
        self.assertIs(message, c.message_to_python(message))

        self.assertEqual(message.body,
                         "the quick brown fox...".encode("utf-8"))
        self.assertTrue(message.delivery_tag, tag)
Beispiel #15
0
Datei: SQS.py Projekt: mher/kombu
    def create_binding(self, queue):
        """Get binding item for queue.

        Creates the item if it doesn't exist.

        """
        item = self.get_queue(queue)
        if item:
            return item, item["id"]
        id = gen_unique_id()
        return self.new_item(id), id
Beispiel #16
0
    def test_reply__collect(self):
        mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
        exchange = mailbox.reply_exchange.name

        ticket = gen_unique_id()
        mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
        mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
        _callback_called = [False]

        def callback(body):
            _callback_called[0] = True

        channel = self.connection.channel()
        reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel)
        self.assertEqual(reply, [{"foo": "bar"}])
        self.assertTrue(_callback_called[0])

        ticket = gen_unique_id()
        mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
        mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
        reply = mailbox._collect(ticket, limit=1, channel=channel)
        self.assertEqual(reply, [{"biz": "boz"}])
Beispiel #17
0
    def test_reply__collect(self):
        mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
        exchange = mailbox.reply_exchange.name

        ticket = gen_unique_id()
        mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
        mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
        _callback_called = [False]

        def callback(body):
            _callback_called[0] = True

        channel = self.connection.channel()
        reply = mailbox._collect(ticket, limit=1, callback=callback,
                                                  channel=channel)
        self.assertEqual(reply, [{"foo": "bar"}])
        self.assertTrue(_callback_called[0])

        ticket = gen_unique_id()
        mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
        mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
        reply = mailbox._collect(ticket, limit=1, channel=channel)
        self.assertEqual(reply, [{"biz": "boz"}])
Beispiel #18
0
 def __init__(self,
              addrport='',
              id=None,
              loglevel=logging.INFO,
              logfile=None,
              without_httpd=False,
              numc=2,
              sup_interval=None,
              ready_event=None,
              colored=None,
              **kwargs):
     self.id = id or gen_unique_id()
     if isinstance(addrport, basestring):
         addr, _, port = addrport.partition(':')
         addrport = (addr, int(port) if port else 8000)
     self.addrport = addrport
     self.connection = celery.broker_connection()
     self.without_httpd = without_httpd
     self.logfile = logfile
     self.loglevel = loglevel
     self.numc = numc
     self.ready_event = ready_event
     self.exit_request = Event()
     self.colored = colored or term.colored(enabled=False)
     self.httpd = None
     gSup = find_symbol(self, self.intsup_cls)
     if not self.without_httpd:
         self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport),
                              signals.httpd_ready)
     self.supervisor = gSup(
         instantiate(self, self.supervisor_cls, sup_interval),
         signals.supervisor_ready)
     self.controllers = [
         gSup(
             instantiate(self,
                         self.controller_cls,
                         id='%s.%s' % (self.id, i),
                         connection=self.connection,
                         branch=self), signals.controller_ready)
         for i in xrange(1, numc + 1)
     ]
     c = [self.supervisor] + self.controllers + [self.httpd]
     c = self.components = list(filter(None, c))
     self._components_ready = dict(
         zip([z.thread for z in c], [False] * len(c)))
     for controller in self.controllers:
         if hasattr(controller.thread, 'presence'):
             self._components_ready[controller.thread.presence] = False
     self._components_shutdown = dict(self._components_ready)
     super(Branch, self).__init__()
Beispiel #19
0
 def __init__(self, addrport='', id=None, loglevel=logging.INFO,
         logfile=None, without_httpd=False, numc=2, sup_interval=None,
         ready_event=None, colored=None, **kwargs):
     self.id = id or gen_unique_id()
     if isinstance(addrport, basestring):
         addr, _, port = addrport.partition(':')
         addrport = (addr, int(port) if port else 8000)
     self.addrport = addrport
     self.connection = celery.broker_connection()
     self.without_httpd = without_httpd
     self.logfile = logfile
     self.loglevel = loglevel
     self.numc = numc
     self.ready_event = ready_event
     self.exit_request = Event()
     self.colored = colored or term.colored(enabled=False)
     self.httpd = None
     gSup = find_symbol(self, self.intsup_cls)
     if not self.without_httpd:
         self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport),
                           signals.httpd_ready)
     self.supervisor = gSup(instantiate(self, self.supervisor_cls,
                             sup_interval), signals.supervisor_ready)
     self.controllers = [gSup(instantiate(self, self.controller_cls,
                                id='%s.%s' % (self.id, i),
                                connection=self.connection,
                                branch=self),
                              signals.controller_ready)
                             for i in xrange(1, numc + 1)]
     c = [self.supervisor] + self.controllers + [self.httpd]
     c = self.components = list(filter(None, c))
     self._components_ready = dict(zip([z.thread for z in c],
                                       [False] * len(c)))
     for controller in self.controllers:
         if hasattr(controller.thread, 'presence'):
             self._components_ready[controller.thread.presence] = False
     self._components_shutdown = dict(self._components_ready)
     super(Branch, self).__init__()
Beispiel #20
0
	def __init__(self,
				server_id = None,
				amqp_host='localhost', 
				amqp_user ='******',
				amqp_password='******',
				amqp_vhost='/',
				amqp_port=5672,
				ssl=False,
				timeout=0):
		
		
		self.logger = logging.getLogger('callme.proxy')
		self.timeout = 0
		self.is_received = False
		self.connection = BrokerConnection(hostname=amqp_host,
							  userid=amqp_user,
							  password=amqp_password,
							  virtual_host=amqp_vhost,
							  port=amqp_port,
							  ssl=ssl)
		self.channel = self.connection.channel()
		self.timeout = timeout
		my_uuid = gen_unique_id()
		self.reply_id = "client_"+amqp_user+"_ex_" + my_uuid
		self.logger.debug("Queue ID: %s" %self.reply_id)
		src_exchange = Exchange(self.reply_id, "direct", durable=False 
							,auto_delete=True)
		src_queue = Queue("client_"+amqp_user+"_queue_"+my_uuid, exchange=src_exchange, 
						auto_delete=True,
						durable=False)
		
		# must declare in advance so reply message isn't
   		# published before.
		src_queue(self.channel).declare()
		
		
		consumer = Consumer(channel=self.channel, queues=src_queue, callbacks=[self._on_response])
		consumer.consume()		
Beispiel #21
0
    def test_basic_publish__get__consume__restore(self,
            n="test_basic_publish"):
        c = memory_client().channel()

        c.exchange_declare(n)
        c.queue_declare(n)
        c.queue_bind(n, n, n)
        c.queue_declare(n + "2")
        c.queue_bind(n + "2", n, n)

        m = c.prepare_message("nthex quick brown fox...")
        c.basic_publish(m, n, n)

        r1 = c.message_to_python(c.basic_get(n))
        self.assertTrue(r1)
        self.assertEqual(r1.body,
                         "nthex quick brown fox...".encode("utf-8"))
        self.assertIsNone(c.basic_get(n))

        consumer_tag = gen_unique_id()

        c.basic_consume(n + "2", False, consumer_tag=consumer_tag,
                                        callback=lambda *a: None)
        self.assertIn(n + "2", c._active_queues)
        r2, _ = c.drain_events()
        r2 = c.message_to_python(r2)
        self.assertEqual(r2.body,
                         "nthex quick brown fox...".encode("utf-8"))
        self.assertEqual(r2.delivery_info["exchange"], n)
        self.assertEqual(r2.delivery_info["routing_key"], n)
        self.assertRaises(virtual.Empty, c.drain_events)
        c.basic_cancel(consumer_tag)

        c._restore(r2)
        r3 = c.message_to_python(c.basic_get(n))
        self.assertTrue(r3)
        self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8"))
        self.assertIsNone(c.basic_get(n))
Beispiel #22
0
 def test_gen_unique_id(self):
     i1 = utils.gen_unique_id()
     i2 = utils.gen_unique_id()
     self.assertIsInstance(i1, str)
     self.assertNotEqual(i1, i2)
Beispiel #23
0
 def __call__(self, body, **options):
     uuid = body.options.setdefault("task_id", gen_unique_id())
     self.Chord.apply_async((list(self.tasks), body), self.options,
                             **options)
     return body.type.app.AsyncResult(uuid)
Beispiel #24
0
 def test_error_mail_sender(self):
     x = ErrorMail.subject % {"name": "task_name",
                              "id": gen_unique_id(),
                              "exc": "FOOBARBAZ",
                              "hostname": "lana"}
     self.assertTrue(x)
Beispiel #25
0
 def test_gen_unique_id(self):
     i1 = utils.gen_unique_id()
     i2 = utils.gen_unique_id()
     self.assertIsInstance(i1, str)
     self.assertNotEqual(i1, i2)
Beispiel #26
0
 def create_queue(self, field_value):
     return Queue(gen_unique_id(), self.exchange, field_value,
                  auto_delete=self.auto_delete)
Beispiel #27
0
 def __call__(self, body, **options):
     uuid = body.options.setdefault("task_id", gen_unique_id())
     self.Chord.apply_async((list(self.tasks), body), self.options,
                             **options)
     return body.type.app.AsyncResult(uuid)
Beispiel #28
0
 def create_queue(self, field_value):
     return Queue(gen_unique_id(),
                  self.exchange,
                  field_value,
                  auto_delete=self.auto_delete)