def run(self, set, body, interval=1, max_retries=None, **kwargs):
if not isinstance(set, TaskSet):
set = TaskSet(set)
r = []
setid = gen_unique_id()
for task in set.tasks:
uuid = gen_unique_id()
task.options.update(task_id=uuid, chord=body)
r.append(current_app.AsyncResult(uuid))
current_app.TaskSetResult(setid, r).save()
self.backend.on_chord_apply(setid, body, interval, max_retries)
return set.apply_async(taskset_id=setid)
def run(self, set, body, interval=1, max_retries=None, **kwargs):
if not isinstance(set, TaskSet):
set = TaskSet(set)
r = []
setid = gen_unique_id()
for task in set.tasks:
uuid = gen_unique_id()
task.options.update(task_id=uuid, chord=body)
r.append(current_app.AsyncResult(uuid))
current_app.TaskSetResult(setid, r).save()
self.backend.on_chord_apply(setid, body, interval, max_retries)
return set.apply_async(taskset_id=setid)
def with_ctypes_masked():
from kombu.utils import ctypes, gen_unique_id
self.assertIsNone(ctypes)
uuid = gen_unique_id()
self.assertTrue(uuid)
self.assertIsInstance(uuid, basestring)
def with_ctypes_masked():
from kombu.utils import ctypes, gen_unique_id
self.assertIsNone(ctypes)
uuid = gen_unique_id()
self.assertTrue(uuid)
self.assertIsInstance(uuid, basestring)
def test_basic_publish__get__consume__restore(self, n="test_basic_publish"):
c = memory_client().channel()
c.exchange_declare(n)
c.queue_declare(n)
c.queue_bind(n, n, n)
c.queue_declare(n + "2")
c.queue_bind(n + "2", n, n)
m = c.prepare_message("nthex quick brown fox...")
c.basic_publish(m, n, n)
r1 = c.message_to_python(c.basic_get(n))
self.assertTrue(r1)
self.assertEqual(r1.body, "nthex quick brown fox...".encode("utf-8"))
self.assertIsNone(c.basic_get(n))
consumer_tag = gen_unique_id()
c.basic_consume(n + "2", False, consumer_tag=consumer_tag, callback=lambda *a: None)
self.assertIn(n + "2", c._active_queues)
r2, _ = c.drain_events()
r2 = c.message_to_python(r2)
self.assertEqual(r2.body, "nthex quick brown fox...".encode("utf-8"))
self.assertEqual(r2.delivery_info["exchange"], n)
self.assertEqual(r2.delivery_info["routing_key"], n)
self.assertRaises(virtual.Empty, c.drain_events)
c.basic_cancel(consumer_tag)
c._restore(r2)
r3 = c.message_to_python(c.basic_get(n))
self.assertTrue(r3)
self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8"))
self.assertIsNone(c.basic_get(n))
def _broadcast(self, command, arguments=None, destination=None,
reply=False, timeout=1, limit=None, callback=None, channel=None):
arguments = arguments or {}
reply_ticket = reply and gen_unique_id() or None
if destination is not None and \
not isinstance(destination, (list, tuple)):
raise ValueError("destination must be a list/tuple not %s" % (
type(destination)))
# Set reply limit to number of destinations (if specificed)
if limit is None and destination:
limit = destination and len(destination) or None
chan = channel or self.connection.channel()
try:
if reply_ticket:
self.get_reply_queue(reply_ticket)(chan).declare()
self._publish(command, arguments, destination=destination,
reply_ticket=reply_ticket,
channel=chan)
if reply_ticket:
return self._collect(reply_ticket, limit=limit,
timeout=timeout,
callback=callback,
channel=chan)
finally:
channel or chan.close()
def test_serializable(self):
c = client().channel()
data = c.prepare_message("the quick brown fox...")
tag = data["properties"]["delivery_tag"] = gen_unique_id()
message = c.message_to_python(data)
dict_ = message.serializable()
self.assertEqual(dict_["body"], "the quick brown fox...")
self.assertEqual(dict_["properties"]["delivery_tag"], tag)
def test_error_mail_sender(self):
x = ErrorMail.subject % {
"name": "task_name",
"id": gen_unique_id(),
"exc": "FOOBARBAZ",
"hostname": "lana"
}
self.assertTrue(x)
def test_serializable(self):
c = client().channel()
data = c.prepare_message("the quick brown fox...")
tag = data["properties"]["delivery_tag"] = gen_unique_id()
message = c.message_to_python(data)
dict_ = message.serializable()
self.assertEqual(dict_["body"],
"the quick brown fox...".encode("utf-8"))
self.assertEqual(dict_["properties"]["delivery_tag"], tag)
def test_create(self):
c = client().channel()
data = c.prepare_message("the quick brown fox...")
tag = data["properties"]["delivery_tag"] = gen_unique_id()
message = c.message_to_python(data)
self.assertIsInstance(message, virtual.Message)
self.assertIs(message, c.message_to_python(message))
self.assertEqual(message.body, "the quick brown fox...")
self.assertTrue(message.delivery_tag, tag)
def test_can_consume(self, stdout, stderr):
_restored = []
class RestoreChannel(virtual.Channel):
do_restore = True
def _restore(self, message):
_restored.append(message)
self.assertTrue(self.q.can_consume())
for i in range(self.q.prefetch_count - 1):
self.q.append(i, gen_unique_id())
self.assertTrue(self.q.can_consume())
self.q.append(i + 1, gen_unique_id())
self.assertFalse(self.q.can_consume())
tag1 = self.q._delivered.keys()[0]
self.q.ack(tag1)
self.assertTrue(self.q.can_consume())
tag2 = gen_unique_id()
self.q.append(i + 2, tag2)
self.assertFalse(self.q.can_consume())
self.q.reject(tag2)
self.assertTrue(self.q.can_consume())
self.q.channel = RestoreChannel(self.q.channel.connection)
tag3 = gen_unique_id()
self.q.append(i + 3, tag3)
self.q.reject(tag3, requeue=True)
self.q._flush()
self.q.restore_unacked_once()
self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1])
self.assertTrue(self.q._delivered.restored)
self.assertFalse(self.q._delivered)
self.q.restore_unacked_once()
self.q._delivered.restored = False
self.q.restore_unacked_once()
self.assertTrue(stderr.getvalue())
self.assertFalse(stdout.getvalue())
def test_can_consume(self, stdout, stderr):
_restored = []
class RestoreChannel(virtual.Channel):
do_restore = True
def _restore(self, message):
_restored.append(message)
self.assertTrue(self.q.can_consume())
for i in range(self.q.prefetch_count - 1):
self.q.append(i, gen_unique_id())
self.assertTrue(self.q.can_consume())
self.q.append(i + 1, gen_unique_id())
self.assertFalse(self.q.can_consume())
tag1 = self.q._delivered.keys()[0]
self.q.ack(tag1)
self.assertTrue(self.q.can_consume())
tag2 = gen_unique_id()
self.q.append(i + 2, tag2)
self.assertFalse(self.q.can_consume())
self.q.reject(tag2)
self.assertTrue(self.q.can_consume())
self.q.channel = RestoreChannel(self.q.channel.connection)
tag3 = gen_unique_id()
self.q.append(i + 3, tag3)
self.q.reject(tag3, requeue=True)
self.q._flush()
self.q.restore_unacked_once()
self.assertListEqual(_restored, [11, 9, 8, 7, 6, 5, 4, 3, 2, 1])
self.assertTrue(self.q._delivered.restored)
self.assertFalse(self.q._delivered)
self.q.restore_unacked_once()
self.q._delivered.restored = False
self.q.restore_unacked_once()
self.assertTrue(stderr.getvalue())
self.assertFalse(stdout.getvalue())
def create_binding(self, queue):
"""Get binding item for queue.
Creates the item if it doesn't exist.
"""
item = self.get_queue(queue)
if item:
return item, item["id"]
id = gen_unique_id()
return self.new_item(id), id
def test_create(self):
c = client().channel()
data = c.prepare_message("the quick brown fox...")
tag = data["properties"]["delivery_tag"] = gen_unique_id()
message = c.message_to_python(data)
self.assertIsInstance(message, virtual.Message)
self.assertIs(message, c.message_to_python(message))
self.assertEqual(message.body,
"the quick brown fox...".encode("utf-8"))
self.assertTrue(message.delivery_tag, tag)
def create_binding(self, queue):
"""Get binding item for queue.
Creates the item if it doesn't exist.
"""
item = self.get_queue(queue)
if item:
return item, item["id"]
id = gen_unique_id()
return self.new_item(id), id
def test_reply__collect(self):
mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
exchange = mailbox.reply_exchange.name
ticket = gen_unique_id()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
channel = self.connection.channel()
reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel)
self.assertEqual(reply, [{"foo": "bar"}])
self.assertTrue(_callback_called[0])
ticket = gen_unique_id()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{"biz": "boz"}])
def test_reply__collect(self):
mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
exchange = mailbox.reply_exchange.name
ticket = gen_unique_id()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
channel = self.connection.channel()
reply = mailbox._collect(ticket, limit=1, callback=callback,
channel=channel)
self.assertEqual(reply, [{"foo": "bar"}])
self.assertTrue(_callback_called[0])
ticket = gen_unique_id()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{"biz": "boz"}])
def __init__(self,
addrport='',
id=None,
loglevel=logging.INFO,
logfile=None,
without_httpd=False,
numc=2,
sup_interval=None,
ready_event=None,
colored=None,
**kwargs):
self.id = id or gen_unique_id()
if isinstance(addrport, basestring):
addr, _, port = addrport.partition(':')
addrport = (addr, int(port) if port else 8000)
self.addrport = addrport
self.connection = celery.broker_connection()
self.without_httpd = without_httpd
self.logfile = logfile
self.loglevel = loglevel
self.numc = numc
self.ready_event = ready_event
self.exit_request = Event()
self.colored = colored or term.colored(enabled=False)
self.httpd = None
gSup = find_symbol(self, self.intsup_cls)
if not self.without_httpd:
self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport),
signals.httpd_ready)
self.supervisor = gSup(
instantiate(self, self.supervisor_cls, sup_interval),
signals.supervisor_ready)
self.controllers = [
gSup(
instantiate(self,
self.controller_cls,
id='%s.%s' % (self.id, i),
connection=self.connection,
branch=self), signals.controller_ready)
for i in xrange(1, numc + 1)
]
c = [self.supervisor] + self.controllers + [self.httpd]
c = self.components = list(filter(None, c))
self._components_ready = dict(
zip([z.thread for z in c], [False] * len(c)))
for controller in self.controllers:
if hasattr(controller.thread, 'presence'):
self._components_ready[controller.thread.presence] = False
self._components_shutdown = dict(self._components_ready)
super(Branch, self).__init__()
def __init__(self, addrport='', id=None, loglevel=logging.INFO,
logfile=None, without_httpd=False, numc=2, sup_interval=None,
ready_event=None, colored=None, **kwargs):
self.id = id or gen_unique_id()
if isinstance(addrport, basestring):
addr, _, port = addrport.partition(':')
addrport = (addr, int(port) if port else 8000)
self.addrport = addrport
self.connection = celery.broker_connection()
self.without_httpd = without_httpd
self.logfile = logfile
self.loglevel = loglevel
self.numc = numc
self.ready_event = ready_event
self.exit_request = Event()
self.colored = colored or term.colored(enabled=False)
self.httpd = None
gSup = find_symbol(self, self.intsup_cls)
if not self.without_httpd:
self.httpd = MockSup(instantiate(self, self.httpd_cls, addrport),
signals.httpd_ready)
self.supervisor = gSup(instantiate(self, self.supervisor_cls,
sup_interval), signals.supervisor_ready)
self.controllers = [gSup(instantiate(self, self.controller_cls,
id='%s.%s' % (self.id, i),
connection=self.connection,
branch=self),
signals.controller_ready)
for i in xrange(1, numc + 1)]
c = [self.supervisor] + self.controllers + [self.httpd]
c = self.components = list(filter(None, c))
self._components_ready = dict(zip([z.thread for z in c],
[False] * len(c)))
for controller in self.controllers:
if hasattr(controller.thread, 'presence'):
self._components_ready[controller.thread.presence] = False
self._components_shutdown = dict(self._components_ready)
super(Branch, self).__init__()
def __init__(self,
server_id = None,
amqp_host='localhost',
amqp_user ='******',
amqp_password='******',
amqp_vhost='/',
amqp_port=5672,
ssl=False,
timeout=0):
self.logger = logging.getLogger('callme.proxy')
self.timeout = 0
self.is_received = False
self.connection = BrokerConnection(hostname=amqp_host,
userid=amqp_user,
password=amqp_password,
virtual_host=amqp_vhost,
port=amqp_port,
ssl=ssl)
self.channel = self.connection.channel()
self.timeout = timeout
my_uuid = gen_unique_id()
self.reply_id = "client_"+amqp_user+"_ex_" + my_uuid
self.logger.debug("Queue ID: %s" %self.reply_id)
src_exchange = Exchange(self.reply_id, "direct", durable=False
,auto_delete=True)
src_queue = Queue("client_"+amqp_user+"_queue_"+my_uuid, exchange=src_exchange,
auto_delete=True,
durable=False)
# must declare in advance so reply message isn't
# published before.
src_queue(self.channel).declare()
consumer = Consumer(channel=self.channel, queues=src_queue, callbacks=[self._on_response])
consumer.consume()
def test_basic_publish__get__consume__restore(self,
n="test_basic_publish"):
c = memory_client().channel()
c.exchange_declare(n)
c.queue_declare(n)
c.queue_bind(n, n, n)
c.queue_declare(n + "2")
c.queue_bind(n + "2", n, n)
m = c.prepare_message("nthex quick brown fox...")
c.basic_publish(m, n, n)
r1 = c.message_to_python(c.basic_get(n))
self.assertTrue(r1)
self.assertEqual(r1.body,
"nthex quick brown fox...".encode("utf-8"))
self.assertIsNone(c.basic_get(n))
consumer_tag = gen_unique_id()
c.basic_consume(n + "2", False, consumer_tag=consumer_tag,
callback=lambda *a: None)
self.assertIn(n + "2", c._active_queues)
r2, _ = c.drain_events()
r2 = c.message_to_python(r2)
self.assertEqual(r2.body,
"nthex quick brown fox...".encode("utf-8"))
self.assertEqual(r2.delivery_info["exchange"], n)
self.assertEqual(r2.delivery_info["routing_key"], n)
self.assertRaises(virtual.Empty, c.drain_events)
c.basic_cancel(consumer_tag)
c._restore(r2)
r3 = c.message_to_python(c.basic_get(n))
self.assertTrue(r3)
self.assertEqual(r3.body, "nthex quick brown fox...".encode("utf-8"))
self.assertIsNone(c.basic_get(n))
def test_gen_unique_id(self):
i1 = utils.gen_unique_id()
i2 = utils.gen_unique_id()
self.assertIsInstance(i1, str)
self.assertNotEqual(i1, i2)
def __call__(self, body, **options):
uuid = body.options.setdefault("task_id", gen_unique_id())
self.Chord.apply_async((list(self.tasks), body), self.options,
**options)
return body.type.app.AsyncResult(uuid)
def test_error_mail_sender(self):
x = ErrorMail.subject % {"name": "task_name",
"id": gen_unique_id(),
"exc": "FOOBARBAZ",
"hostname": "lana"}
self.assertTrue(x)
def test_gen_unique_id(self):
i1 = utils.gen_unique_id()
i2 = utils.gen_unique_id()
self.assertIsInstance(i1, str)
self.assertNotEqual(i1, i2)
def create_queue(self, field_value):
return Queue(gen_unique_id(), self.exchange, field_value,
auto_delete=self.auto_delete)
def __call__(self, body, **options):
uuid = body.options.setdefault("task_id", gen_unique_id())
self.Chord.apply_async((list(self.tasks), body), self.options,
**options)
return body.type.app.AsyncResult(uuid)
def create_queue(self, field_value):
return Queue(gen_unique_id(),
self.exchange,
field_value,
auto_delete=self.auto_delete)