def test_eq(self):
q1 = Queue('xxx', Exchange('xxx', 'direct'), 'xxx')
q2 = Queue('xxx', Exchange('xxx', 'direct'), 'xxx')
self.assertEqual(q1, q2)
self.assertFalse(q1.__eq__(True))
q3 = Queue('yyy', Exchange('xxx', 'direct'), 'xxx')
self.assertNotEqual(q1, q3)
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn('exchange_declare', chan)
self.assertIn('queue_declare', chan)
self.assertIn('queue_bind', chan)
def test_declare(self):
chan = Channel()
b = Queue("foo", self.exchange, "foo", channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn("exchange_declare", chan)
self.assertIn("queue_declare", chan)
self.assertIn("queue_bind", chan)
def test_eq(self):
q1 = Queue("xxx", Exchange("xxx", "direct"), "xxx")
q2 = Queue("xxx", Exchange("xxx", "direct"), "xxx")
self.assertEqual(q1, q2)
self.assertFalse(q1.__eq__(True))
q3 = Queue("yyy", Exchange("xxx", "direct"), "xxx")
self.assertNotEqual(q1, q3)
class MessConsumer(object):
def __init__(self, mess, connection, name, exchange_name):
self._mess = mess
self._conn = connection
self._name = name
self._exchange = Exchange(name=exchange_name, type='topic',
durable=False, auto_delete=True) #TODO parameterize
self._channel = None
self._ops = {}
self.connect()
def connect(self):
self._channel = self._conn.channel()
self._queue = Queue(channel=self._channel, name=self._name,
exchange=self._exchange, routing_key=self._name)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def consume(self):
while True:
self._conn.drain_events()
def _callback(self, body, message):
reply_to = message.headers.get('reply-to')
#TODO error handling for message format
op = body['op']
args = body['args']
kwargs = body['kwargs']
#TODO error handling for unknown op
op_fun = self._ops[op]
ret, err = None, None
try:
ret = op_fun(*args, **kwargs)
except Exception:
err = sys.exc_info()
finally:
if reply_to:
if err:
tb = traceback.format_exception(*err)
err = (err[0].__name__, str(err[1]), tb)
reply = dict(result=ret, error=err)
self._mess.reply(reply_to, reply)
message.ack()
def add_op(self, name, fun):
self._ops[name] = fun
def test_also_binds_exchange(self):
chan = Channel()
b = Queue("foo", self.exchange)
self.assertFalse(b.is_bound)
self.assertFalse(b.exchange.is_bound)
b = b.bind(chan)
self.assertTrue(b.is_bound)
self.assertTrue(b.exchange.is_bound)
self.assertIs(b.channel, b.exchange.channel)
self.assertIsNot(b.exchange, self.exchange)
def _connect(self, channel):
self._queue = Queue(channel=channel, **self._queue_kwargs)
self._queue.declare()
self._consumer = Consumer(channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def connect(self):
self._channel = self._conn.channel()
self._queue = Queue(channel=self._channel, name=self._name,
exchange=self._exchange, routing_key=self._name)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def call(self, name, op, *args, **kwargs):
# create a direct exchange and queue for the reply
# TODO probably better to pool these or something?
msg_id = uuid.uuid4().hex
exchange = Exchange(name=msg_id, type='direct',
durable=False, auto_delete=True) #TODO parameterize
# check out a connection from the pool
with connections[self._conn].acquire(block=True) as conn:
channel = conn.channel()
queue = Queue(channel=channel, name=msg_id, exchange=exchange,
routing_key=msg_id, exclusive=True, durable=False,
auto_delete=True)
queue.declare()
# I think this can be done without gevent directly, but need to
# learn more about kombu first
messages = []
def _callback(body, message):
messages.append(body)
message.ack()
consumer = Consumer(channel=channel, queues=[queue],
callbacks=[_callback])
d = dict(op=op, args=args, kwargs=kwargs)
headers = {'reply-to' : msg_id}
with producers[self._conn].acquire(block=True) as producer:
producer.publish(d, routing_key=name, headers=headers)
with consumer:
# only expecting one event
conn.drain_events()
msg_body = messages[0]
if msg_body.get('error'):
raise Exception(*msg_body['error'])
else:
return msg_body.get('result')
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def connect(self):
self._channel = self._conn.channel()
self._queue = Queue(channel=self._channel, name=self._name,
exchange=self._exchange, routing_key=self._name,
durable=self._dashi.durable,
auto_delete=self._dashi.auto_delete)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
def __init__(self, connection, handlers=None, routing_key="#",
node_id=None, app=None):
self.app = app_or_default(app)
self.connection = connection
if handlers is not None:
self.handlers = handlers
self.routing_key = routing_key
self.node_id = node_id or gen_unique_id()
self.queue = Queue("%s.%s" % ("celeryev", self.node_id),
exchange=event_exchange,
routing_key=self.routing_key,
auto_delete=True,
durable=False)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def worker_direct(hostname):
"""Returns :class:`kombu.Queue` that is a direct route to
a worker by hostname.
:param hostname: The fully qualified node name of a worker
(e.g. ``[email protected]``). If passed a
:class:`kombu.Queue` instance it will simply return
that instead.
"""
if isinstance(hostname, Queue):
return hostname
return Queue(WORKER_DIRECT_QUEUE_FORMAT.format(hostname=hostname),
WORKER_DIRECT_EXCHANGE,
hostname, auto_delete=True)
def getCabbage(self):
connectUri = CabbageHolder._getConnectUri()
work = CacheHolder.getCache().get(HOST_NAME, WORKS)
if work.queues and len(work.queues) > 0:
queues = []
for queueName in work.queues:
brokerQueue = StoreHolder.getStore().getQueue(queueName)
queues.append(
Queue(name=brokerQueue.queueName,
exchange=Exchange(brokerQueue.exchangeName),
routing_key=brokerQueue.routingKey))
celeryconfig.CELERY_QUEUES = tuple(queues)
return Cabbage(broker=str(connectUri))
def setUp(self):
try:
data_folder_in = tempfile.mkdtemp()
data_folder_out = tempfile.mkdtemp()
except Exception:
raise SkipTest('filesystem transport: cannot create tempfiles')
self.c = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_in,
'data_folder_out': data_folder_out,
})
self.p = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_out,
'data_folder_out': data_folder_in,
})
self.e = Exchange('test_transport_filesystem')
self.q = Queue('test_transport_filesystem',
exchange=self.e,
routing_key='test_transport_filesystem')
self.q2 = Queue('test_transport_filesystem2',
exchange=self.e,
routing_key='test_transport_filesystem2')
def __init__(self, connection, handlers=None, routing_key="#",
node_id=None, app=None, queue_prefix="celeryev"):
self.app = app_or_default(app)
self.connection = connection
if handlers is not None:
self.handlers = handlers
self.routing_key = routing_key
self.node_id = node_id or uuid()
self.queue_prefix = queue_prefix
self.queue = Queue('.'.join([self.queue_prefix, self.node_id]),
exchange=event_exchange,
routing_key=self.routing_key,
auto_delete=True,
durable=False)
def test_declare_but_no_exchange(self):
q = Queue('a')
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
q.queue_bind.assert_called_with(False)
def worker_direct(hostname):
"""Return the :class:`kombu.Queue` being a direct route to a worker.
Arguments:
hostname (str, ~kombu.Queue): The fully qualified node name of
a worker (e.g., ``[email protected]``). If passed a
:class:`kombu.Queue` instance it will simply return
that instead.
"""
if isinstance(hostname, Queue):
return hostname
return Queue(
WORKER_DIRECT_QUEUE_FORMAT.format(hostname=hostname),
WORKER_DIRECT_EXCHANGE,
hostname,
)
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({"foo": "bar"})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {"foo": "bar"})
self.assertIsInstance(exc, ValueError)
class Worker(ConsumerMixin):
task_queue = Queue('xadmin-notify', Exchange(name='xnotify', type='fanout'), routing_key='xnotify')
def __init__(self, app):
self._app = app
self.logger = app.logger
self.connection = celery_app.connection_for_read()
def get_consumers(self, Consumer, channel):
return [Consumer(queues=[self.task_queue],
callbacks=[self.on_event])]
def on_event(self, body, message):
# print('Got task: {0!r}'.format(body))
# type_, data = json.loads(body)
type_, data = body
if type_.startswith('job-'):
self._on_job(type_, data)
message.ack()
def _on_job(self, type_, data):
self.logger.info('Received: %s - %s', type_, data)
obj = Job.objects.filter(alias=data['job']).first()
# print('Job =>', obj)
if obj.last_batch_id != data['batch_id']:
# print('!!last_batch_id==', obj.last_batch_id)
if obj.type == 0:
return
elif obj.type == 1:
obj.last_batch_id = data['batch_id']
job = JobAction(obj.project.alias, obj.alias, obj.last_batch_id)
if type_ == 'job-finished': # job-finished
r = job.stop()
obj.status = obj.status & 0xf0 | STATUS_STOP
elif type_ == 'job-pause':
r = job.pause()
obj.status = obj.status & 0xf0 | STATUS_PAUSE
elif type_ == 'job-started':
if obj.type == 0:
return
elif obj.type == 1:
obj.status = obj.status & 0xf0 | STATUS_START
obj.save()
def stop(self):
self.should_stop = True
def connect(self):
self._channel = self._conn.channel()
if self._sysname is not None:
name = "%s.%s" % (self._sysname, self._name)
else:
name = self._name
self._queue = Queue(channel=self._channel, name=name,
exchange=self._exchange, routing_key=name,
durable=self._dashi.durable,
auto_delete=self._dashi.auto_delete)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u'foo': u'bar'})
self.assertIn('basic_ack', channel)
self.assertIn('message_to_python', channel)
self.assertEqual(received[0], {u'foo': u'bar'})
def testUrl(self):
client = KombuClient(
url=
"amqp://*****:*****@172.16.4.134:5672/cabbage_vhost"
)
conn = client.conn
conn.connect()
video_queue = Queue('hdfs',
exchange=Exchange("hdfs"),
routing_key='hdfs')
with conn.Consumer(video_queue,
callbacks=[process_media],
accept=['json', 'pickle', 'msgpack',
'yaml']) as consumer:
# Process messages and handle events on all channels
while True:
conn.drain_events()
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u"foo": u"bar"})
self.assertIn("basic_ack", channel)
self.assertIn("message_to_python", channel)
self.assertEqual(received[0], {u"foo": u"bar"})
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare", "queue_declare", "basic_consume"):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ("exchange_declare", "queue_declare", "queue_bind"):
self.assertIn(meth, channel)
self.assertNotIn("basic_consume", channel)
consumer.consume()
self.assertIn("basic_consume", channel)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare", "queue_declare", "queue_bind",
"basic_consume"):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count("basic_consume"), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def _initJobs(self, cabbage):
store = StoreHolder.getRetryStore()
jobs = store.getJobs()
work = store.getWork(HOST_NAME)
queues = work.queues
routes = {}
queues_celery = []
for que in queues:
que = store.getQueue(que)
queues_celery.append(
Queue(que.queueName,
Exchange(que.queueName),
routing_key=que.queueName,
queue_arguments={'x-max-priority': int(que.priority)}))
for job in jobs:
if job.status != JOB_DELETE and job.brokerQueue in queues:
#fixbug 动态扩容时，缓存JOB
if not CacheHolder.getCache().hasKey(job.jobId, JOBS):
CacheHolder.getCache().put(job.jobId, job, JOBS)
clientDir = ConfigHolder.getConfig().getProperty(
BASE, CLIENT_FILE_DIRECTORY)
path = clientDir + "/" + job.jobId
if not os.path.isdir(path):
# @FIX BUG 文件不同步
syncJob(job.jobId, store)
self.addScriptJobId(job.jobId, cabbage)
for taskName in job.tasks:
que = store.getQueue(job.brokerQueue)
routes[taskName] = {
'queue': que.queueName,
'routing_key': que.routingKey
}
log.info(routes)
celeryconfig.CELERY_QUEUES = tuple(queues_celery)
celeryconfig.CELERY_ROUTES = routes
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare',
'queue_declare',
'basic_consume'):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ('exchange_declare',
'queue_declare',
'queue_bind'):
self.assertIn(meth, channel)
self.assertNotIn('basic_consume', channel)
consumer.consume()
self.assertIn('basic_consume', channel)
def test_as_dict(self):
q = Queue("foo", self.exchange, "rk")
d = q.as_dict(recurse=True)
self.assertEqual(d["exchange"]["name"], self.exchange.name)
def test_unbind(self):
b = Queue("foo", self.exchange, "foo", channel=get_conn().channel())
b.unbind()
self.assertIn("queue_unbind", b.channel)
def test_delete(self):
b = Queue("foo", self.exchange, "foo", channel=get_conn().channel())
b.delete()
self.assertIn("queue_delete", b.channel)
def test_cancel(self):
b = Queue("foo", self.exchange, "foo", channel=get_conn().channel())
b.cancel("fifafo")
self.assertIn("basic_cancel", b.channel)
class DashiConsumer(object):
def __init__(self, dashi, connection, name, exchange, sysname=None):
self._dashi = dashi
self._conn = connection
self._name = name
self._exchange = exchange
self._sysname = sysname
self._channel = None
self._ops = {}
self._cancelled = False
self._consumer_lock = threading.Lock()
self._last_heartbeat_check = datetime.min
self.connect()
def connect(self):
self._channel = self._conn.channel()
if self._sysname is not None:
name = "%s.%s" % (self._sysname, self._name)
else:
name = self._name
self._queue = Queue(channel=self._channel, name=name,
exchange=self._exchange, routing_key=name,
durable=self._dashi.durable,
auto_delete=self._dashi.auto_delete)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def disconnect(self):
self._consumer.cancel()
self._channel.close()
self._conn.release()
def consume(self, count=None, timeout=None):
# hold a lock for the duration of the consuming. this prevents
# multiple consumers and allows cancel to detect when consuming
# has ended.
if not self._consumer_lock.acquire(False):
raise Exception("only one consumer thread may run concurrently")
try:
if count:
i = 0
while i < count and not self._cancelled:
self._consume_one(timeout)
i += 1
else:
while not self._cancelled:
self._consume_one(timeout)
finally:
self._consumer_lock.release()
self._cancelled = False
def _consume_one(self, timeout=None):
# do consuming in a busy-ish loop, checking for cancel. There doesn't
# seem to be an easy way to interrupt drain_events other than the
# timeout. This could probably be added to kombu if needed. In
# practice cancellation is likely infrequent (except in tests) so this
# should hold for now. Can use a long timeout for production and a
# short one for tests.
inner_timeout = self._dashi.consumer_timeout
elapsed = 0
# keep trying until a single event is drained or timeout hit
while not self._cancelled:
self.heartbeat()
try:
self._conn.drain_events(timeout=inner_timeout)
break
except socket.timeout:
if timeout:
elapsed += inner_timeout
if elapsed >= timeout:
raise
if elapsed + inner_timeout > timeout:
inner_timeout = timeout - elapsed
def heartbeat(self):
if self._dashi._heartbeat_interval is None:
return
time_between_tics = timedelta(seconds=self._dashi._heartbeat_interval / 2)
if self._dashi.consumer_timeout > time_between_tics.seconds:
msg = "dashi consumer timeout (%s) must be half or smaller than the heartbeat interval %s" % (
self._dashi.consumer_timeout, self._dashi._heartbeat_interval)
raise DashiError(msg)
if datetime.now() - self._last_heartbeat_check > time_between_tics:
self._last_heartbeat_check = datetime.now()
self._conn.heartbeat_check()
def cancel(self, block=True):
self._cancelled = True
if block:
# acquire the lock and release it immediately
with self._consumer_lock:
pass
def _callback(self, body, message):
reply_to = None
ret = None
err = None
try:
reply_to = message.headers.get('reply-to')
try:
op = str(body['op'])
args = body.get('args')
except Exception, e:
log.warn("Failed to interpret message body: %s", body,
exc_info=True)
raise BadRequestError("Invalid request: %s" % str(e))
op_spec = self._ops.get(op)
if not op_spec:
raise UnknownOperationError("Unknown operation: " + op)
op_fun = op_spec.function
# stick the sender into kwargs if handler requested it
if op_spec.sender_kwarg:
sender = message.headers.get('sender')
args[op_spec.sender_kwarg] = sender
try:
ret = op_fun(**args)
except TypeError, e:
log.exception("Type error with handler for %s:%s", self._name, op)
raise BadRequestError("Type error: %s" % str(e))
except Exception:
log.exception("Error in handler for %s:%s", self._name, op)
raise
def test_cancel(self):
b = Queue("foo", self.exchange, "foo", channel=Channel())
b.cancel("fifafo")
self.assertIn("basic_cancel", b.channel)
def test_purge(self):
b = Queue("foo", self.exchange, "foo", channel=Channel())
b.purge()
self.assertIn("queue_purge", b.channel)
def test_consume(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.consume('fifafo', None)
self.assertIn('basic_consume', b.channel)
def get_queue(self, hostname):
return Queue("%s.%s.pidbox" % (hostname, self.namespace),
exchange=self.exchange,
durable=False,
auto_delete=True)
def test__repr__(self):
b = Queue("foo", self.exchange, "foo")
self.assertIn("foo", repr(b))
self.assertIn("Queue", repr(b))
def test_as_dict(self):
q = Queue('foo', self.exchange, 'rk')
d = q.as_dict(recurse=True)
self.assertEqual(d['exchange']['name'], self.exchange.name)
def test_get(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.get()
self.assertIn('basic_get', b.channel)
def test_when_bound_but_no_exchange(self):
q = Queue('a')
q.exchange = None
self.assertIsNone(q.when_bound())
def get_reply_queue(self, ticket):
return Queue("%s.%s" % (ticket, self.reply_exchange.name),
exchange=self.reply_exchange,
routing_key=ticket,
durable=False,
auto_delete=True)
class DashiConsumer(object):
def __init__(self, dashi, connection, name, exchange, sysname=None):
self._dashi = dashi
self._conn = connection
self._name = name
self._exchange = exchange
self._sysname = sysname
self._ops = {}
self._cancelled = threading.Event()
self._consumer_lock = threading.Lock()
if self._sysname is not None:
self._queue_name = "%s.%s" % (self._sysname, self._name)
else:
self._queue_name = self._name
self._queue_kwargs = dict(
name=self._queue_name,
exchange=self._exchange,
routing_key=self._queue_name,
durable=self._dashi.durable,
auto_delete=self._dashi.auto_delete,
queue_arguments={'x-expires': int(DEFAULT_QUEUE_EXPIRATION * 1000)})
self.connect()
def connect(self):
self._dashi.ensure(self._conn, self._connect)
def _connect(self, channel):
self._queue = Queue(channel=channel, **self._queue_kwargs)
self._queue.declare()
self._consumer = Consumer(channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def disconnect(self):
self._consumer.cancel()
self._conn.release()
def consume(self, count=None, timeout=None):
# hold a lock for the duration of the consuming. this prevents
# multiple consumers and allows cancel to detect when consuming
# has ended.
if not self._consumer_lock.acquire(False):
raise Exception("only one consumer thread may run concurrently")
try:
self._dashi._consume(self._conn, self._consumer, count=count,
timeout=timeout, until_event=self._cancelled)
finally:
self._consumer_lock.release()
self._cancelled.clear()
def cancel(self, block=True):
self._cancelled.set()
if block:
# acquire the lock and release it immediately
with self._consumer_lock:
pass
def _callback(self, body, message):
reply_to = None
ret = None
err = None
err_dict = None
try:
reply_to = message.headers.get('reply-to')
try:
op = str(body['op'])
args = body.get('args')
except Exception, e:
log.warn("Failed to interpret message body: %s", body,
exc_info=True)
raise BadRequestError("Invalid request: %s" % str(e))
op_spec = self._ops.get(op)
if not op_spec:
raise UnknownOperationError("Unknown operation: " + op)
op_fun = op_spec.function
# stick the sender into kwargs if handler requested it
if op_spec.sender_kwarg:
sender = message.headers.get('sender')
args[op_spec.sender_kwarg] = sender
try:
ret = op_fun(**args)
except TypeError, e:
log.exception("Type error with handler for %s:%s", self._name, op)
raise BadRequestError("Type error: %s" % str(e))
except Exception:
raise
def call(self, name, operation, timeout=10, args=None, **kwargs):
"""Send a message and wait for reply
@param name: name of destination service queue
@param operation: name of service operation to invoke
@param timeout: RPC timeout to await a reply
@param args: dictionary of keyword args to pass to operation.
Use this OR kwargs.
@param kwargs: additional args to pass to operation
"""
if args:
if kwargs:
raise TypeError("specify args dict or keyword arguments, not both")
else:
args = kwargs
# create a direct queue for the reply. This may end up being a
# bottleneck for performance: each rpc call gets a brand new
# exclusive queue. However this approach is used nova.rpc and
# seems to have carried them pretty far. If/when this
# becomes a bottleneck we can set up a long-lived backend queue and
# use correlation_id to deal with concurrent RPC calls. See:
# http://www.rabbitmq.com/tutorials/tutorial-six-python.html
msg_id = uuid.uuid4().hex
# expire the reply queue shortly after the timeout. it will be
# (lazily) deleted by the broker if we don't clean it up first
queue_arguments = {'x-expires': int((timeout + 1) * 1000)}
queue = Queue(name=msg_id, exchange=self._exchange, routing_key=msg_id,
durable=False, queue_arguments=queue_arguments)
messages = []
event = threading.Event()
def _callback(body, message):
messages.append(body)
message.ack()
event.set()
d = dict(op=operation, args=args)
headers = {'reply-to': msg_id, 'sender': self.add_sysname(self.name)}
dest = self.add_sysname(name)
def _declare_and_send(channel):
consumer = Consumer(channel, (queue,), callbacks=(_callback,))
with Producer(channel) as producer:
producer.publish(d, routing_key=dest, headers=headers,
exchange=self._exchange, serializer=self._serializer)
return consumer
log.debug("sending call to %s:%s", dest, operation)
with connections[self._pool_conn].acquire(block=True) as conn:
consumer, channel = self.ensure(conn, _declare_and_send)
try:
self._consume(conn, consumer, timeout=timeout, until_event=event)
# try to delete queue, but don't worry if it fails (will expire)
try:
queue = queue.bind(channel)
queue.delete(nowait=True)
except Exception:
log.exception("error deleting queue")
finally:
conn.maybe_close_channel(channel)
msg_body = messages[0]
if msg_body.get('error'):
raise_error(msg_body['error'])
else:
return msg_body.get('result')
def test_unbind(self):
b = Queue("foo", self.exchange, "foo", channel=Channel())
b.unbind()
self.assertIn("queue_unbind", b.channel)
def test_get(self):
b = Queue("foo", self.exchange, "foo", channel=Channel())
b.get()
self.assertIn("basic_get", b.channel)
def test_unbind(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.unbind()
self.assertIn('queue_unbind', b.channel)
def test_consume(self):
b = Queue("foo", self.exchange, "foo", channel=Channel())
b.consume("fifafo", None)
self.assertIn("basic_consume", b.channel)
def test_consume(self):
b = Queue("foo", self.exchange, "foo", channel=get_conn().channel())
b.consume("fifafo", None)
self.assertIn("basic_consume", b.channel)
def test_delete(self):
b = Queue("foo", self.exchange, "foo", channel=Channel())
b.delete()
self.assertIn("queue_delete", b.channel)
def test_cancel(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.cancel('fifafo')
self.assertIn('basic_cancel', b.channel)
def test_get(self):
b = Queue("foo", self.exchange, "foo", channel=get_conn().channel())
b.get()
self.assertIn("basic_get", b.channel)
def test_delete(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.delete()
self.assertIn('queue_delete', b.channel)
CELERY_SEND_TASK_SENT_EVENT = True
CELERY_SEND_EVENTS = True
#CELERY_RESULT_BACKEND = 'amqp'
#CELERY_RESULT_EXCHANGE = 'celereyResults'
# CELERY_ACCEPT_CONTENT = ['json']
# CELERY_TASK_SERIALIZER = 'json'
# CELERY_RESULT_SERIALIZER = 'json'
# from kombu import serialization
# serialization.registry._decoders.pop("application/x-python-serialize")
#CELERY_IGNORE_RESULT = False # this is less important
BROKER_HEARTBEAT = 0
#CELERY_QUEUES = ( Queue('celery', Exchange('celery'), routing_key='celery'),)
CELERYD_MAX_TASKS_PER_CHILD = 40
CELERY_ROUTES = {}
CELERYD_LOG_FORMAT = "[%(asctime)s: %(levelname)s/" + HOST_NAME + ":" + LOCAL_IP + "/%(processName)s] %(message)s"
#print "[%(asctime)s: %(levelname)s/"+getHostName()+"/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s"
CELERYD_TASK_LOG_FORMAT = "[%(asctime)s: %(levelname)s/" + HOST_NAME + ":" + LOCAL_IP + "/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s"
#
CELERY_QUEUES = (Queue('celery', Exchange('celery'), routing_key='celery'), )
#CELERY_IMPORTS = ('cabbage.test.task.test_mac_task.TestMacTask',)
def test_as_dict(self):
q = Queue("foo", self.exchange, "rk")
d = q.as_dict(recurse=True)
self.assertEqual(d["exchange"]["name"], self.exchange.name)
