def test_declare(self):
chan = get_conn().channel()
b = Queue("foo", self.exchange, "foo", channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn("exchange_declare", chan)
self.assertIn("queue_declare", chan)
self.assertIn("queue_bind", chan)
def test_declare(self):
chan = Channel()
b = Queue("foo", self.exchange, "foo", channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn("exchange_declare", chan)
self.assertIn("queue_declare", chan)
self.assertIn("queue_bind", chan)
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn('exchange_declare', chan)
self.assertIn('queue_declare', chan)
self.assertIn('queue_bind', chan)
def test_declare(self):
chan = get_conn().channel()
b = Queue('foo', self.exchange, 'foo', channel=chan)
self.assertTrue(b.is_bound)
b.declare()
self.assertIn('exchange_declare', chan)
self.assertIn('queue_declare', chan)
self.assertIn('queue_bind', chan)
def test_declare_but_no_exchange(self):
q = Queue("a")
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
q.queue_bind.assert_called_with(False)
def test_declare_but_no_exchange(self):
q = Queue('a')
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
q.queue_bind.assert_called_with(False)
class MessConsumer(object):
def __init__(self, mess, connection, name, exchange_name):
self._mess = mess
self._conn = connection
self._name = name
self._exchange = Exchange(name=exchange_name, type='topic',
durable=False, auto_delete=True) #TODO parameterize
self._channel = None
self._ops = {}
self.connect()
def connect(self):
self._channel = self._conn.channel()
self._queue = Queue(channel=self._channel, name=self._name,
exchange=self._exchange, routing_key=self._name)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def consume(self):
while True:
self._conn.drain_events()
def _callback(self, body, message):
reply_to = message.headers.get('reply-to')
#TODO error handling for message format
op = body['op']
args = body['args']
kwargs = body['kwargs']
#TODO error handling for unknown op
op_fun = self._ops[op]
ret, err = None, None
try:
ret = op_fun(*args, **kwargs)
except Exception:
err = sys.exc_info()
finally:
if reply_to:
if err:
tb = traceback.format_exception(*err)
err = (err[0].__name__, str(err[1]), tb)
reply = dict(result=ret, error=err)
self._mess.reply(reply_to, reply)
message.ack()
def add_op(self, name, fun):
self._ops[name] = fun
def call(self, name, op, *args, **kwargs):
# create a direct exchange and queue for the reply
# TODO probably better to pool these or something?
msg_id = uuid.uuid4().hex
exchange = Exchange(name=msg_id, type='direct',
durable=False, auto_delete=True) #TODO parameterize
# check out a connection from the pool
with connections[self._conn].acquire(block=True) as conn:
channel = conn.channel()
queue = Queue(channel=channel, name=msg_id, exchange=exchange,
routing_key=msg_id, exclusive=True, durable=False,
auto_delete=True)
queue.declare()
# I think this can be done without gevent directly, but need to
# learn more about kombu first
messages = []
def _callback(body, message):
messages.append(body)
message.ack()
consumer = Consumer(channel=channel, queues=[queue],
callbacks=[_callback])
d = dict(op=op, args=args, kwargs=kwargs)
headers = {'reply-to' : msg_id}
with producers[self._conn].acquire(block=True) as producer:
producer.publish(d, routing_key=name, headers=headers)
with consumer:
# only expecting one event
conn.drain_events()
msg_body = messages[0]
if msg_body.get('error'):
raise Exception(*msg_body['error'])
else:
return msg_body.get('result')
class DashiConsumer(object):
def __init__(self, dashi, connection, name, exchange, sysname=None):
self._dashi = dashi
self._conn = connection
self._name = name
self._exchange = exchange
self._sysname = sysname
self._channel = None
self._ops = {}
self._cancelled = False
self._consumer_lock = threading.Lock()
self._last_heartbeat_check = datetime.min
self.connect()
def connect(self):
self._channel = self._conn.channel()
if self._sysname is not None:
name = "%s.%s" % (self._sysname, self._name)
else:
name = self._name
self._queue = Queue(channel=self._channel, name=name,
exchange=self._exchange, routing_key=name,
durable=self._dashi.durable,
auto_delete=self._dashi.auto_delete)
self._queue.declare()
self._consumer = Consumer(self._channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def disconnect(self):
self._consumer.cancel()
self._channel.close()
self._conn.release()
def consume(self, count=None, timeout=None):
# hold a lock for the duration of the consuming. this prevents
# multiple consumers and allows cancel to detect when consuming
# has ended.
if not self._consumer_lock.acquire(False):
raise Exception("only one consumer thread may run concurrently")
try:
if count:
i = 0
while i < count and not self._cancelled:
self._consume_one(timeout)
i += 1
else:
while not self._cancelled:
self._consume_one(timeout)
finally:
self._consumer_lock.release()
self._cancelled = False
def _consume_one(self, timeout=None):
# do consuming in a busy-ish loop, checking for cancel. There doesn't
# seem to be an easy way to interrupt drain_events other than the
# timeout. This could probably be added to kombu if needed. In
# practice cancellation is likely infrequent (except in tests) so this
# should hold for now. Can use a long timeout for production and a
# short one for tests.
inner_timeout = self._dashi.consumer_timeout
elapsed = 0
# keep trying until a single event is drained or timeout hit
while not self._cancelled:
self.heartbeat()
try:
self._conn.drain_events(timeout=inner_timeout)
break
except socket.timeout:
if timeout:
elapsed += inner_timeout
if elapsed >= timeout:
raise
if elapsed + inner_timeout > timeout:
inner_timeout = timeout - elapsed
def heartbeat(self):
if self._dashi._heartbeat_interval is None:
return
time_between_tics = timedelta(seconds=self._dashi._heartbeat_interval / 2)
if self._dashi.consumer_timeout > time_between_tics.seconds:
msg = "dashi consumer timeout (%s) must be half or smaller than the heartbeat interval %s" % (
self._dashi.consumer_timeout, self._dashi._heartbeat_interval)
raise DashiError(msg)
if datetime.now() - self._last_heartbeat_check > time_between_tics:
self._last_heartbeat_check = datetime.now()
self._conn.heartbeat_check()
def cancel(self, block=True):
self._cancelled = True
if block:
# acquire the lock and release it immediately
with self._consumer_lock:
pass
def _callback(self, body, message):
reply_to = None
ret = None
err = None
try:
reply_to = message.headers.get('reply-to')
try:
op = str(body['op'])
args = body.get('args')
except Exception, e:
log.warn("Failed to interpret message body: %s", body,
exc_info=True)
raise BadRequestError("Invalid request: %s" % str(e))
op_spec = self._ops.get(op)
if not op_spec:
raise UnknownOperationError("Unknown operation: " + op)
op_fun = op_spec.function
# stick the sender into kwargs if handler requested it
if op_spec.sender_kwarg:
sender = message.headers.get('sender')
args[op_spec.sender_kwarg] = sender
try:
ret = op_fun(**args)
except TypeError, e:
log.exception("Type error with handler for %s:%s", self._name, op)
raise BadRequestError("Type error: %s" % str(e))
except Exception:
log.exception("Error in handler for %s:%s", self._name, op)
raise
from kombu.entity import Exchange, Queue
from kombu.messaging import Producer
from kombu.connection import Connection
with Connection(
'amqp://*****:*****@10.46.0.39:5672/rapid_alpha_dev') as connection:
with connection.channel() as channel:
for i in range(1, 10):
science_news = Queue(
name='kombu_queue',
exchange=Exchange('kombu_queue', type='direct'),
routing_key='kombu_queue',
channel=channel,
durable=False,
)
science_news.declare()
producer = Producer(channel,
serializer='json',
routing_key='kombu_queue')
producer.publish({'name': 'kombu_queue', 'size': i})
science_news = Queue(
name='kombu_queue_1',
exchange=Exchange('kombu_queue_1', type='direct'),
routing_key='kombu_queue_1',
channel=channel,
max_priority=10, # 优先级
durable=False,
)
science_news.declare()
producer = Producer(channel,
class DashiConsumer(object):
def __init__(self, dashi, connection, name, exchange, sysname=None):
self._dashi = dashi
self._conn = connection
self._name = name
self._exchange = exchange
self._sysname = sysname
self._ops = {}
self._cancelled = threading.Event()
self._consumer_lock = threading.Lock()
if self._sysname is not None:
self._queue_name = "%s.%s" % (self._sysname, self._name)
else:
self._queue_name = self._name
self._queue_kwargs = dict(
name=self._queue_name,
exchange=self._exchange,
routing_key=self._queue_name,
durable=self._dashi.durable,
auto_delete=self._dashi.auto_delete,
queue_arguments={'x-expires': int(DEFAULT_QUEUE_EXPIRATION * 1000)})
self.connect()
def connect(self):
self._dashi.ensure(self._conn, self._connect)
def _connect(self, channel):
self._queue = Queue(channel=channel, **self._queue_kwargs)
self._queue.declare()
self._consumer = Consumer(channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def disconnect(self):
self._consumer.cancel()
self._conn.release()
def consume(self, count=None, timeout=None):
# hold a lock for the duration of the consuming. this prevents
# multiple consumers and allows cancel to detect when consuming
# has ended.
if not self._consumer_lock.acquire(False):
raise Exception("only one consumer thread may run concurrently")
try:
self._dashi._consume(self._conn, self._consumer, count=count,
timeout=timeout, until_event=self._cancelled)
finally:
self._consumer_lock.release()
self._cancelled.clear()
def cancel(self, block=True):
self._cancelled.set()
if block:
# acquire the lock and release it immediately
with self._consumer_lock:
pass
def _callback(self, body, message):
reply_to = None
ret = None
err = None
err_dict = None
try:
reply_to = message.headers.get('reply-to')
try:
op = str(body['op'])
args = body.get('args')
except Exception, e:
log.warn("Failed to interpret message body: %s", body,
exc_info=True)
raise BadRequestError("Invalid request: %s" % str(e))
op_spec = self._ops.get(op)
if not op_spec:
raise UnknownOperationError("Unknown operation: " + op)
op_fun = op_spec.function
# stick the sender into kwargs if handler requested it
if op_spec.sender_kwarg:
sender = message.headers.get('sender')
args[op_spec.sender_kwarg] = sender
try:
ret = op_fun(**args)
except TypeError, e:
log.exception("Type error with handler for %s:%s", self._name, op)
raise BadRequestError("Type error: %s" % str(e))
except Exception:
raise
