def test_declare(self): chan = get_conn().channel() b = Queue("foo", self.exchange, "foo", channel=chan) self.assertTrue(b.is_bound) b.declare() self.assertIn("exchange_declare", chan) self.assertIn("queue_declare", chan) self.assertIn("queue_bind", chan)
def test_declare(self): chan = Channel() b = Queue("foo", self.exchange, "foo", channel=chan) self.assertTrue(b.is_bound) b.declare() self.assertIn("exchange_declare", chan) self.assertIn("queue_declare", chan) self.assertIn("queue_bind", chan)
def test_declare(self): chan = get_conn().channel() b = Queue('foo', self.exchange, 'foo', channel=chan) self.assertTrue(b.is_bound) b.declare() self.assertIn('exchange_declare', chan) self.assertIn('queue_declare', chan) self.assertIn('queue_bind', chan)
def test_declare_but_no_exchange(self): q = Queue("a") q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() q.queue_declare.assert_called_with(False, passive=False) q.queue_bind.assert_called_with(False)
def test_declare_but_no_exchange(self): q = Queue('a') q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() q.queue_declare.assert_called_with(False, passive=False) q.queue_bind.assert_called_with(False)
class MessConsumer(object): def __init__(self, mess, connection, name, exchange_name): self._mess = mess self._conn = connection self._name = name self._exchange = Exchange(name=exchange_name, type='topic', durable=False, auto_delete=True) #TODO parameterize self._channel = None self._ops = {} self.connect() def connect(self): self._channel = self._conn.channel() self._queue = Queue(channel=self._channel, name=self._name, exchange=self._exchange, routing_key=self._name) self._queue.declare() self._consumer = Consumer(self._channel, [self._queue], callbacks=[self._callback]) self._consumer.consume() def consume(self): while True: self._conn.drain_events() def _callback(self, body, message): reply_to = message.headers.get('reply-to') #TODO error handling for message format op = body['op'] args = body['args'] kwargs = body['kwargs'] #TODO error handling for unknown op op_fun = self._ops[op] ret, err = None, None try: ret = op_fun(*args, **kwargs) except Exception: err = sys.exc_info() finally: if reply_to: if err: tb = traceback.format_exception(*err) err = (err[0].__name__, str(err[1]), tb) reply = dict(result=ret, error=err) self._mess.reply(reply_to, reply) message.ack() def add_op(self, name, fun): self._ops[name] = fun
def call(self, name, op, *args, **kwargs): # create a direct exchange and queue for the reply # TODO probably better to pool these or something? msg_id = uuid.uuid4().hex exchange = Exchange(name=msg_id, type='direct', durable=False, auto_delete=True) #TODO parameterize # check out a connection from the pool with connections[self._conn].acquire(block=True) as conn: channel = conn.channel() queue = Queue(channel=channel, name=msg_id, exchange=exchange, routing_key=msg_id, exclusive=True, durable=False, auto_delete=True) queue.declare() # I think this can be done without gevent directly, but need to # learn more about kombu first messages = [] def _callback(body, message): messages.append(body) message.ack() consumer = Consumer(channel=channel, queues=[queue], callbacks=[_callback]) d = dict(op=op, args=args, kwargs=kwargs) headers = {'reply-to' : msg_id} with producers[self._conn].acquire(block=True) as producer: producer.publish(d, routing_key=name, headers=headers) with consumer: # only expecting one event conn.drain_events() msg_body = messages[0] if msg_body.get('error'): raise Exception(*msg_body['error']) else: return msg_body.get('result')
class DashiConsumer(object): def __init__(self, dashi, connection, name, exchange, sysname=None): self._dashi = dashi self._conn = connection self._name = name self._exchange = exchange self._sysname = sysname self._channel = None self._ops = {} self._cancelled = False self._consumer_lock = threading.Lock() self._last_heartbeat_check = datetime.min self.connect() def connect(self): self._channel = self._conn.channel() if self._sysname is not None: name = "%s.%s" % (self._sysname, self._name) else: name = self._name self._queue = Queue(channel=self._channel, name=name, exchange=self._exchange, routing_key=name, durable=self._dashi.durable, auto_delete=self._dashi.auto_delete) self._queue.declare() self._consumer = Consumer(self._channel, [self._queue], callbacks=[self._callback]) self._consumer.consume() def disconnect(self): self._consumer.cancel() self._channel.close() self._conn.release() def consume(self, count=None, timeout=None): # hold a lock for the duration of the consuming. this prevents # multiple consumers and allows cancel to detect when consuming # has ended. if not self._consumer_lock.acquire(False): raise Exception("only one consumer thread may run concurrently") try: if count: i = 0 while i < count and not self._cancelled: self._consume_one(timeout) i += 1 else: while not self._cancelled: self._consume_one(timeout) finally: self._consumer_lock.release() self._cancelled = False def _consume_one(self, timeout=None): # do consuming in a busy-ish loop, checking for cancel. There doesn't # seem to be an easy way to interrupt drain_events other than the # timeout. This could probably be added to kombu if needed. In # practice cancellation is likely infrequent (except in tests) so this # should hold for now. Can use a long timeout for production and a # short one for tests. inner_timeout = self._dashi.consumer_timeout elapsed = 0 # keep trying until a single event is drained or timeout hit while not self._cancelled: self.heartbeat() try: self._conn.drain_events(timeout=inner_timeout) break except socket.timeout: if timeout: elapsed += inner_timeout if elapsed >= timeout: raise if elapsed + inner_timeout > timeout: inner_timeout = timeout - elapsed def heartbeat(self): if self._dashi._heartbeat_interval is None: return time_between_tics = timedelta(seconds=self._dashi._heartbeat_interval / 2) if self._dashi.consumer_timeout > time_between_tics.seconds: msg = "dashi consumer timeout (%s) must be half or smaller than the heartbeat interval %s" % ( self._dashi.consumer_timeout, self._dashi._heartbeat_interval) raise DashiError(msg) if datetime.now() - self._last_heartbeat_check > time_between_tics: self._last_heartbeat_check = datetime.now() self._conn.heartbeat_check() def cancel(self, block=True): self._cancelled = True if block: # acquire the lock and release it immediately with self._consumer_lock: pass def _callback(self, body, message): reply_to = None ret = None err = None try: reply_to = message.headers.get('reply-to') try: op = str(body['op']) args = body.get('args') except Exception, e: log.warn("Failed to interpret message body: %s", body, exc_info=True) raise BadRequestError("Invalid request: %s" % str(e)) op_spec = self._ops.get(op) if not op_spec: raise UnknownOperationError("Unknown operation: " + op) op_fun = op_spec.function # stick the sender into kwargs if handler requested it if op_spec.sender_kwarg: sender = message.headers.get('sender') args[op_spec.sender_kwarg] = sender try: ret = op_fun(**args) except TypeError, e: log.exception("Type error with handler for %s:%s", self._name, op) raise BadRequestError("Type error: %s" % str(e)) except Exception: log.exception("Error in handler for %s:%s", self._name, op) raise
from kombu.entity import Exchange, Queue from kombu.messaging import Producer from kombu.connection import Connection with Connection( 'amqp://*****:*****@10.46.0.39:5672/rapid_alpha_dev') as connection: with connection.channel() as channel: for i in range(1, 10): science_news = Queue( name='kombu_queue', exchange=Exchange('kombu_queue', type='direct'), routing_key='kombu_queue', channel=channel, durable=False, ) science_news.declare() producer = Producer(channel, serializer='json', routing_key='kombu_queue') producer.publish({'name': 'kombu_queue', 'size': i}) science_news = Queue( name='kombu_queue_1', exchange=Exchange('kombu_queue_1', type='direct'), routing_key='kombu_queue_1', channel=channel, max_priority=10, # 优先级 durable=False, ) science_news.declare() producer = Producer(channel,
class DashiConsumer(object): def __init__(self, dashi, connection, name, exchange, sysname=None): self._dashi = dashi self._conn = connection self._name = name self._exchange = exchange self._sysname = sysname self._ops = {} self._cancelled = threading.Event() self._consumer_lock = threading.Lock() if self._sysname is not None: self._queue_name = "%s.%s" % (self._sysname, self._name) else: self._queue_name = self._name self._queue_kwargs = dict( name=self._queue_name, exchange=self._exchange, routing_key=self._queue_name, durable=self._dashi.durable, auto_delete=self._dashi.auto_delete, queue_arguments={'x-expires': int(DEFAULT_QUEUE_EXPIRATION * 1000)}) self.connect() def connect(self): self._dashi.ensure(self._conn, self._connect) def _connect(self, channel): self._queue = Queue(channel=channel, **self._queue_kwargs) self._queue.declare() self._consumer = Consumer(channel, [self._queue], callbacks=[self._callback]) self._consumer.consume() def disconnect(self): self._consumer.cancel() self._conn.release() def consume(self, count=None, timeout=None): # hold a lock for the duration of the consuming. this prevents # multiple consumers and allows cancel to detect when consuming # has ended. if not self._consumer_lock.acquire(False): raise Exception("only one consumer thread may run concurrently") try: self._dashi._consume(self._conn, self._consumer, count=count, timeout=timeout, until_event=self._cancelled) finally: self._consumer_lock.release() self._cancelled.clear() def cancel(self, block=True): self._cancelled.set() if block: # acquire the lock and release it immediately with self._consumer_lock: pass def _callback(self, body, message): reply_to = None ret = None err = None err_dict = None try: reply_to = message.headers.get('reply-to') try: op = str(body['op']) args = body.get('args') except Exception, e: log.warn("Failed to interpret message body: %s", body, exc_info=True) raise BadRequestError("Invalid request: %s" % str(e)) op_spec = self._ops.get(op) if not op_spec: raise UnknownOperationError("Unknown operation: " + op) op_fun = op_spec.function # stick the sender into kwargs if handler requested it if op_spec.sender_kwarg: sender = message.headers.get('sender') args[op_spec.sender_kwarg] = sender try: ret = op_fun(**args) except TypeError, e: log.exception("Type error with handler for %s:%s", self._name, op) raise BadRequestError("Type error: %s" % str(e)) except Exception: raise