def test_call_queue_deleted(self): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test", "hats") receiver.consume_in_thread(1) conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) args1 = dict(a=1, b="sandwich") rpc_consumers = [] # patch in this fake consumer so we can save a copy class _Consumer(dashi.Consumer): def __init__(self, *args, **kwargs): super(_Consumer, self).__init__(*args, **kwargs) rpc_consumers.append(self) with patch.object(dashi, "Consumer", new=_Consumer): self.assertEqual(conn.call(receiver.name, "test", **args1), "hats") self.assertEqual(len(rpc_consumers), 1) self.assertEqual(len(rpc_consumers[0].queues), 1) queue = rpc_consumers[0].queues[0] self.assertRaises(NotFoundError, get_queue_info, conn, queue) receiver.join_consumer_thread() assert_kombu_pools_empty()
def test_pool_problems(self): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test1") receiver.consume_in_thread() conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) t = threading.Thread(target=self._thread_erroneous_replies, args=(conn, 100)) t.daemon = True t.start() try: for i in range(100): conn.fire(receiver.name, "test1", i=i) finally: t.join() pred = lambda received: len(received) == 100 receiver.wait(pred=pred) self.assertEqual(len(receiver.received), 100) receiver.cancel() receiver.join_consumer_thread() receiver.disconnect()
def test_cancel_resume_cancel(self): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test", 1) receiver.consume_in_thread() conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) self.assertEqual(1, conn.call(receiver.name, "test")) receiver.cancel() receiver.join_consumer_thread() receiver.clear() # send message while receiver is cancelled conn.fire(receiver.name, "test", hats=4) # start up consumer again. message should arrive. receiver.consume_in_thread() receiver.wait() self.assertEqual(receiver.received[-1], ("test", dict(hats=4))) receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_call_kill_pool_connection(self): # use a pool connection, kill the connection, and then try to reuse it # put receiver directly on rabbit. not via proxy receiver = TestReceiver(uri=self.real_uri, exchange="x1", transport_options=self.transport_options, retry=retry) replies = [5, 4, 3, 2, 1] receiver.handle("test", replies.pop) receiver.consume_in_thread() conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options, retry=retry) ret = conn.call(receiver.name, "test") self.assertEqual(ret, 1) for i in list(reversed(replies)): self.proxy.restart() ret = conn.call(receiver.name, "test") self.assertEqual(ret, i) receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_call_kill_before_reply(self): # have the receiver handler restart the sender's connection # while it is waiting for a reply def killit(): self.proxy.restart() return True # put receiver directly on rabbit. not via proxy receiver = TestReceiver(uri=self.real_uri, exchange="x1", transport_options=self.transport_options, retry=retry) receiver.handle("killme", killit) receiver.consume_in_thread() for _ in range(5): conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options, retry=retry) ret = conn.call(receiver.name, "killme") self.assertEqual(ret, True) receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_fire_many_receivers(self): extras = {} receivers = [] receiver_name = None for i in range(3): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options, **extras) if not receiver_name: receiver_name = receiver.name extras['name'] = receiver.name receiver.handle("test") receivers.append(receiver) conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) for i in range(10): conn.fire(receiver_name, "test", n=i) # walk the receivers and have each one consume a single message receiver_cycle = itertools.cycle(receivers) for i in range(10): receiver = next(receiver_cycle) receiver.consume(1) opname, args = receiver.received[-1] self.assertEqual(opname, "test") self.assertEqual(args['n'], i) for receiver in receivers: receiver.disconnect() assert_kombu_pools_empty()
def test_sysname(self): sysname = "sysname" receiver = TestReceiver(uri=self.uri, exchange="x1", sysname=sysname, transport_options=self.transport_options) replies = [5, 4, 3, 2, 1] receiver.handle("test", replies.pop) receiver.consume_in_thread(1) conn = dashi.DashiConnection("s1", self.uri, "x1", sysname=sysname, transport_options=self.transport_options) args1 = dict(a=1, b="sandwich") ret = conn.call(receiver.name, "test", **args1) self.assertEqual(ret, 1) receiver.join_consumer_thread() receiver.consume_in_thread(4) for i in list(reversed(replies)): ret = conn.call(receiver.name, "test", **args1) self.assertEqual(ret, i) receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_fire_kill_pool_connection(self): # use a pool connection, kill the connection, and then try to reuse it # put receiver directly on rabbit. not via proxy receiver = TestReceiver(uri=self.real_uri, exchange="x1", transport_options=self.transport_options, retry=retry) receiver.handle("test") receiver.consume_in_thread() conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options, retry=retry) conn.fire(receiver.name, "test", hats=0) receiver.wait(pred=lambda r: len(r) == 1) self.assertEqual(receiver.received[0], ("test", {"hats": 0})) for i in range(1, 4): self.proxy.restart() conn.fire(receiver.name, "test", hats=i) receiver.wait(pred=lambda r: len(r) == 4) self.assertEqual(receiver.received[1], ("test", {"hats": 1})) self.assertEqual(receiver.received[2], ("test", {"hats": 2})) self.assertEqual(receiver.received[3], ("test", {"hats": 3})) receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_receiver_kill_connection(self): def errback(): log.debug("Errback called", exc_info=True) # restart a consumer's connection. it should reconnect and keep consuming receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options, retry=retry, errback=errback) receiver.handle("test", "hats") receiver.consume_in_thread() # put caller directly on rabbit, not proxy conn = dashi.DashiConnection("s1", self.real_uri, "x1", transport_options=self.transport_options, retry=retry) self.assertEqual(conn.call(receiver.name, "test"), "hats") self.proxy.restart() log.debug("Queue consumer count: %s", get_queue_info(conn, receiver.queue)[2]) self.assertEqual(conn.call(receiver.name, "test", timeout=60), "hats") self.assertEqual(conn.call(receiver.name, "test"), "hats") receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_call_channel_free(self): # hackily ensure that call() releases its channel receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test", "myreply") receiver.consume_in_thread(1) conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) # peek into connection to grab a channel and note its id with connections[conn._conn].acquire(block=True) as kombuconn: with kombuconn.channel() as channel: channel_id = channel.channel_id log.debug("got channel ID %s", channel.channel_id) ret = conn.call(receiver.name, "test") self.assertEqual(ret, "myreply") receiver.join_consumer_thread() # peek into connection to grab a channel and note its id with connections[conn._conn].acquire(block=True) as kombuconn: with kombuconn.channel() as channel: log.debug("got channel ID %s", channel.channel_id) self.assertEqual(channel_id, channel.channel_id) receiver.disconnect() assert_kombu_pools_empty()
def test_exceptions(self): class CustomNotFoundError(Exception): pass receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.conn.link_exceptions(custom_exception=CustomNotFoundError, dashi_exception=dashi.exceptions.NotFoundError) receiver.handle("test_exception", raise_exception=CustomNotFoundError, sender_kwarg="sender") receiver.consume_in_thread(1) conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) args1 = dict(a=1, b="sandwich") try: conn.call(receiver.name, "test_exception", **args1) except dashi.exceptions.NotFoundError: pass else: self.fail("Expected NotFoundError") finally: receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_call_timeout(self): conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) countdown = dashi.util.Countdown(0.6) # call with no receiver should timeout self.assertRaises(conn.timeout_error, conn.call, "notarealname", "test", timeout=0.5) delta = countdown.delta_seconds assert 0 < delta < 1, "delta: %s" % delta
def test_dashi(self): import dashi dashi_conn = dashi.DashiConnection("something", self._haa_dashi_uri, self._haa_dashi_exchange) status = dashi_conn.call(self._haa_dashi_name, "status") assert status in ('PENDING', 'READY', 'STEADY') new_policy = {'preserve_n': 0} dashi_conn.call(self._haa_dashi_name, "reconfigure_policy", new_policy=new_policy)
def test_heartbeat_kill(self): # create a second tier proxy. then we can kill the backend proxy # and our connection remains "open" chained_proxy, chained_uri = self._make_chained_proxy(self.proxy) event = threading.Event() # attach an errback to the receiver that is called by dashi # with any connection failures def errback(): log.debug("Errback called", exc_info=True) exc = sys.exc_info()[1] if "Too many heartbeats missed" in str(exc): log.debug("we got the beat!") event.set() receiver = TestReceiver(uri=chained_uri, exchange="x1", transport_options=self.transport_options, heartbeat=2.0, errback=errback, retry=retry) receiver.handle("test", "hats") receiver.consume_in_thread() # put caller directly on rabbit, not proxy conn = dashi.DashiConnection("s1", self.real_uri, "x1", transport_options=self.transport_options, retry=retry) self.assertEqual(conn.call(receiver.name, "test"), "hats") # kill the proxy and wait for the errback from amqp client self.proxy.stop() # try a few times to get the heartbeat loss error. depending on # timing, sometimes we just get a connectionloss error for _ in range(4): event.wait(5) if event.is_set(): break else: self.proxy.start() time.sleep(2) # give it time to reconnect self.proxy.stop() assert event.is_set() # restart and we should be back up and running self.proxy.start() self.assertEqual(conn.call(receiver.name, "test"), "hats") receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def _init_dashi(self): # we are avoiding directly depending on dashi as this bridging approach # is short term and only used from CEI launches. And we have enough # deps. Where needed we install dashi specially via a separate # buildout config. try: import dashi except ImportError: log.warn( "Attempted to use Process Dispatcher bridge mode but the " + "dashi library dependency is not available.") raise return dashi.DashiConnection(self.dashi_name, self.uri, self.exchange)
def __init__(self, **kwargs): if 'name' in kwargs: self.name = kwargs['name'] else: self.name = who_is_calling() + "." + uuid.uuid4().hex kwargs['name'] = self.name self.conn = dashi.DashiConnection(**kwargs) self.conn.consumer_timeout = 0.01 self.received = [] self.reply_with = {} self.consumer_thread = None self.condition = threading.Condition()
def test_call_unknown_op(self): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test", True) receiver.consume_in_thread(1) conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) try: conn.call(receiver.name, "notarealop") except dashi.UnknownOperationError: pass else: self.fail("Expected UnknownOperationError") finally: receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_handle_sender_kwarg(self): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test1", "hello", sender_kwarg="sender") receiver.handle("test2", "hi", sender_kwarg="spender") receiver.consume_in_thread() sender_name = uuid.uuid4().hex conn = dashi.DashiConnection(sender_name, self.uri, "x1", transport_options=self.transport_options) args = dict(a=1, b="sandwich") expected_args1 = args.copy() expected_args1['sender'] = sender_name expected_args2 = args.copy() expected_args2['spender'] = sender_name # first one is a fire conn.fire(receiver.name, "test1", args=args) receiver.wait() opname, gotargs = receiver.received[0] self.assertEqual(opname, "test1") self.assertEqual(gotargs, expected_args1) receiver.clear() # next try a call reply = conn.call(receiver.name, "test2", **args) self.assertEqual(reply, "hi") opname, gotargs = receiver.received[0] self.assertEqual(opname, "test2") self.assertEqual(gotargs, expected_args2) receiver.cancel() receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def test_call_timeout_during_recovery(self): conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) got_timeout = threading.Event() def doit(): self.assertRaises(conn.timeout_error, conn.call, "notarealname", "test", timeout=3) got_timeout.set() countdown = dashi.util.Countdown(4) t = threading.Thread(target=doit) t.daemon = True t.start() time.sleep(0.5) try: got_timeout.wait(5) finally: t.join() delta = countdown.delta_seconds assert 0 < delta < 1, "delta: %s" % delta
def test_fire(self): receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("test") receiver.handle("test2") conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) args1 = dict(a=1, b="sandwich") conn.fire(receiver.name, "test", **args1) receiver.consume(1) self.assertEqual(len(receiver.received), 1) opname, gotargs = receiver.received[0] self.assertEqual(opname, "test") self.assertEqual(gotargs, args1) args2 = dict(a=2, b="burrito") args3 = dict(a=3) conn.fire(receiver.name, "test", **args2) conn.fire(receiver.name, "test2", **args3) receiver.clear() receiver.consume(2) self.assertEqual(len(receiver.received), 2) opname, gotargs = receiver.received[0] self.assertEqual(opname, "test") self.assertEqual(gotargs, args2) opname, gotargs = receiver.received[1] self.assertEqual(opname, "test2") self.assertEqual(gotargs, args3) receiver.disconnect() assert_kombu_pools_empty()
def test_call_handler_error(self): def raise_hell(): raise Exception("hell") receiver = TestReceiver(uri=self.uri, exchange="x1", transport_options=self.transport_options) receiver.handle("raiser", raise_hell) receiver.consume_in_thread(1) conn = dashi.DashiConnection("s1", self.uri, "x1", transport_options=self.transport_options) try: conn.call(receiver.name, "raiser") except dashi.DashiError: pass else: self.fail("Expected DashiError") finally: receiver.join_consumer_thread() receiver.disconnect() assert_kombu_pools_empty()
def _get_dashi(self, *args, **kwargs): # broken out to ease testing when dashi is not present import dashi return dashi.DashiConnection(*args, **kwargs)