def test_publish_heartbeat(self): broker = FakeAMQPBroker() client = WorkerHelper.get_fake_amqp_client(broker) channel = yield client.get_channel() pub = MockHeartBeatPublisher(self.gen_fake_attrs) pub.start(channel) pub._beat() [msg] = broker.get_dispatched("vumi.health", "heartbeat.inbound") self.assertEqual(json.loads(msg.body), self.gen_fake_attrs())
class TestHeartBeatPublisher(VumiTestCase): def gen_fake_attrs(self): attrs = { 'version': publisher.HeartBeatMessage.VERSION_20130319, 'system_id': "system-1", 'worker_id': "worker-1", 'worker_name': "worker-1", 'hostname': "test-host-1", 'timestamp': 100, 'pid': 43, } return attrs @inlineCallbacks def test_publish_heartbeat(self): self.broker = FakeAMQPBroker() channel = yield get_stubbed_channel(self.broker) pub = MockHeartBeatPublisher(self.gen_fake_attrs) pub.start(channel) pub._beat() [msg] = self.broker.get_dispatched("vumi.health", "heartbeat.inbound") self.assertEqual(json.loads(msg.body), self.gen_fake_attrs()) def test_message_validation(self): attrs = self.gen_fake_attrs() attrs.pop("version") self.assertRaises(MissingMessageField, publisher.HeartBeatMessage, **attrs) attrs = self.gen_fake_attrs() attrs.pop("system_id") self.assertRaises(MissingMessageField, publisher.HeartBeatMessage, **attrs) attrs = self.gen_fake_attrs() attrs.pop("worker_id") self.assertRaises(MissingMessageField, publisher.HeartBeatMessage, **attrs)
class Vas2NetsFailureWorkerTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.today = datetime.utcnow().date() self.port = 9999 self.path = "/api/v1/sms/vas2nets/receive/" self.config = { "transport_name": "vas2nets", "url": "http://localhost:%s%s" % (self.port, self.path), "username": "******", "password": "******", "owner": "owner", "service": "service", "subservice": "subservice", "web_receive_path": "/receive", "web_receipt_path": "/receipt", "web_port": 9998, } self.fail_config = { "transport_name": "vas2nets", "retry_routing_key": "sms.outbound.%(transport_name)s", "failures_routing_key": "sms.outbound.%(transport_name)s.failures", } self.workers = [] self.broker = FakeAMQPBroker() self.redis = FakeRedis() self.worker = yield self.mk_transport_worker(self.config, self.broker) self.fail_worker = yield self.mk_failure_worker(self.fail_config, self.broker, self.redis) def tearDown(self): for worker in self.workers: worker.stopWorker() @inlineCallbacks def mk_transport_worker(self, config, broker): worker = get_stubbed_worker(Vas2NetsTransport, config, broker) self.workers.append(worker) yield worker.startWorker() returnValue(worker) @inlineCallbacks def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(Vas2NetsFailureWorker, config, broker) self.workers.append(w) yield w.startWorker() w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis returnValue(w) @inlineCallbacks def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w) def get_dispatched(self, rkey): return self.broker.get_dispatched("vumi", rkey) def get_retry_keys(self): timestamps = self.redis.zrange(self.fail_worker._retry_timestamps_key, 0, 0) retry_keys = set() for timestamp in timestamps: bucket_key = self.fail_worker.r_key("retry_keys." + timestamp) retry_keys.update(self.redis.smembers(bucket_key)) return retry_keys def mkmsg_out(self, in_reply_to=None): msg = TransportSMS( transport="vas2nets", to_addr="+27761234567", from_addr="9292", message_id="1", in_reply_to=in_reply_to, transport_metadata={"network_id": "network-id"}, message="hello world", ) return msg def assert_dispatched_count(self, count, routing_key): self.assertEqual(count, len(self.get_dispatched(routing_key))) @inlineCallbacks def test_send_sms_success(self): yield self.mk_resource_worker("Result_code: 00, Message OK") yield self.worker.handle_outbound_message(self.mkmsg_out()) self.assert_dispatched_count(1, "sms.ack.vas2nets") self.assert_dispatched_count(0, "sms.failures.vas2nets") @inlineCallbacks def test_send_sms_fail(self): """ A 'No SmsId Header' error should not be retried. """ self.worker.failure_published = FailureCounter(1) self.worker.SUPPRESS_EXCEPTIONS = True yield self.mk_resource_worker("Result_code: 04, Internal system error " "occurred while processing message", {}) yield self.worker.handle_outbound_message(self.mkmsg_out()) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, "sms.ack.vas2nets") self.assert_dispatched_count(1, "sms.outbound.vas2nets.failures") [fmsg] = self.get_dispatched("sms.outbound.vas2nets.failures") fmsg = from_json(fmsg.body) self.assertTrue("Vas2NetsTransportError: No SmsId Header" in fmsg["reason"]) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set(), self.get_retry_keys()) @inlineCallbacks def test_send_sms_noconn(self): """ A 'connection refused' error should be retried. """ self.worker.failure_published = FailureCounter(1) self.worker.SUPPRESS_EXCEPTIONS = True msg = self.mkmsg_out() yield self.worker.handle_outbound_message(msg) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, "sms.ack.vas2nets") self.assert_dispatched_count(1, "sms.outbound.vas2nets.failures") [fmsg] = self.get_dispatched("sms.outbound.vas2nets.failures") fmsg = from_json(fmsg.body) self.assertEqual(msg, fmsg["message"]) self.assertEqual("connection refused", fmsg["reason"]) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set([key]), self.get_retry_keys())
class Vas2NetsFailureWorkerTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.today = datetime.utcnow().date() self.port = 9999 self.path = '/api/v1/sms/vas2nets/receive/' self.config = { 'transport_name': 'vas2nets', 'url': 'http://localhost:%s%s' % (self.port, self.path), 'username': '******', 'password': '******', 'owner': 'owner', 'service': 'service', 'subservice': 'subservice', 'web_receive_path': '/receive', 'web_receipt_path': '/receipt', 'web_port': 9998, } self.fail_config = { 'transport_name': 'vas2nets', 'retry_routing_key': '%(transport_name)s.outbound', 'failures_routing_key': '%(transport_name)s.failures', } self.workers = [] self.broker = FakeAMQPBroker() self.redis = FakeRedis() self.worker = yield self.mk_transport_worker(self.config, self.broker) self.fail_worker = yield self.mk_failure_worker( self.fail_config, self.broker, self.redis) def tearDown(self): for worker in self.workers: worker.stopWorker() @inlineCallbacks def mk_transport_worker(self, config, broker): worker = get_stubbed_worker(Vas2NetsTransport, config, broker) self.workers.append(worker) yield worker.startWorker() returnValue(worker) @inlineCallbacks def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(FailureWorker, config, broker) self.workers.append(w) yield w.startWorker() w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis returnValue(w) @inlineCallbacks def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w) def get_dispatched(self, rkey): return self.broker.get_dispatched('vumi', rkey) def get_retry_keys(self): timestamps = self.redis.zrange( self.fail_worker._retry_timestamps_key, 0, 0) retry_keys = set() for timestamp in timestamps: bucket_key = self.fail_worker.r_key("retry_keys." + timestamp) retry_keys.update(self.redis.smembers(bucket_key)) return retry_keys def mkmsg_out(self, in_reply_to=None): return TransportUserMessage( to_addr='+41791234567', from_addr='9292', message_id='1', transport_name='vas2nets', transport_type='sms', transport_metadata={ 'network_id': 'network-id', }, content='hello world', in_reply_to=in_reply_to, ) def assert_dispatched_count(self, count, routing_key): self.assertEqual(count, len(self.get_dispatched(routing_key))) @inlineCallbacks def test_send_sms_success(self): yield self.mk_resource_worker("Result_code: 00, Message OK") yield self.worker._process_message(self.mkmsg_out()) self.assert_dispatched_count(1, 'vas2nets.event') self.assert_dispatched_count(0, 'vas2nets.failures') @inlineCallbacks def test_send_sms_fail(self): """ A 'No SmsId Header' error should not be retried. """ self.worker.failure_published = FailureCounter(1) yield self.mk_resource_worker("Result_code: 04, Internal system error " "occurred while processing message", {}) yield self.worker._process_message(self.mkmsg_out()) yield self.worker.failure_published.deferred yield self.broker.kick_delivery() self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertTrue( "Vas2NetsTransportError: No SmsId Header" in fmsg['reason']) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set(), self.get_retry_keys()) @inlineCallbacks def test_send_sms_noconn(self): """ A 'connection refused' error should be retried. """ self.worker.failure_published = FailureCounter(1) msg = self.mkmsg_out() yield self.worker._process_message(msg) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg.payload, fmsg['message']) self.assertEqual(FailureMessage.FC_TEMPORARY, fmsg['failure_code']) self.assertTrue(fmsg['reason'].strip().endswith("connection refused")) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set([key]), self.get_retry_keys())
class Vas2NetsTransportTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.path = '/api/v1/sms/vas2nets/receive/' self.port = 9999 self.config = { 'transport_name': 'vas2nets', 'url': 'http://localhost:%s%s' % (self.port, self.path), 'username': '******', 'password': '******', 'owner': 'owner', 'service': 'service', 'subservice': 'subservice', 'web_receive_path': '/receive', 'web_receipt_path': '/receipt', 'web_port': 9998, } self.broker = FakeAMQPBroker() self.worker = get_stubbed_worker(Vas2NetsTransport, self.config, self.broker) self.publisher = yield self.worker.publish_to('some.routing.key') self.today = datetime.utcnow().date() self.workers = [self.worker] def tearDown(self): for worker in self.workers: worker.stopWorker() def make_resource_worker(self, msg_id, msg, code=http.OK, send_id=None): w = get_stubbed_worker(TestResourceWorker, {}) w.set_resources([ (self.path, TestResource, (msg_id, msg, code, send_id))]) self.workers.append(w) return w.startWorker() def get_dispatched(self, rkey): return self.broker.get_dispatched('vumi', rkey) def create_request(self, dictionary={}, path='/', method='POST'): """ Creates a dummy Vas2Nets request for testing our resources with """ request = DummyRequest(path) request.method = method args = { 'messageid': [str(uuid4())], 'time': [self.today.strftime('%Y.%m.%d %H:%M:%S')], 'sender': ['0041791234567'], 'destination': ['9292'], 'provider': ['provider'], 'keyword': [''], 'header': [''], 'text': [''], 'keyword': [''], } args.update(dictionary) request.args = args return request def make_delivery_message(self, status, tr_status, tr_message): return TransportSMSDeliveryReport( transport='vas2nets', transport_message_id='1', transport_metadata={ 'delivery_status': tr_status, 'network_id': 'provider', 'timestamp': self.today.strftime('%Y-%m-%dT%H:%M:%S'), 'delivery_message': tr_message, }, to_addr='+41791234567', message_id='internal id', delivery_status=status, timestamp=UTCNearNow(), ) def mkmsg_in(self): msg = TransportSMS( message_type='sms', message_version='20110907', from_addr='+41791234567', to_addr='9292', message_id='1', transport='vas2nets', transport_message_id='1', transport_metadata={ 'network_id': 'provider', 'keyword': '', 'timestamp': self.today.strftime('%Y-%m-%dT%H:%M:%S'), }, message='hello world', timestamp=UTCNearNow(), ) return msg def mkmsg_out(self, in_reply_to=None): msg = TransportSMS( transport='vas2nets', to_addr='+27761234567', from_addr='9292', message_id='1', in_reply_to=in_reply_to, transport_metadata={ 'network_id': 'network-id', }, message='hello world', ) return msg @inlineCallbacks def test_receive_sms(self): resource = ReceiveSMSResource(self.config, self.publisher) request = self.create_request({ 'messageid': ['1'], 'text': ['hello world'], }) d = request.notifyFinish() response = resource.render(request) self.assertEquals(response, NOT_DONE_YET) yield d self.assertEquals('', ''.join(request.written)) self.assertEquals(request.outgoingHeaders['content-type'], 'text/plain') self.assertEquals(request.responseCode, http.OK) msg = self.mkmsg_in() [smsg] = self.get_dispatched('sms.inbound.vas2nets.9292') self.assertEquals(msg.payload, from_json(smsg.body)) @inlineCallbacks def test_delivery_receipt_pending(self): resource = DeliveryReceiptResource(self.config, self.publisher) request = self.create_request({ 'smsid': ['1'], 'messageid': ['internal id'], 'sender': ['+41791234567'], 'status': ['1'], 'text': ['Message submitted to Provider for delivery.'], }) d = request.notifyFinish() response = resource.render(request) self.assertEquals(response, NOT_DONE_YET) yield d self.assertEquals('', ''.join(request.written)) self.assertEquals(request.outgoingHeaders['content-type'], 'text/plain') self.assertEquals(request.responseCode, http.OK) msg = self.make_delivery_message( 'pending', '1', 'Message submitted to Provider for delivery.') [smsg] = self.get_dispatched('sms.receipt.vas2nets') self.assertEquals(msg, from_json(smsg.body)) @inlineCallbacks def test_delivery_receipt_failed(self): resource = DeliveryReceiptResource(self.config, self.publisher) request = self.create_request({ 'smsid': ['1'], 'messageid': ['internal id'], 'sender': ['+41791234567'], 'status': ['-9'], 'text': ['Message could not be delivered.'], }) d = request.notifyFinish() response = resource.render(request) self.assertEquals(response, NOT_DONE_YET) yield d self.assertEquals('', ''.join(request.written)) self.assertEquals(request.outgoingHeaders['content-type'], 'text/plain') self.assertEquals(request.responseCode, http.OK) msg = self.make_delivery_message( 'failed', '-9', 'Message could not be delivered.') [smsg] = self.get_dispatched('sms.receipt.vas2nets') self.assertEquals(from_json(smsg.body), msg) @inlineCallbacks def test_delivery_receipt_delivered(self): resource = DeliveryReceiptResource(self.config, self.publisher) request = self.create_request({ 'smsid': ['1'], 'messageid': ['internal id'], 'sender': ['+41791234567'], 'status': ['2'], 'text': ['Message delivered to MSISDN.'], }) d = request.notifyFinish() response = resource.render(request) self.assertEquals(response, NOT_DONE_YET) yield d self.assertEquals('', ''.join(request.written)) self.assertEquals(request.outgoingHeaders['content-type'], 'text/plain') self.assertEquals(request.responseCode, http.OK) msg = self.make_delivery_message( 'delivered', '2', 'Message delivered to MSISDN.') [smsg] = self.get_dispatched('sms.receipt.vas2nets') self.assertEquals(from_json(smsg.body), msg) def test_validate_characters(self): self.assertRaises(Vas2NetsEncodingError, validate_characters, u"ïøéå¬∆˚") self.assertTrue(validate_characters(string.ascii_lowercase)) self.assertTrue(validate_characters(string.ascii_uppercase)) self.assertTrue(validate_characters('0123456789')) self.assertTrue(validate_characters(u'äöü ÄÖÜ àùò ìèé §Ññ £$@')) self.assertTrue(validate_characters(u'/?!#%&()*+,-:;<=>.')) self.assertTrue(validate_characters(u'testing\ncarriage\rreturns')) self.assertTrue(validate_characters(u'testing "quotes"')) self.assertTrue(validate_characters(u"testing 'quotes'")) @inlineCallbacks def test_send_sms_success(self): mocked_message_id = str(uuid4()) mocked_message = "Result_code: 00, Message OK" # open an HTTP resource that mocks the Vas2Nets response for the # duration of this test yield self.make_resource_worker(mocked_message_id, mocked_message) yield self.worker.startWorker() yield self.worker.handle_outbound_message(self.mkmsg_out()) [smsg] = self.get_dispatched('sms.ack.vas2nets') self.assertEqual(TransportSMSAck( message_id='1', transport_message_id=mocked_message_id, timestamp=UTCNearNow(), transport='vas2nets', ), from_json(smsg.body)) @inlineCallbacks def test_send_sms_reply_success(self): mocked_message_id = str(uuid4()) reply_to_msgid = str(uuid4()) mocked_message = "Result_code: 00, Message OK" # open an HTTP resource that mocks the Vas2Nets response for the # duration of this test yield self.make_resource_worker(mocked_message_id, mocked_message, send_id=reply_to_msgid) yield self.worker.startWorker() yield self.worker.handle_outbound_message(self.mkmsg_out( in_reply_to=reply_to_msgid)) [smsg] = self.get_dispatched('sms.ack.vas2nets') self.assertEqual(TransportSMSAck( message_id='1', transport_message_id=mocked_message_id, timestamp=UTCNearNow(), transport='vas2nets', ), from_json(smsg.body)) @inlineCallbacks def test_send_sms_fail(self): mocked_message_id = False mocked_message = "Result_code: 04, Internal system error occurred " \ "while processing message" yield self.make_resource_worker(mocked_message_id, mocked_message) yield self.worker.startWorker() msg = self.mkmsg_out() deferred = self.worker.handle_outbound_message(msg) yield deferred [fmsg] = self.get_dispatched('sms.outbound.vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg.payload, fmsg['message']) self.assertTrue( "Vas2NetsTransportError: No SmsId Header" in fmsg['reason']) @inlineCallbacks def test_send_sms_noconn(self): yield self.worker.startWorker() msg = self.mkmsg_out() deferred = self.worker.handle_outbound_message(msg) yield deferred [fmsg] = self.get_dispatched('sms.outbound.vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg, fmsg['message']) self.assertEqual("connection refused", fmsg['reason']) @inlineCallbacks def test_send_sms_not_OK(self): mocked_message = "Page not found." yield self.make_resource_worker(None, mocked_message, http.NOT_FOUND) yield self.worker.startWorker() msg = self.mkmsg_out() deferred = self.worker.handle_outbound_message(msg) yield deferred [fmsg] = self.get_dispatched('sms.outbound.vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg, fmsg['message']) self.assertTrue(fmsg['reason'].startswith("server error")) def test_normalize_outbound_msisdn(self): self.assertEquals(normalize_outbound_msisdn('+27761234567'), '0027761234567')
class Vas2NetsFailureWorkerTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.today = datetime.utcnow().date() self.port = 9999 self.path = '/api/v1/sms/vas2nets/receive/' self.config = { 'transport_name': 'vas2nets', 'url': 'http://localhost:%s%s' % (self.port, self.path), 'username': '******', 'password': '******', 'owner': 'owner', 'service': 'service', 'subservice': 'subservice', 'web_receive_path': '/receive', 'web_receipt_path': '/receipt', 'web_port': 9998, } self.fail_config = { 'transport_name': 'vas2nets', 'retry_routing_key': '%(transport_name)s.outbound', 'failures_routing_key': '%(transport_name)s.failures', } self.workers = [] self.broker = FakeAMQPBroker() self.redis = FakeRedis() self.worker = yield self.mk_transport_worker(self.config, self.broker) self.fail_worker = yield self.mk_failure_worker( self.fail_config, self.broker, self.redis) def tearDown(self): for worker in self.workers: worker.stopWorker() @inlineCallbacks def mk_transport_worker(self, config, broker): worker = get_stubbed_worker(Vas2NetsTransport, config, broker) self.workers.append(worker) yield worker.startWorker() returnValue(worker) @inlineCallbacks def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(FailureWorker, config, broker) self.workers.append(w) w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis yield w.startWorker() returnValue(w) @inlineCallbacks def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w) def get_dispatched(self, rkey): return self.broker.get_dispatched('vumi', rkey) def get_retry_keys(self): timestamps = self.redis.zrange( self.fail_worker._retry_timestamps_key, 0, 0) retry_keys = set() for timestamp in timestamps: bucket_key = self.fail_worker.r_key("retry_keys." + timestamp) retry_keys.update(self.redis.smembers(bucket_key)) return retry_keys def mkmsg_out(self, in_reply_to=None): return TransportUserMessage( to_addr='+41791234567', from_addr='9292', message_id='1', transport_name='vas2nets', transport_type='sms', transport_metadata={ 'network_id': 'network-id', }, content='hello world', in_reply_to=in_reply_to, ) def assert_dispatched_count(self, count, routing_key): self.assertEqual(count, len(self.get_dispatched(routing_key))) @inlineCallbacks def test_send_sms_success(self): yield self.mk_resource_worker("Result_code: 00, Message OK") yield self.worker._process_message(self.mkmsg_out()) self.assert_dispatched_count(1, 'vas2nets.event') self.assert_dispatched_count(0, 'vas2nets.failures') @inlineCallbacks def test_send_sms_fail(self): """ A 'No SmsId Header' error should not be retried. """ self.worker.failure_published = FailureCounter(1) yield self.mk_resource_worker("Result_code: 04, Internal system error " "occurred while processing message", {}) yield self.worker._process_message(self.mkmsg_out()) yield self.worker.failure_published.deferred yield self.broker.kick_delivery() self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [twisted_failure] = self.flushLoggedErrors(Vas2NetsTransportError) failure = twisted_failure.value self.assertTrue("No SmsId Header" in str(failure)) [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertTrue( "Vas2NetsTransportError: No SmsId Header" in fmsg['reason']) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set(), self.get_retry_keys()) @inlineCallbacks def test_send_sms_noconn(self): """ A 'connection refused' error should be retried. """ self.worker.failure_published = FailureCounter(1) msg = self.mkmsg_out() yield self.worker._process_message(msg) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [twisted_failure] = self.flushLoggedErrors(TemporaryFailure) failure = twisted_failure.value self.assertTrue("connection refused" in str(failure)) [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg.payload, fmsg['message']) self.assertEqual(FailureMessage.FC_TEMPORARY, fmsg['failure_code']) self.assertTrue(fmsg['reason'].strip().endswith("connection refused")) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set([key]), self.get_retry_keys())