class DispatcherTestCase(TestCase): """ This is a base class for testing dispatcher workers. """ # base timeout of 5s for all dispatcher tests timeout = 5 dispatcher_name = "sphex_dispatcher" dispatcher_class = None def setUp(self): self._workers = [] self._amqp = FakeAMQPBroker() def tearDown(self): for worker in self._workers: worker.stopWorker() @inlineCallbacks def get_dispatcher(self, config, cls=None, start=True): """ Get an instance of a dispatcher class. :param config: Config dict. :param cls: The Dispatcher class to instantiate. Defaults to :attr:`dispatcher_class` :param start: True to start the displatcher (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``dispatcher_name`` defaults to :attr:`self.dispatcher_name` """ if cls is None: cls = self.dispatcher_class config.setdefault('dispatcher_name', self.dispatcher_name) worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def mkmsg_in(self, content='hello world', message_id='abc', to_addr='9292', from_addr='+41791234567', session_event=None, transport_type=None, helper_metadata=None, transport_metadata=None, transport_name=None): if helper_metadata is None: helper_metadata = {} if transport_metadata is None: transport_metadata = {} return TransportUserMessage( from_addr=from_addr, to_addr=to_addr, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, helper_metadata=helper_metadata, content=content, session_event=session_event, timestamp=datetime.now(), ) def mkmsg_out(self, content='hello world', message_id='1', to_addr='+41791234567', from_addr='9292', session_event=None, in_reply_to=None, transport_type=None, transport_metadata=None, transport_name=None): if transport_metadata is None: transport_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, ) return TransportUserMessage(**params) def mkmsg_ack(self, event_type='ack', user_message_id='1', send_message_id='abc', transport_name=None, transport_metadata=None): if transport_metadata is None: transport_metadata = {} params = dict( event_type=event_type, user_message_id=user_message_id, sent_message_id=send_message_id, transport_name=transport_name, transport_metadata=transport_metadata, ) return TransportEvent(**params) def get_dispatched_messages(self, transport_name, direction='outbound'): return self._amqp.get_messages('vumi', '%s.%s' % ( transport_name, direction)) def wait_for_dispatched_messages(self, transport_name, amount, direction='outbound'): return self._amqp.wait_messages('vumi', '%s.%s' % ( transport_name, direction), amount) def dispatch(self, message, transport_name, direction='inbound', exchange='vumi'): rkey = '%s.%s' % (transport_name, direction) self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery()
class Vas2NetsFailureWorkerTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.today = datetime.utcnow().date() self.port = 9999 self.path = "/api/v1/sms/vas2nets/receive/" self.config = { "transport_name": "vas2nets", "url": "http://localhost:%s%s" % (self.port, self.path), "username": "******", "password": "******", "owner": "owner", "service": "service", "subservice": "subservice", "web_receive_path": "/receive", "web_receipt_path": "/receipt", "web_port": 9998, } self.fail_config = { "transport_name": "vas2nets", "retry_routing_key": "sms.outbound.%(transport_name)s", "failures_routing_key": "sms.outbound.%(transport_name)s.failures", } self.workers = [] self.broker = FakeAMQPBroker() self.redis = FakeRedis() self.worker = yield self.mk_transport_worker(self.config, self.broker) self.fail_worker = yield self.mk_failure_worker(self.fail_config, self.broker, self.redis) def tearDown(self): for worker in self.workers: worker.stopWorker() @inlineCallbacks def mk_transport_worker(self, config, broker): worker = get_stubbed_worker(Vas2NetsTransport, config, broker) self.workers.append(worker) yield worker.startWorker() returnValue(worker) @inlineCallbacks def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(Vas2NetsFailureWorker, config, broker) self.workers.append(w) yield w.startWorker() w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis returnValue(w) @inlineCallbacks def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w) def get_dispatched(self, rkey): return self.broker.get_dispatched("vumi", rkey) def get_retry_keys(self): timestamps = self.redis.zrange(self.fail_worker._retry_timestamps_key, 0, 0) retry_keys = set() for timestamp in timestamps: bucket_key = self.fail_worker.r_key("retry_keys." + timestamp) retry_keys.update(self.redis.smembers(bucket_key)) return retry_keys def mkmsg_out(self, in_reply_to=None): msg = TransportSMS( transport="vas2nets", to_addr="+27761234567", from_addr="9292", message_id="1", in_reply_to=in_reply_to, transport_metadata={"network_id": "network-id"}, message="hello world", ) return msg def assert_dispatched_count(self, count, routing_key): self.assertEqual(count, len(self.get_dispatched(routing_key))) @inlineCallbacks def test_send_sms_success(self): yield self.mk_resource_worker("Result_code: 00, Message OK") yield self.worker.handle_outbound_message(self.mkmsg_out()) self.assert_dispatched_count(1, "sms.ack.vas2nets") self.assert_dispatched_count(0, "sms.failures.vas2nets") @inlineCallbacks def test_send_sms_fail(self): """ A 'No SmsId Header' error should not be retried. """ self.worker.failure_published = FailureCounter(1) self.worker.SUPPRESS_EXCEPTIONS = True yield self.mk_resource_worker("Result_code: 04, Internal system error " "occurred while processing message", {}) yield self.worker.handle_outbound_message(self.mkmsg_out()) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, "sms.ack.vas2nets") self.assert_dispatched_count(1, "sms.outbound.vas2nets.failures") [fmsg] = self.get_dispatched("sms.outbound.vas2nets.failures") fmsg = from_json(fmsg.body) self.assertTrue("Vas2NetsTransportError: No SmsId Header" in fmsg["reason"]) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set(), self.get_retry_keys()) @inlineCallbacks def test_send_sms_noconn(self): """ A 'connection refused' error should be retried. """ self.worker.failure_published = FailureCounter(1) self.worker.SUPPRESS_EXCEPTIONS = True msg = self.mkmsg_out() yield self.worker.handle_outbound_message(msg) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, "sms.ack.vas2nets") self.assert_dispatched_count(1, "sms.outbound.vas2nets.failures") [fmsg] = self.get_dispatched("sms.outbound.vas2nets.failures") fmsg = from_json(fmsg.body) self.assertEqual(msg, fmsg["message"]) self.assertEqual("connection refused", fmsg["reason"]) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set([key]), self.get_retry_keys())
class TransportTestCase(unittest.TestCase): """ This is a base class for testing transports. Not to be confused with BaseTransportTestCase below. """ # have transport tests timeout after 5s by default timeout = 5 transport_name = "sphex" transport_type = None transport_class = None def setUp(self): self._workers = [] self._amqp = FakeAMQPBroker() @inlineCallbacks def tearDown(self): for worker in self._workers: yield worker.stopWorker() def rkey(self, name): return "%s.%s" % (self.transport_name, name) @inlineCallbacks def get_transport(self, config, cls=None, start=True): """ Get an instance of a transport class. :param config: Config dict. :param cls: The transport class to instantiate. Defaults to :attr:`transport_class` :param start: True to start the transport (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``transport_name`` defaults to :attr:`self.transport_name` """ if cls is None: cls = self.transport_class config.setdefault('transport_name', self.transport_name) worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def assert_rkey_attr(self, rkey_suffix, obj, tr_name=None): if tr_name is None: tr_name = self.transport_name self.assertEqual("%s.%s" % (tr_name, rkey_suffix), obj.routing_key) def assert_basic_rkeys(self, transport): self.assert_rkey_attr('event', transport.event_publisher) self.assert_rkey_attr('inbound', transport.message_publisher) self.assert_rkey_attr('failures', transport.failure_publisher) self.assert_rkey_attr('outbound', transport.message_consumer) def mkmsg_ack(self, user_message_id='1', sent_message_id='abc', transport_metadata=None): if transport_metadata is None: transport_metadata = {} return TransportEvent( event_id=RegexMatcher(r'^[0-9a-fA-F]{32}$'), event_type='ack', user_message_id=user_message_id, sent_message_id=sent_message_id, timestamp=UTCNearNow(), transport_name=self.transport_name, transport_metadata=transport_metadata, ) def mkmsg_delivery(self, status='delivered', user_message_id='abc', transport_metadata=None): if transport_metadata is None: transport_metadata = {} return TransportEvent( event_id=RegexMatcher(r'^[0-9a-fA-F]{32}$'), event_type='delivery_report', transport_name=self.transport_name, user_message_id=user_message_id, delivery_status=status, to_addr='+41791234567', timestamp=UTCNearNow(), transport_metadata=transport_metadata, ) def mkmsg_in(self, content='hello world', session_event=TransportUserMessage.SESSION_NONE, message_id='abc', transport_type=None, transport_metadata=None): if transport_type is None: transport_type = self.transport_type if transport_metadata is None: transport_metadata = {} return TransportUserMessage( from_addr='+41791234567', to_addr='9292', group=None, message_id=message_id, transport_name=self.transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, timestamp=UTCNearNow(), ) def mkmsg_out(self, content='hello world', session_event=TransportUserMessage.SESSION_NONE, message_id='1', to_addr='+41791234567', from_addr='9292', group=None, in_reply_to=None, transport_type=None, transport_metadata=None, helper_metadata=None): if transport_type is None: transport_type = self.transport_type if transport_metadata is None: transport_metadata = {} if helper_metadata is None: helper_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, group=group, message_id=message_id, transport_name=self.transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, helper_metadata=helper_metadata, ) return TransportUserMessage(**params) def mkmsg_fail(self, message, reason, failure_code=FailureMessage.FC_UNSPECIFIED): return FailureMessage( timestamp=UTCNearNow(), failure_code=failure_code, message=message, reason=reason, ) def get_dispatched_events(self): return self._amqp.get_messages('vumi', self.rkey('event')) def get_dispatched_messages(self): return self._amqp.get_messages('vumi', self.rkey('inbound')) def get_dispatched_failures(self): return self._amqp.get_messages('vumi', self.rkey('failures')) def wait_for_dispatched_messages(self, amount): return self._amqp.wait_messages('vumi', self.rkey('inbound'), amount) def clear_dispatched_messages(self): self._amqp.clear_messages('vumi', self.rkey('inbound')) def dispatch(self, message, rkey=None, exchange='vumi'): if rkey is None: rkey = self.rkey('outbound') self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery()
class Vas2NetsFailureWorkerTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.today = datetime.utcnow().date() self.port = 9999 self.path = '/api/v1/sms/vas2nets/receive/' self.config = { 'transport_name': 'vas2nets', 'url': 'http://localhost:%s%s' % (self.port, self.path), 'username': '******', 'password': '******', 'owner': 'owner', 'service': 'service', 'subservice': 'subservice', 'web_receive_path': '/receive', 'web_receipt_path': '/receipt', 'web_port': 9998, } self.fail_config = { 'transport_name': 'vas2nets', 'retry_routing_key': '%(transport_name)s.outbound', 'failures_routing_key': '%(transport_name)s.failures', } self.workers = [] self.broker = FakeAMQPBroker() self.redis = FakeRedis() self.worker = yield self.mk_transport_worker(self.config, self.broker) self.fail_worker = yield self.mk_failure_worker( self.fail_config, self.broker, self.redis) def tearDown(self): for worker in self.workers: worker.stopWorker() @inlineCallbacks def mk_transport_worker(self, config, broker): worker = get_stubbed_worker(Vas2NetsTransport, config, broker) self.workers.append(worker) yield worker.startWorker() returnValue(worker) @inlineCallbacks def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(FailureWorker, config, broker) self.workers.append(w) yield w.startWorker() w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis returnValue(w) @inlineCallbacks def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w) def get_dispatched(self, rkey): return self.broker.get_dispatched('vumi', rkey) def get_retry_keys(self): timestamps = self.redis.zrange( self.fail_worker._retry_timestamps_key, 0, 0) retry_keys = set() for timestamp in timestamps: bucket_key = self.fail_worker.r_key("retry_keys." + timestamp) retry_keys.update(self.redis.smembers(bucket_key)) return retry_keys def mkmsg_out(self, in_reply_to=None): return TransportUserMessage( to_addr='+41791234567', from_addr='9292', message_id='1', transport_name='vas2nets', transport_type='sms', transport_metadata={ 'network_id': 'network-id', }, content='hello world', in_reply_to=in_reply_to, ) def assert_dispatched_count(self, count, routing_key): self.assertEqual(count, len(self.get_dispatched(routing_key))) @inlineCallbacks def test_send_sms_success(self): yield self.mk_resource_worker("Result_code: 00, Message OK") yield self.worker._process_message(self.mkmsg_out()) self.assert_dispatched_count(1, 'vas2nets.event') self.assert_dispatched_count(0, 'vas2nets.failures') @inlineCallbacks def test_send_sms_fail(self): """ A 'No SmsId Header' error should not be retried. """ self.worker.failure_published = FailureCounter(1) yield self.mk_resource_worker("Result_code: 04, Internal system error " "occurred while processing message", {}) yield self.worker._process_message(self.mkmsg_out()) yield self.worker.failure_published.deferred yield self.broker.kick_delivery() self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertTrue( "Vas2NetsTransportError: No SmsId Header" in fmsg['reason']) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set(), self.get_retry_keys()) @inlineCallbacks def test_send_sms_noconn(self): """ A 'connection refused' error should be retried. """ self.worker.failure_published = FailureCounter(1) msg = self.mkmsg_out() yield self.worker._process_message(msg) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg.payload, fmsg['message']) self.assertEqual(FailureMessage.FC_TEMPORARY, fmsg['failure_code']) self.assertTrue(fmsg['reason'].strip().endswith("connection refused")) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set([key]), self.get_retry_keys())
class WikipediaWorkerTestCase(TestCase): transport_name = 'sphex' timeout = 10 @inlineCallbacks def setUp(self): self.broker = FakeAMQPBroker() self._workers = [] self.worker = yield self.get_worker() @inlineCallbacks def tearDown(self): for w in self._workers: yield w.stopWorker() @inlineCallbacks def get_worker(self, config=None): if not config: config = { 'worker_name': 'wikitest', 'sms_transport': 'sphex', } config.setdefault('transport_name', self.transport_name) worker = get_stubbed_worker(WikipediaWorker, config, self.broker) self._workers.append(worker) yield worker.startWorker() returnValue(worker) def mkmsg_in(self, content): return TransportUserMessage( from_addr='+41791234567', to_addr='9292', message_id='abc', transport_name=self.transport_name, transport_type='ussd', transport_metadata={}, content=content, ) def rkey(self, name): return "%s.%s" % (self.transport_name, name) def dispatch(self, message, rkey=None, exchange='vumi'): if rkey is None: rkey = self.rkey('inbound') self.broker.publish_message(exchange, rkey, message) return self.broker.kick_delivery() def get_dispatched_messages(self): return self.broker.get_messages('vumi', self.rkey('outbound')) @inlineCallbacks def test_happy_flow(self): # TODO: Stub out the API calls. yield self.dispatch(self.mkmsg_in(None)) self.assertEqual('What would you like to search Wikipedia for?', self.get_dispatched_messages()[-1]['content']) yield self.dispatch(self.mkmsg_in('africa')) self.assertEqual('\n'.join([ '1. Africa', '2. .africa', '3. African American', '4. North Africa', '5. Kenya', '6. Sub-Saharan Africa', '7. Africa (Roman province)', '8. African people', '9. Confederation of African Football']), self.get_dispatched_messages()[-1]['content']) yield self.dispatch(self.mkmsg_in('1')) self.assertEqual('\n'.join([ '1. Africa', '2. Etymology', '3. History', '4. Geography', '5. Biodiversity', '6. Politics', '7. Economy', '8. Demographics', '9. Languages', '10. Culture', '11. Religion', '12. Territories and regions', '13. See also', '14. References', '15. Further reading', '16. External links', ]), self.get_dispatched_messages()[-1]['content']) yield self.dispatch(self.mkmsg_in('2')) content = ( u'==Etymology==\n\n[[Afri]] was a Latin name used to refer to the ' u'[[Carthaginians]] who dwelt in [[North Africa]] in modern-day ' u'[[Tunisia]]. Their name is usually connected with [[Phoenician ' u'language|Phoenician]] \'\'afar\'\', "dust", but a 1981 ' u'hypothesis<ref>[http://michel-desfayes.org/namesofcountries.html' u' Names of countries], Decret and Fantar, 1981</ref> has ' u'asserted that it stems from the [[Berber language|Berber]] word ' u'\'\'ifri\'\' or \'\'ifran\'\' meaning "cave" and "caves", in ' u'reference to cave dweller') self.assertEqual( "%s...\n(Full content sent by SMS.)" % (content[:100],), self.get_dispatched_messages()[-2]['content']) self.assertEqual(content[:250], self.get_dispatched_messages()[-1]['content'])
class VumiWorkerTestCase(VumiTestCase): """Base test class for vumi workers. This (or a subclass of this) should be the starting point for any test cases that involve vumi workers. """ transport_name = "sphex" transport_type = None MSG_ID_MATCHER = RegexMatcher(r'^[0-9a-fA-F]{32}$') def setUp(self): warnings.warn("VumiWorkerTestCase and its subclasses are deprecated. " "Use VumiTestCase and other tools from " "vumi.tests.helpers instead.", category=DeprecationWarning) self._workers = [] self._amqp = FakeAMQPBroker() @inlineCallbacks def tearDown(self): yield super(VumiWorkerTestCase, self).tearDown() # Wait for any pending message deliveries to avoid a race with a dirty # reactor. yield self._amqp.wait_delivery() # Now stop all the workers. for worker in self._workers: yield worker.stopWorker() def rkey(self, name): return "%s.%s" % (self.transport_name, name) def _rkey(self, name, connector_name=None): if connector_name is None: return self.rkey(name) return "%s.%s" % (connector_name, name) @inlineCallbacks def get_worker(self, config, cls, start=True): """Create and return an instance of a vumi worker. :param config: Config dict. :param cls: The worker class to instantiate. :param start: True to start the worker (default), False otherwise. """ # When possible, always try and enable heartbeat setup in tests. # so make sure worker_name is set if (config is not None) and ('worker_name' not in config): config['worker_name'] = "unnamed" worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def mkmsg_ack(self, user_message_id='1', sent_message_id='abc', transport_metadata=None, transport_name=None): if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportEvent( event_type='ack', user_message_id=user_message_id, sent_message_id=sent_message_id, transport_name=transport_name, transport_metadata=transport_metadata, ) def mkmsg_nack(self, user_message_id='1', transport_metadata=None, transport_name=None, nack_reason='unknown'): if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportEvent( event_type='nack', nack_reason=nack_reason, user_message_id=user_message_id, transport_name=transport_name, transport_metadata=transport_metadata, ) def mkmsg_delivery(self, status='delivered', user_message_id='abc', transport_metadata=None, transport_name=None): if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportEvent( event_type='delivery_report', transport_name=transport_name, user_message_id=user_message_id, delivery_status=status, to_addr='+41791234567', transport_metadata=transport_metadata, ) def mkmsg_in(self, content='hello world', message_id='abc', to_addr='9292', from_addr='+41791234567', group=None, session_event=None, transport_type=None, helper_metadata=None, transport_metadata=None, transport_name=None): if transport_type is None: transport_type = self.transport_type if helper_metadata is None: helper_metadata = {} if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportUserMessage( from_addr=from_addr, to_addr=to_addr, group=group, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, helper_metadata=helper_metadata, content=content, session_event=session_event, timestamp=datetime.now(), ) def mkmsg_out(self, content='hello world', message_id='1', to_addr='+41791234567', from_addr='9292', group=None, session_event=None, in_reply_to=None, transport_type=None, transport_metadata=None, transport_name=None, helper_metadata=None, ): if transport_type is None: transport_type = self.transport_type if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name if helper_metadata is None: helper_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, group=group, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, helper_metadata=helper_metadata, ) return TransportUserMessage(**params) def _make_matcher(self, msg, *id_fields): msg['timestamp'] = UTCNearNow() for field in id_fields: msg[field] = self.MSG_ID_MATCHER return msg def _get_dispatched(self, name, connector_name=None): rkey = self._rkey(name, connector_name) return self._amqp.get_messages('vumi', rkey) def _wait_for_dispatched(self, name, amount, connector_name=None): rkey = self._rkey(name, connector_name) return self._amqp.wait_messages('vumi', rkey, amount) def clear_all_dispatched(self): self._amqp.clear_messages('vumi') def _clear_dispatched(self, name, connector_name=None): rkey = self._rkey(name, connector_name) return self._amqp.clear_messages('vumi', rkey) def get_dispatched_events(self, connector_name=None): return self._get_dispatched('event', connector_name) def get_dispatched_inbound(self, connector_name=None): return self._get_dispatched('inbound', connector_name) def get_dispatched_outbound(self, connector_name=None): return self._get_dispatched('outbound', connector_name) def get_dispatched_failures(self, connector_name=None): return self._get_dispatched('failures', connector_name) def wait_for_dispatched_events(self, amount, connector_name=None): return self._wait_for_dispatched('event', amount, connector_name) def wait_for_dispatched_inbound(self, amount, connector_name=None): return self._wait_for_dispatched('inbound', amount, connector_name) def wait_for_dispatched_outbound(self, amount, connector_name=None): return self._wait_for_dispatched('outbound', amount, connector_name) def wait_for_dispatched_failures(self, amount, connector_name=None): return self._wait_for_dispatched('failures', amount, connector_name) def clear_dispatched_events(self, connector_name=None): return self._clear_dispatched('event', connector_name) def clear_dispatched_inbound(self, connector_name=None): return self._clear_dispatched('inbound', connector_name) def clear_dispatched_outbound(self, connector_name=None): return self._clear_dispatched('outbound', connector_name) def clear_dispatched_failures(self, connector_name=None): return self._clear_dispatched('failures', connector_name) def _dispatch(self, message, rkey, exchange='vumi'): self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery() def dispatch_inbound(self, message, connector_name=None): rkey = self._rkey('inbound', connector_name) return self._dispatch(message, rkey) def dispatch_outbound(self, message, connector_name=None): rkey = self._rkey('outbound', connector_name) return self._dispatch(message, rkey) def dispatch_event(self, message, connector_name=None): rkey = self._rkey('event', connector_name) return self._dispatch(message, rkey) def dispatch_failure(self, message, connector_name=None): rkey = self._rkey('failure', connector_name) return self._dispatch(message, rkey)
class ApplicationTestCase(TestCase): """ This is a base class for testing application workers. """ # base timeout of 5s for all application tests timeout = 5 transport_name = "sphex" transport_type = None application_class = None def setUp(self): self._workers = [] self._amqp = FakeAMQPBroker() def tearDown(self): for worker in self._workers: worker.stopWorker() def rkey(self, name): return "%s.%s" % (self.transport_name, name) @inlineCallbacks def get_application(self, config, cls=None, start=True): """ Get an instance of a worker class. :param config: Config dict. :param cls: The Application class to instantiate. Defaults to :attr:`application_class` :param start: True to start the application (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``transport_name`` defaults to :attr:`self.transport_name` * ``send_to`` defaults to a dictionary with config for each tag defined in worker's SEND_TO_TAGS attribute. Each tag's config contains a transport_name set to ``<tag>_outbound``. """ if cls is None: cls = self.application_class config.setdefault('transport_name', self.transport_name) if 'send_to' not in config and cls.SEND_TO_TAGS: config['send_to'] = {} for tag in cls.SEND_TO_TAGS: config['send_to'][tag] = { 'transport_name': '%s_outbound' % tag} worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def mkmsg_in(self, content='hello world', message_id='abc', to_addr='9292', from_addr='+41791234567', group=None, session_event=None, transport_type=None, helper_metadata=None, transport_metadata=None): if transport_type is None: transport_type = self.transport_type if helper_metadata is None: helper_metadata = {} if transport_metadata is None: transport_metadata = {} return TransportUserMessage( from_addr=from_addr, to_addr=to_addr, group=group, message_id=message_id, transport_name=self.transport_name, transport_type=transport_type, transport_metadata=transport_metadata, helper_metadata=helper_metadata, content=content, session_event=session_event, timestamp=datetime.now(), ) def mkmsg_out(self, content='hello world', message_id='1', to_addr='+41791234567', from_addr='9292', group=None, session_event=None, in_reply_to=None, transport_type=None, transport_metadata=None, ): if transport_type is None: transport_type = self.transport_type if transport_metadata is None: transport_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, group=group, message_id=message_id, transport_name=self.transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, ) return TransportUserMessage(**params) def mkmsg_ack(self, user_message_id='1', sent_message_id='abc', transport_metadata=None): if transport_metadata is None: transport_metadata = {} return TransportEvent( event_type='ack', user_message_id=user_message_id, sent_message_id=sent_message_id, transport_name=self.transport_name, transport_metadata=transport_metadata, ) def mkmsg_delivery(self, status='delivered', user_message_id='abc', transport_metadata=None): if transport_metadata is None: transport_metadata = {} return TransportEvent( event_type='delivery_report', transport_name=self.transport_name, user_message_id=user_message_id, delivery_status=status, to_addr='+41791234567', transport_metadata=transport_metadata, ) def get_dispatched_messages(self): return self._amqp.get_messages('vumi', self.rkey('outbound')) def wait_for_dispatched_messages(self, amount): return self._amqp.wait_messages('vumi', self.rkey('outbound'), amount) def dispatch(self, message, rkey=None, exchange='vumi'): if rkey is None: rkey = self.rkey('inbound') self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery()
class Vas2NetsFailureWorkerTestCase(unittest.TestCase): @inlineCallbacks def setUp(self): self.today = datetime.utcnow().date() self.port = 9999 self.path = '/api/v1/sms/vas2nets/receive/' self.config = { 'transport_name': 'vas2nets', 'url': 'http://localhost:%s%s' % (self.port, self.path), 'username': '******', 'password': '******', 'owner': 'owner', 'service': 'service', 'subservice': 'subservice', 'web_receive_path': '/receive', 'web_receipt_path': '/receipt', 'web_port': 9998, } self.fail_config = { 'transport_name': 'vas2nets', 'retry_routing_key': '%(transport_name)s.outbound', 'failures_routing_key': '%(transport_name)s.failures', } self.workers = [] self.broker = FakeAMQPBroker() self.redis = FakeRedis() self.worker = yield self.mk_transport_worker(self.config, self.broker) self.fail_worker = yield self.mk_failure_worker( self.fail_config, self.broker, self.redis) def tearDown(self): for worker in self.workers: worker.stopWorker() @inlineCallbacks def mk_transport_worker(self, config, broker): worker = get_stubbed_worker(Vas2NetsTransport, config, broker) self.workers.append(worker) yield worker.startWorker() returnValue(worker) @inlineCallbacks def mk_failure_worker(self, config, broker, redis): w = get_stubbed_worker(FailureWorker, config, broker) self.workers.append(w) w.retry_publisher = yield self.worker.publish_to("foo") w.r_server = redis yield w.startWorker() returnValue(w) @inlineCallbacks def mk_resource_worker(self, body, headers=None, code=http.OK): w = get_stubbed_worker(TestResourceWorker, {}, self.broker) self.workers.append(w) w.set_resources([(self.path, BadVas2NetsResource, (body, headers, code))]) yield w.startWorker() returnValue(w) def get_dispatched(self, rkey): return self.broker.get_dispatched('vumi', rkey) def get_retry_keys(self): timestamps = self.redis.zrange( self.fail_worker._retry_timestamps_key, 0, 0) retry_keys = set() for timestamp in timestamps: bucket_key = self.fail_worker.r_key("retry_keys." + timestamp) retry_keys.update(self.redis.smembers(bucket_key)) return retry_keys def mkmsg_out(self, in_reply_to=None): return TransportUserMessage( to_addr='+41791234567', from_addr='9292', message_id='1', transport_name='vas2nets', transport_type='sms', transport_metadata={ 'network_id': 'network-id', }, content='hello world', in_reply_to=in_reply_to, ) def assert_dispatched_count(self, count, routing_key): self.assertEqual(count, len(self.get_dispatched(routing_key))) @inlineCallbacks def test_send_sms_success(self): yield self.mk_resource_worker("Result_code: 00, Message OK") yield self.worker._process_message(self.mkmsg_out()) self.assert_dispatched_count(1, 'vas2nets.event') self.assert_dispatched_count(0, 'vas2nets.failures') @inlineCallbacks def test_send_sms_fail(self): """ A 'No SmsId Header' error should not be retried. """ self.worker.failure_published = FailureCounter(1) yield self.mk_resource_worker("Result_code: 04, Internal system error " "occurred while processing message", {}) yield self.worker._process_message(self.mkmsg_out()) yield self.worker.failure_published.deferred yield self.broker.kick_delivery() self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [twisted_failure] = self.flushLoggedErrors(Vas2NetsTransportError) failure = twisted_failure.value self.assertTrue("No SmsId Header" in str(failure)) [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertTrue( "Vas2NetsTransportError: No SmsId Header" in fmsg['reason']) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set(), self.get_retry_keys()) @inlineCallbacks def test_send_sms_noconn(self): """ A 'connection refused' error should be retried. """ self.worker.failure_published = FailureCounter(1) msg = self.mkmsg_out() yield self.worker._process_message(msg) yield self.worker.failure_published.deferred self.assert_dispatched_count(0, 'vas2nets.event') self.assert_dispatched_count(1, 'vas2nets.failures') [twisted_failure] = self.flushLoggedErrors(TemporaryFailure) failure = twisted_failure.value self.assertTrue("connection refused" in str(failure)) [fmsg] = self.get_dispatched('vas2nets.failures') fmsg = from_json(fmsg.body) self.assertEqual(msg.payload, fmsg['message']) self.assertEqual(FailureMessage.FC_TEMPORARY, fmsg['failure_code']) self.assertTrue(fmsg['reason'].strip().endswith("connection refused")) yield self.broker.kick_delivery() [key] = self.fail_worker.get_failure_keys() self.assertEqual(set([key]), self.get_retry_keys())
class ApplicationTestCase(TestCase): """ This is a base class for testing application workers. """ # base timeout of 5s for all application tests timeout = 5 transport_name = "sphex" transport_type = None application_class = None def setUp(self): self._workers = [] self._amqp = FakeAMQPBroker() def tearDown(self): for worker in self._workers: worker.stopWorker() def rkey(self, name): return "%s.%s" % (self.transport_name, name) @inlineCallbacks def get_application(self, config, cls=None, start=True): """ Get an instance of a worker class. :param config: Config dict. :param cls: The Application class to instantiate. Defaults to :attr:`application_class` :param start: True to start the application (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``transport_name`` defaults to :attr:`self.transport_name` * ``send_to`` defaults to a dictionary with config for each tag defined in worker's SEND_TO_TAGS attribute. Each tag's config contains a transport_name set to ``<tag>_outbound``. """ if cls is None: cls = self.application_class config.setdefault('transport_name', self.transport_name) if 'send_to' not in config and cls.SEND_TO_TAGS: config['send_to'] = {} for tag in cls.SEND_TO_TAGS: config['send_to'][tag] = { 'transport_name': '%s_outbound' % tag } worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def mkmsg_in(self, content='hello world', message_id='abc', to_addr='9292', from_addr='+41791234567', group=None, session_event=None, transport_type=None, helper_metadata=None, transport_metadata=None): if transport_type is None: transport_type = self.transport_type if helper_metadata is None: helper_metadata = {} if transport_metadata is None: transport_metadata = {} return TransportUserMessage( from_addr=from_addr, to_addr=to_addr, group=group, message_id=message_id, transport_name=self.transport_name, transport_type=transport_type, transport_metadata=transport_metadata, helper_metadata=helper_metadata, content=content, session_event=session_event, timestamp=datetime.now(), ) def mkmsg_out( self, content='hello world', message_id='1', to_addr='+41791234567', from_addr='9292', group=None, session_event=None, in_reply_to=None, transport_type=None, transport_metadata=None, ): if transport_type is None: transport_type = self.transport_type if transport_metadata is None: transport_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, group=group, message_id=message_id, transport_name=self.transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, ) return TransportUserMessage(**params) def mkmsg_ack(self, user_message_id='1', sent_message_id='abc', transport_metadata=None): if transport_metadata is None: transport_metadata = {} return TransportEvent( event_type='ack', user_message_id=user_message_id, sent_message_id=sent_message_id, transport_name=self.transport_name, transport_metadata=transport_metadata, ) def mkmsg_delivery(self, status='delivered', user_message_id='abc', transport_metadata=None): if transport_metadata is None: transport_metadata = {} return TransportEvent( event_type='delivery_report', transport_name=self.transport_name, user_message_id=user_message_id, delivery_status=status, to_addr='+41791234567', transport_metadata=transport_metadata, ) def get_dispatched_messages(self): return self._amqp.get_messages('vumi', self.rkey('outbound')) def wait_for_dispatched_messages(self, amount): return self._amqp.wait_messages('vumi', self.rkey('outbound'), amount) def dispatch(self, message, rkey=None, exchange='vumi'): if rkey is None: rkey = self.rkey('inbound') self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery()
class VumiWorkerTestCase(VumiTestCase): """Base test class for vumi workers. This (or a subclass of this) should be the starting point for any test cases that involve vumi workers. """ transport_name = "sphex" transport_type = None MSG_ID_MATCHER = RegexMatcher(r'^[0-9a-fA-F]{32}$') def setUp(self): warnings.warn( "VumiWorkerTestCase and its subclasses are deprecated. " "Use VumiTestCase and other tools from " "vumi.tests.helpers instead.", category=DeprecationWarning) self._workers = [] self._amqp = FakeAMQPBroker() @inlineCallbacks def tearDown(self): yield super(VumiWorkerTestCase, self).tearDown() # Wait for any pending message deliveries to avoid a race with a dirty # reactor. yield self._amqp.wait_delivery() # Now stop all the workers. for worker in self._workers: yield worker.stopWorker() def rkey(self, name): return "%s.%s" % (self.transport_name, name) def _rkey(self, name, connector_name=None): if connector_name is None: return self.rkey(name) return "%s.%s" % (connector_name, name) @inlineCallbacks def get_worker(self, config, cls, start=True): """Create and return an instance of a vumi worker. :param config: Config dict. :param cls: The worker class to instantiate. :param start: True to start the worker (default), False otherwise. """ # When possible, always try and enable heartbeat setup in tests. # so make sure worker_name is set if (config is not None) and ('worker_name' not in config): config['worker_name'] = "unnamed" worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def mkmsg_ack(self, user_message_id='1', sent_message_id='abc', transport_metadata=None, transport_name=None): if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportEvent( event_type='ack', user_message_id=user_message_id, sent_message_id=sent_message_id, transport_name=transport_name, transport_metadata=transport_metadata, ) def mkmsg_nack(self, user_message_id='1', transport_metadata=None, transport_name=None, nack_reason='unknown'): if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportEvent( event_type='nack', nack_reason=nack_reason, user_message_id=user_message_id, transport_name=transport_name, transport_metadata=transport_metadata, ) def mkmsg_delivery(self, status='delivered', user_message_id='abc', transport_metadata=None, transport_name=None): if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportEvent( event_type='delivery_report', transport_name=transport_name, user_message_id=user_message_id, delivery_status=status, to_addr='+41791234567', transport_metadata=transport_metadata, ) def mkmsg_in(self, content='hello world', message_id='abc', to_addr='9292', from_addr='+41791234567', group=None, session_event=None, transport_type=None, helper_metadata=None, transport_metadata=None, transport_name=None): if transport_type is None: transport_type = self.transport_type if helper_metadata is None: helper_metadata = {} if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name return TransportUserMessage( from_addr=from_addr, to_addr=to_addr, group=group, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, helper_metadata=helper_metadata, content=content, session_event=session_event, timestamp=datetime.now(), ) def mkmsg_out( self, content='hello world', message_id='1', to_addr='+41791234567', from_addr='9292', group=None, session_event=None, in_reply_to=None, transport_type=None, transport_metadata=None, transport_name=None, helper_metadata=None, ): if transport_type is None: transport_type = self.transport_type if transport_metadata is None: transport_metadata = {} if transport_name is None: transport_name = self.transport_name if helper_metadata is None: helper_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, group=group, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, helper_metadata=helper_metadata, ) return TransportUserMessage(**params) def _make_matcher(self, msg, *id_fields): msg['timestamp'] = UTCNearNow() for field in id_fields: msg[field] = self.MSG_ID_MATCHER return msg def _get_dispatched(self, name, connector_name=None): rkey = self._rkey(name, connector_name) return self._amqp.get_messages('vumi', rkey) def _wait_for_dispatched(self, name, amount, connector_name=None): rkey = self._rkey(name, connector_name) return self._amqp.wait_messages('vumi', rkey, amount) def clear_all_dispatched(self): self._amqp.clear_messages('vumi') def _clear_dispatched(self, name, connector_name=None): rkey = self._rkey(name, connector_name) return self._amqp.clear_messages('vumi', rkey) def get_dispatched_events(self, connector_name=None): return self._get_dispatched('event', connector_name) def get_dispatched_inbound(self, connector_name=None): return self._get_dispatched('inbound', connector_name) def get_dispatched_outbound(self, connector_name=None): return self._get_dispatched('outbound', connector_name) def get_dispatched_failures(self, connector_name=None): return self._get_dispatched('failures', connector_name) def wait_for_dispatched_events(self, amount, connector_name=None): return self._wait_for_dispatched('event', amount, connector_name) def wait_for_dispatched_inbound(self, amount, connector_name=None): return self._wait_for_dispatched('inbound', amount, connector_name) def wait_for_dispatched_outbound(self, amount, connector_name=None): return self._wait_for_dispatched('outbound', amount, connector_name) def wait_for_dispatched_failures(self, amount, connector_name=None): return self._wait_for_dispatched('failures', amount, connector_name) def clear_dispatched_events(self, connector_name=None): return self._clear_dispatched('event', connector_name) def clear_dispatched_inbound(self, connector_name=None): return self._clear_dispatched('inbound', connector_name) def clear_dispatched_outbound(self, connector_name=None): return self._clear_dispatched('outbound', connector_name) def clear_dispatched_failures(self, connector_name=None): return self._clear_dispatched('failures', connector_name) def _dispatch(self, message, rkey, exchange='vumi'): self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery() def dispatch_inbound(self, message, connector_name=None): rkey = self._rkey('inbound', connector_name) return self._dispatch(message, rkey) def dispatch_outbound(self, message, connector_name=None): rkey = self._rkey('outbound', connector_name) return self._dispatch(message, rkey) def dispatch_event(self, message, connector_name=None): rkey = self._rkey('event', connector_name) return self._dispatch(message, rkey) def dispatch_failure(self, message, connector_name=None): rkey = self._rkey('failure', connector_name) return self._dispatch(message, rkey)
class DispatcherTestCase(TestCase): """ This is a base class for testing dispatcher workers. """ # base timeout of 5s for all dispatcher tests timeout = 5 dispatcher_name = "sphex_dispatcher" dispatcher_class = None def setUp(self): self._workers = [] self._amqp = FakeAMQPBroker() def tearDown(self): for worker in self._workers: worker.stopWorker() @inlineCallbacks def get_dispatcher(self, config, cls=None, start=True): """ Get an instance of a dispatcher class. :param config: Config dict. :param cls: The Dispatcher class to instantiate. Defaults to :attr:`dispatcher_class` :param start: True to start the displatcher (default), False otherwise. Some default config values are helpfully provided in the interests of reducing boilerplate: * ``dispatcher_name`` defaults to :attr:`self.dispatcher_name` """ if cls is None: cls = self.dispatcher_class config.setdefault('dispatcher_name', self.dispatcher_name) worker = get_stubbed_worker(cls, config, self._amqp) self._workers.append(worker) if start: yield worker.startWorker() returnValue(worker) def mkmsg_in(self, content='hello world', message_id='abc', to_addr='9292', from_addr='+41791234567', session_event=None, transport_type=None, helper_metadata=None, transport_metadata=None, transport_name=None): if helper_metadata is None: helper_metadata = {} if transport_metadata is None: transport_metadata = {} return TransportUserMessage( from_addr=from_addr, to_addr=to_addr, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, helper_metadata=helper_metadata, content=content, session_event=session_event, timestamp=datetime.now(), ) def mkmsg_out(self, content='hello world', message_id='1', to_addr='+41791234567', from_addr='9292', session_event=None, in_reply_to=None, transport_type=None, transport_metadata=None, transport_name=None): if transport_metadata is None: transport_metadata = {} params = dict( to_addr=to_addr, from_addr=from_addr, message_id=message_id, transport_name=transport_name, transport_type=transport_type, transport_metadata=transport_metadata, content=content, session_event=session_event, in_reply_to=in_reply_to, ) return TransportUserMessage(**params) def mkmsg_ack(self, event_type='ack', user_message_id='1', send_message_id='abc', transport_name=None, transport_metadata=None): if transport_metadata is None: transport_metadata = {} params = dict( event_type=event_type, user_message_id=user_message_id, sent_message_id=send_message_id, transport_name=transport_name, transport_metadata=transport_metadata, ) return TransportEvent(**params) def get_dispatched_messages(self, transport_name, direction='outbound'): return self._amqp.get_messages('vumi', '%s.%s' % (transport_name, direction)) def wait_for_dispatched_messages(self, transport_name, amount, direction='outbound'): return self._amqp.wait_messages('vumi', '%s.%s' % (transport_name, direction), amount) def dispatch(self, message, transport_name, direction='inbound', exchange='vumi'): rkey = '%s.%s' % (transport_name, direction) self._amqp.publish_message(exchange, rkey, message) return self._amqp.kick_delivery()