def setUp(self):
self.workerhelper = WorkerHelper()
self.addCleanup(self.workerhelper.cleanup)
self.persistencehelper = PersistenceHelper()
yield self.persistencehelper.setup()
self.addCleanup(self.persistencehelper.cleanup)
self.messagehelper = MessageHelper()
self.addCleanup(self.messagehelper.cleanup)
def __init__(self, dispatcher_class, use_riak=False, **msg_helper_args):
self.dispatcher_class = dispatcher_class
self.worker_helper = WorkerHelper()
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
def __init__(self, application_class, use_riak=False, **msg_helper_args):
self.application_class = application_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(self.msg_helper.transport_name)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
class TestMessageStoreAPI(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
try:
from vumi.components.message_store_api import (
MatchResource, MessageStoreAPIWorker)
except ImportError, e:
import_skip(e, 'riakasaurus', 'riakasaurus.riak')
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
self.match_resource = MatchResource
self.base_path = '/api/v1/'
self.worker = yield self.worker_helper.get_worker(
MessageStoreAPIWorker,
self.persistence_helper.mk_config({
'web_path': self.base_path,
'web_port': 0,
'health_path': '/health/',
}))
self.store = self.worker.store
self.addr = self.worker.webserver.getHost()
self.url = 'http://%s:%s%s' % (self.addr.host, self.addr.port,
self.base_path)
self.tag = ("pool", "tag")
self.batch_id = yield self.store.batch_start([self.tag])
class TestMessageStoreResource(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
try:
from vumi.components.message_store_resource import (
MessageStoreResourceWorker)
except ImportError, e:
import_skip(e, 'riakasaurus', 'riakasaurus.riak')
self.worker_helper = self.add_helper(WorkerHelper())
config = self.persistence_helper.mk_config({
'twisted_endpoint':
'tcp:0',
'web_path':
'/resource_path/',
})
worker = yield self.worker_helper.get_worker(
MessageStoreResourceWorker, config)
yield worker.startService()
port = yield worker.services[0]._waitingForPort
addr = port.getHost()
self.msg_helper = self.add_helper(MessageHelper())
self.url = 'http://%s:%s' % (addr.host, addr.port)
self.store = worker.store
self.addCleanup(self.stop_server, port)
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.worker_helper = self.add_helper(WorkerHelper())
config = {
'deadline': 30,
'redis_manager': {
'key_prefix': 'heartbeats',
'db': 5,
'FAKE_REDIS': True,
},
'monitored_systems': {
'system-1': {
'system_name': 'system-1',
'system_id': 'system-1',
'workers': {
'twitter_transport': {
'name': 'twitter_transport',
'min_procs': 2,
}
}
}
}
}
self.worker = yield self.worker_helper.get_worker(
monitor.HeartBeatMonitor, config, start=False)
def get_worker_helper(self, connector_name=None):
if connector_name not in self._worker_helpers:
worker_helper = WorkerHelper(connector_name, self.broker)
# If this is our first worker helper, we need to grab the broker it
# created. If it isn't, its broker will be self.broker anyway.
self.broker = worker_helper.broker
self._worker_helpers[connector_name] = worker_helper
return self._worker_helpers[connector_name]
def test_publish_metric(self):
datapoint = ("vumi.test.v1", 1.0, 1234)
client = WorkerHelper.get_fake_amqp_client(self.worker_helper.broker)
channel = yield client.get_channel()
pub = metrics_workers.GraphitePublisher()
pub.start(channel)
pub.publish_metric(*datapoint)
self._check_msg(channel, *datapoint)
def test_publish_metric(self):
datapoint = ("vumi.test.v1", 1.0, 1234)
client = WorkerHelper.get_fake_amqp_client(self.worker_helper.broker)
channel = yield client.get_channel()
pub = metrics_workers.GraphitePublisher()
pub.start(channel)
pub.publish_metric(*datapoint)
self._check_msg(channel, *datapoint)
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
self.worker_helper = self.add_helper(WorkerHelper())
self.msg_helper = self.add_helper(MessageHelper())
riak, redis = yield self.create_managers()
self.operational_store = OperationalMessageStore(riak, redis)
self.batch_manager = MessageStoreBatchManager(riak, redis)
def __init__(self, transport_class, use_riak=False, **msg_helper_args):
self.transport_class = transport_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(connector_name=self.transport_name,
status_connector_name="%s.status" %
(self.transport_name, ))
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def test_subclassing_api(self):
worker = WorkerHelper.get_worker_raw(ApplicationWorker, {"transport_name": "test"})
worker.consume_ack(self.app_helper.make_ack())
worker.consume_nack(self.app_helper.make_nack())
worker.consume_delivery_report(self.app_helper.make_delivery_report())
worker.consume_unknown_event(self.app_helper.make_inbound("foo"))
worker.consume_user_message(self.app_helper.make_inbound("foo"))
worker.new_session(self.app_helper.make_inbound("foo"))
worker.close_session(self.app_helper.make_inbound("foo"))
def test_subclassing_api(self):
worker = WorkerHelper.get_worker_raw(ApplicationWorker,
{'transport_name': 'test'})
worker.consume_ack(self.app_helper.make_ack())
worker.consume_nack(self.app_helper.make_nack())
worker.consume_delivery_report(self.app_helper.make_delivery_report())
worker.consume_unknown_event(self.app_helper.make_inbound("foo"))
worker.consume_user_message(self.app_helper.make_inbound("foo"))
worker.new_session(self.app_helper.make_inbound("foo"))
worker.close_session(self.app_helper.make_inbound("foo"))
def setUp(self):
self.worker_helper = self.add_helper(WorkerHelper())
self.worker = yield self.worker_helper.get_worker(
SimpleDispatcher, {
"route_mappings": {
"test1.inbound": ["test2.outbound", "test3.outbound"],
"test2.inbound": ["test1.outbound"],
"test3.inbound": ["test1.outbound"],
},
})
def test_publish_heartbeat(self):
broker = FakeAMQPBroker()
client = WorkerHelper.get_fake_amqp_client(broker)
channel = yield client.get_channel()
pub = MockHeartBeatPublisher(self.gen_fake_attrs)
pub.start(channel)
pub._beat()
[msg] = broker.get_dispatched("vumi.health", "heartbeat.inbound")
self.assertEqual(json.loads(msg.body), self.gen_fake_attrs())
def setUp(self):
self.workerhelper = WorkerHelper()
self.addCleanup(self.workerhelper.cleanup)
self.persistencehelper = PersistenceHelper()
yield self.persistencehelper.setup()
self.addCleanup(self.persistencehelper.cleanup)
self.messagehelper = MessageHelper()
self.addCleanup(self.messagehelper.cleanup)
def test_publish_heartbeat(self):
broker = FakeAMQPBroker()
client = WorkerHelper.get_fake_amqp_client(broker)
channel = yield client.get_channel()
pub = MockHeartBeatPublisher(self.gen_fake_attrs)
pub.start(channel)
pub._beat()
[msg] = broker.get_dispatched("vumi.health", "heartbeat.inbound")
self.assertEqual(json.loads(msg.body), self.gen_fake_attrs())
def __init__(self, dispatcher_class, use_riak=False, **msg_helper_args):
self.dispatcher_class = dispatcher_class
self.worker_helper = WorkerHelper()
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper, self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
def setUp(self):
self.worker_helper = self.add_helper(WorkerHelper())
self.broker = BrokerWrapper(self.worker_helper.broker)
self.udp_protocol = UDPMetricsCatcher()
self.udp_server = yield reactor.listenUDP(0, self.udp_protocol)
self.add_cleanup(self.udp_server.stopListening)
self.worker = yield self.worker_helper.get_worker(
metrics_workers.UDPMetricsCollector, {
'metrics_host': '127.0.0.1',
'metrics_port': self.udp_server.getHost().port,
})
def test_start_publisher(self):
"""The publisher should publish"""
worker = WorkerHelper.get_worker_raw(Worker, {})
publisher = yield worker.publish_to('test.routing.key')
self.assertEquals(publisher.routing_key, 'test.routing.key')
publisher.publish_message(Message(key="value"))
[published_msg] = publisher.channel.broker.get_dispatched(
'vumi', 'test.routing.key')
self.assertEquals(published_msg.body, '{"key": "value"}')
self.assertEquals(published_msg.properties, {'delivery mode': 2})
def test_start_publisher(self):
"""The publisher should publish"""
worker = WorkerHelper.get_worker_raw(Worker, {})
publisher = yield worker.publish_to('test.routing.key')
self.assertEquals(publisher.routing_key, 'test.routing.key')
publisher.publish_message(Message(key="value"))
[published_msg
] = publisher.channel.broker.get_dispatched('vumi',
'test.routing.key')
self.assertEquals(published_msg.body, '{"key": "value"}')
self.assertEquals(published_msg.properties, {'delivery mode': 2})
def __init__(self, dispatcher_helper, connector_name):
self.msg_helper = dispatcher_helper.msg_helper
self.worker_helper = WorkerHelper(
connector_name, dispatcher_helper.worker_helper.broker)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
# We don't want to be able to make workers with this helper.
del self.get_worker
del self.cleanup_worker
def setUp(self):
self.worker_helper = self.add_helper(WorkerHelper())
self.config = {
'web_port': 9999,
'web_receive_path': '/t/receive',
'web_receipt_path': '/t/receipt',
'url': 'http://127.0.0.1:9998/t/send',
}
self.worker = yield self.worker_helper.get_worker(
StubbedFakeVas2NetsWorker, self.config, start=False)
self.test_worker = yield self.worker_helper.get_worker(TestWorker,
self.config,
start=False)
self.today = datetime.utcnow().date()
def __init__(self, application_class, use_riak=False, **msg_helper_args):
self.application_class = application_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(self.msg_helper.transport_name)
self.dispatch_helper = MessageDispatchHelper(
self.msg_helper, self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def make_worker(self, retry_delivery_period=0):
self.worker_helper = self.add_helper(WorkerHelper('sphex'))
config = self.persistence_helper.mk_config({
'transport_name':
'sphex',
'retry_routing_key':
'sms.outbound.%(transport_name)s',
'failures_routing_key':
'sms.failures.%(transport_name)s',
'retry_delivery_period':
retry_delivery_period,
})
self.worker = yield self.worker_helper.get_worker(
FailureWorker, config)
self.redis = self.worker.redis
yield self.redis._purge_all() # Just in case
def __init__(self, transport_class, use_riak=False, **msg_helper_args):
self.transport_class = transport_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(
connector_name=self.transport_name,
status_connector_name="%s.status" % (self.transport_name,))
self.dispatch_helper = MessageDispatchHelper(
self.msg_helper, self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def setUp(self):
self.now = 0
self.worker_helper = self.add_helper(WorkerHelper())
self.broker = BrokerWrapper(self.worker_helper.broker)
def setUp(self):
self.worker_helper = self.add_helper(WorkerHelper())
self.broker = BrokerWrapper(self.worker_helper.broker)
self._on_run = Deferred()
self.add_cleanup(lambda: self._on_run.callback(None))
def setUp(self):
self.worker_helper = self.add_helper(WorkerHelper())
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.worker_helper = self.add_helper(WorkerHelper())
class DispatcherHelper(object):
"""
Test helper for dispatcher workers.
This helper construct and wraps several lower-level helpers and provides
higher-level functionality for dispatcher tests.
:param dispatcher_class:
The worker class for the dispatcher being tested.
:param bool use_riak:
Set to ``True`` if the test requires Riak. This is passed to the
underlying :class:`~vumi.tests.helpers.PersistenceHelper`.
:param \**msg_helper_args:
All other keyword params are passed to the underlying
:class:`~vumi.tests.helpers.MessageHelper`.
"""
implements(IHelper)
def __init__(self, dispatcher_class, use_riak=False, **msg_helper_args):
self.dispatcher_class = dispatcher_class
self.worker_helper = WorkerHelper()
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper, self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
def setup(self):
self.persistence_helper.setup()
self.worker_helper.setup()
@inlineCallbacks
def cleanup(self):
yield self.worker_helper.cleanup()
yield self.persistence_helper.cleanup()
def get_dispatcher(self, config, cls=None, start=True):
"""
Get an instance of a dispatcher class.
:param dict config: Config dict.
:param cls:
The transport class to instantiate. Defaults to
:attr:`dispatcher_class`
:param bool start:
``True`` to start the dispatcher (default), ``False`` otherwise.
"""
if cls is None:
cls = self.dispatcher_class
config = self.persistence_helper.mk_config(config)
return self.get_worker(cls, config, start)
def get_connector_helper(self, connector_name):
"""
Construct a :class:`~DispatcherConnectorHelper` for the provided
``connector_name``.
"""
return DispatcherConnectorHelper(self, connector_name)
class TransportHelper(object):
"""
Test helper for transport workers.
This helper construct and wraps several lower-level helpers and provides
higher-level functionality for transport tests.
:param transport_class:
The worker class for the transport being tested.
:param bool use_riak:
Set to ``True`` if the test requires Riak. This is passed to the
underlying :class:`~vumi.tests.helpers.PersistenceHelper`.
:param \**msg_helper_args:
All other keyword params are passed to the underlying
:class:`~vumi.tests.helpers.MessageHelper`.
"""
implements(IHelper)
def __init__(self, transport_class, use_riak=False, **msg_helper_args):
self.transport_class = transport_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(self.transport_name)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def setup(self):
self.persistence_helper.setup()
self.worker_helper.setup()
@inlineCallbacks
def cleanup(self):
yield self.worker_helper.cleanup()
yield self.persistence_helper.cleanup()
def get_transport(self, config, cls=None, start=True):
"""
Get an instance of a transport class.
:param config: Config dict.
:param cls: The transport class to instantiate.
Defaults to :attr:`transport_class`
:param start: True to start the transport (default), False otherwise.
Some default config values are helpfully provided in the
interests of reducing boilerplate:
* ``transport_name`` defaults to :attr:`self.transport_name`
"""
if cls is None:
cls = self.transport_class
config = self.mk_config(config)
config.setdefault('transport_name', self.transport_name)
return self.get_worker(cls, config, start)
def get_dispatched_failures(self, connector_name=None):
"""
Get failures dispatched by a transport.
:param str connector_name:
Connector name. If ``None``, the default connector name for the
helper instance will be used.
:returns:
A list of :class:`~vumi.transports.failures.FailureMessage`
instances.
"""
return self.get_dispatched(connector_name, 'failures', FailureMessage)
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
self.clear_events()
self.add_cleanup(self.clear_events)
class TestRouter(JunebugTestBase):
DEFAULT_ROUTER_WORKER_CONFIG = {
'inbound_ttl': 60,
'outbound_ttl': 60 * 60 * 24 * 2,
'metric_window': 1.0,
'destinations': [],
}
@inlineCallbacks
def setUp(self):
yield self.start_server()
self.workerhelper = WorkerHelper()
self.addCleanup(self.workerhelper.cleanup)
self.persistencehelper = PersistenceHelper()
yield self.persistencehelper.setup()
self.addCleanup(self.persistencehelper.cleanup)
self.messagehelper = MessageHelper()
self.addCleanup(self.messagehelper.cleanup)
@inlineCallbacks
def get_router_worker(self, config=None):
if config is None:
config = {}
config = conjoin(
self.persistencehelper.mk_config(
self.DEFAULT_ROUTER_WORKER_CONFIG), config)
FromAddressRouter._create_worker = self.workerhelper.get_worker
worker = yield self.workerhelper.get_worker(FromAddressRouter, config)
returnValue(worker)
@inlineCallbacks
def test_validate_router_config_invalid_channel_uuid(self):
"""
If the provided channel UUID is not a valid UUID a config error should
be raised
"""
with self.assertRaises(InvalidRouterConfig) as e:
yield FromAddressRouter.validate_router_config(
self.api, {'channel': "bad-uuid"})
self.assertEqual(e.exception.message,
"Field 'channel' is not a valid UUID")
@inlineCallbacks
def test_validate_router_config_missing_channel(self):
"""
If the provided channel UUID is not for an existing channel, a config
error should be raised
"""
channel_id = str(uuid.uuid4())
with self.assertRaises(InvalidRouterConfig) as e:
yield FromAddressRouter.validate_router_config(
self.api, {'channel': channel_id})
self.assertEqual(e.exception.message,
"Channel {} does not exist".format(channel_id))
@inlineCallbacks
def test_validate_router_config_existing_destination(self):
"""
If the specified channel already has a destination specified, then
a config error should be raised
"""
channel = yield self.create_channel(self.api.service, self.redis)
with self.assertRaises(InvalidRouterConfig) as e:
yield FromAddressRouter.validate_router_config(
self.api, {'channel': channel.id})
self.assertEqual(
e.exception.message,
"Channel {} already has a destination specified".format(
channel.id))
@inlineCallbacks
def test_validate_router_config_existing_router(self):
"""
If an existing router is already listening to the specified channel,
then a config error should be raised
"""
channel = yield self.create_channel(self.api.service,
self.redis,
properties={
'type': 'telnet',
'config': {
'twisted_endpoint':
'tcp:0',
},
})
config = self.create_router_config(config={
'test': 'pass',
'channel': channel.id
})
router = Router(self.api, config)
yield router.save()
router.start(self.api.service)
with self.assertRaises(InvalidRouterConfig) as e:
yield FromAddressRouter.validate_router_config(
self.api, {'channel': channel.id})
self.assertEqual(
e.exception.message,
"Router {} is already routing channel {}".format(
router.id, channel.id))
@inlineCallbacks
def test_validate_router_destination_config_invalid_regex(self):
"""
If invalid regex is passed into the regex field, a config error should
be raised
"""
with self.assertRaises(InvalidRouterDestinationConfig) as e:
yield FromAddressRouter.validate_destination_config(
self.api, {'regular_expression': "("})
self.assertEqual(
e.exception.message,
"Field 'regular_expression' is not a valid regular expression: "
"unbalanced parenthesis")
@inlineCallbacks
def test_validate_router_destination_config_missing_field(self):
"""
regular_expression should be a required field
"""
with self.assertRaises(InvalidRouterDestinationConfig) as e:
yield FromAddressRouter.validate_destination_config(self.api, {})
self.assertEqual(e.exception.message,
"Missing required config field 'regular_expression'")
@inlineCallbacks
def test_inbound_message_routing(self):
"""
Inbound messages should be routed to the correct destination worker(s)
"""
yield self.get_router_worker({
'destinations': [{
'id': "test-destination1",
'amqp_queue': "testqueue1",
'config': {
'regular_expression': '^1.*$'
},
}, {
'id': "test-destination2",
'amqp_queue': "testqueue2",
'config': {
'regular_expression': '^2.*$'
},
}, {
'id': "test-destination3",
'amqp_queue': "testqueue3",
'config': {
'regular_expression': '^2.*$'
},
}],
'channel':
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14',
})
inbound = self.messagehelper.make_inbound('test message',
to_addr='1234')
yield self.workerhelper.dispatch_inbound(
inbound, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[message] = yield self.workerhelper.wait_for_dispatched_inbound(
connector_name='testqueue1')
self.assertEqual(inbound, message)
inbound = self.messagehelper.make_inbound('test message',
to_addr='2234')
yield self.workerhelper.dispatch_inbound(
inbound, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[message] = yield self.workerhelper.wait_for_dispatched_inbound(
connector_name='testqueue2')
self.assertEqual(inbound, message)
[message] = yield self.workerhelper.wait_for_dispatched_inbound(
connector_name='testqueue3')
self.assertEqual(inbound, message)
@inlineCallbacks
def test_inbound_message_routing_no_to_addr(self):
"""
If an inbound message doesn't have a to address, then an error should
be logged
"""
yield self.get_router_worker({
'channel':
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14',
})
logs = []
log.addObserver(logs.append)
inbound = self.messagehelper.make_inbound('test message', to_addr=None)
yield self.workerhelper.dispatch_inbound(
inbound, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[error_log] = logs
self.assertIn("Message has no to address, cannot route message: ",
error_log['log_text'])
@inlineCallbacks
def test_inbound_event_routing(self):
"""
Inbound events should be routed to the correct destination worker(s)
"""
yield self.get_router_worker({
'destinations': [{
'id': "test-destination1",
'amqp_queue': "testqueue1",
'config': {
'regular_expression': '^1.*$'
},
}, {
'id': "test-destination2",
'amqp_queue': "testqueue2",
'config': {
'regular_expression': '^2.*$'
},
}, {
'id': "test-destination3",
'amqp_queue': "testqueue3",
'config': {
'regular_expression': '^2.*$'
},
}],
'channel':
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14',
})
outbound = self.messagehelper.make_outbound("test message",
from_addr="1234")
yield self.workerhelper.dispatch_outbound(outbound, 'testqueue1')
ack = self.messagehelper.make_ack(outbound)
yield self.workerhelper.dispatch_event(
ack, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[event] = yield self.workerhelper.wait_for_dispatched_events(
connector_name='testqueue1')
self.assertEqual(ack, event)
outbound = self.messagehelper.make_outbound("test message",
from_addr="2234")
yield self.workerhelper.dispatch_outbound(outbound, 'testqueue2')
ack = self.messagehelper.make_ack(outbound)
yield self.workerhelper.dispatch_event(
ack, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[event] = yield self.workerhelper.wait_for_dispatched_events(
connector_name='testqueue2')
self.assertEqual(ack, event)
[event] = yield self.workerhelper.wait_for_dispatched_events(
connector_name='testqueue3')
self.assertEqual(ack, event)
@inlineCallbacks
def test_inbound_event_routing_no_inbound_message(self):
"""
If no message can be found in the message store for the event, then an
error message should be logged
"""
yield self.get_router_worker({
'destinations': [{
'id': "test-destination1",
'amqp_queue': "testqueue1",
'config': {
'regular_expression': '^1.*$'
},
}],
'channel':
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14',
})
logs = []
log.addObserver(logs.append)
ack = self.messagehelper.make_ack()
yield self.workerhelper.dispatch_event(
ack, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[error_log] = logs
self.assertIn("Cannot find message", error_log['log_text'])
self.assertIn("for event, not routing event: ", error_log['log_text'])
@inlineCallbacks
def test_inbound_event_routing_no_from_address(self):
"""
If the message for an event doesn't have a from address, then an error
message should be logged
"""
yield self.get_router_worker({
'destinations': [{
'id': "test-destination1",
'amqp_queue': "testqueue1",
'config': {
'regular_expression': '^1.*$'
},
}],
'channel':
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14',
})
logs = []
log.addObserver(logs.append)
outbound = self.messagehelper.make_outbound("test message",
from_addr=None)
yield self.workerhelper.dispatch_outbound(outbound, 'testqueue1')
ack = self.messagehelper.make_ack(outbound)
yield self.workerhelper.dispatch_event(
ack, '41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
[error_log] = logs
self.assertIn("Message has no from address, cannot route event: ",
error_log['log_text'])
@inlineCallbacks
def test_outbound_message_routing(self):
"""
Outbound messages should be routed to the configured channel, no matter
which destination they came from. They should also be stored so that
events can be routed correctly.
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': "test-destination1",
'amqp_queue': "testqueue1",
'config': {
'regular_expression': '^1.*$'
},
}, {
'id': "test-destination2",
'amqp_queue': "testqueue2",
'config': {
'regular_expression': '^2.*$'
},
}],
'channel':
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14',
})
outbound = self.messagehelper.make_outbound('test message')
yield self.workerhelper.dispatch_outbound(outbound, 'testqueue1')
[message] = yield self.workerhelper.wait_for_dispatched_outbound(
connector_name='41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
self.assertEqual(outbound, message)
stored_message = yield worker.outbounds.load_message(
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14', outbound['message_id'])
self.assertEqual(api_from_message(outbound), stored_message)
yield self.workerhelper.clear_dispatched_outbound(
connector_name='41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
outbound = self.messagehelper.make_outbound('test message')
yield self.workerhelper.dispatch_outbound(outbound, 'testqueue2')
[message] = yield self.workerhelper.wait_for_dispatched_outbound(
connector_name='41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14')
self.assertEqual(outbound, message)
stored_message = yield worker.outbounds.load_message(
'41e58f4a-2acc-442f-b3e5-3cf2b2f1cf14', outbound['message_id'])
self.assertEqual(api_from_message(outbound), stored_message)
class TestBaseRouterWorker(VumiTestCase, JunebugTestBase):
DEFAULT_ROUTER_WORKER_CONFIG = {
'inbound_ttl': 60,
'outbound_ttl': 60 * 60 * 24 * 2,
'metric_window': 1.0,
'destinations': [],
}
@inlineCallbacks
def setUp(self):
self.workerhelper = WorkerHelper()
self.addCleanup(self.workerhelper.cleanup)
self.persistencehelper = PersistenceHelper()
yield self.persistencehelper.setup()
self.addCleanup(self.persistencehelper.cleanup)
self.messagehelper = MessageHelper()
self.addCleanup(self.messagehelper.cleanup)
@inlineCallbacks
def get_router_worker(self, config=None):
if config is None:
config = {}
config = conjoin(
self.persistencehelper.mk_config(
self.DEFAULT_ROUTER_WORKER_CONFIG), config)
TestRouter._create_worker = self.workerhelper.get_worker
worker = yield self.workerhelper.get_worker(TestRouter, config)
returnValue(worker)
@inlineCallbacks
def test_start_router_worker_no_destinations(self):
"""
If there are no destinations specified, no workers should be started.
The setup_router function should be called on the implementation.
"""
worker = yield self.get_router_worker()
self.assertEqual(len(worker.namedServices), 0)
self.assertTrue(worker.setup_called)
@inlineCallbacks
def test_start_router_with_destinations(self):
"""
If there are destinations specified, then a worker should be started
for every destination.
"""
worker = yield self.get_router_worker({
'destinations': [
{
'id': 'test-destination1',
},
{
'id': 'test-destination2',
},
],
})
self.assertTrue(worker.setup_called)
self.assertEqual(sorted(worker.namedServices.keys()),
['test-destination1', 'test-destination2'])
for connector in worker.connectors.values():
self.assertFalse(connector.paused)
@inlineCallbacks
def test_teardown_router(self):
"""
Tearing down a router should pause all connectors, and call the
teardown method of the router implementation
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination1'
}],
})
self.assertFalse(worker.teardown_called)
for connector in worker.connectors.values():
self.assertFalse(connector.paused)
yield worker.teardown_worker()
self.assertTrue(worker.teardown_called)
for connector in worker.connectors.values():
self.assertTrue(connector.paused)
@inlineCallbacks
def test_consume_channel(self):
"""
consume_channel should set up the appropriate connector, as well as
attach the specified callbacks for messages and events.
"""
worker = yield self.get_router_worker({})
messages = []
events = []
def message_callback(channelid, message):
assert channelid == 'testchannel'
messages.append(message)
def event_callback(channelid, event):
assert channelid == 'testchannel'
events.append(event)
yield worker.consume_channel('testchannel', message_callback,
event_callback)
# Because this is only called in setup, and we're creating connectors
# after setup, we need to unpause them
worker.unpause_connectors()
self.assertEqual(messages, [])
inbound = self.messagehelper.make_inbound('test message')
yield self.workerhelper.dispatch_inbound(inbound, 'testchannel')
self.assertEqual(messages, [inbound])
self.assertEqual(events, [])
event = self.messagehelper.make_ack()
yield self.workerhelper.dispatch_event(event, 'testchannel')
self.assertEqual(events, [event])
@inlineCallbacks
def test_send_inbound_to_destination(self):
"""
send_inbound_to_destination should send the provided inbound message
to the specified destination worker
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination',
'amqp_queue': 'testqueue',
}],
})
inbound = self.messagehelper.make_inbound('test_message')
yield worker.send_inbound_to_destination('test-destination', inbound)
[message] = yield self.workerhelper.wait_for_dispatched_inbound(
connector_name='testqueue')
self.assertEqual(message, inbound)
@inlineCallbacks
def test_send_event_to_destination(self):
"""
send_event_to_destination should send the provided event message
to the specified destination worker
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination',
'amqp_queue': 'testqueue',
}],
})
ack = self.messagehelper.make_ack()
yield worker.send_event_to_destination('test-destination', ack)
[event] = yield self.workerhelper.wait_for_dispatched_events(
connector_name='testqueue')
self.assertEqual(event, ack)
@inlineCallbacks
def test_consume_destination(self):
"""
If a callback is attached to a destination, then that callback should
be called when an outbound is sent from a destination
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination',
'amqp_queue': 'testqueue',
}],
})
messages = []
def message_callback(destinationid, message):
assert destinationid == 'test-destination'
messages.append(message)
yield worker.consume_destination('test-destination', message_callback)
# Because this is only called in setup, and we're creating connectors
# after setup, we need to unpause them
worker.unpause_connectors()
self.assertEqual(messages, [])
msg = self.messagehelper.make_outbound('testmessage')
yield self.workerhelper.dispatch_outbound(msg, 'test-destination')
self.assertEqual(messages, [msg])
@inlineCallbacks
def test_send_outbound_to_channel(self):
"""
send_outbound_to_channel should send the provided outbound message to
the specified channel
"""
worker = yield self.get_router_worker({})
yield worker.consume_channel('testchannel', lambda m: m, lambda e: e)
outbound = self.messagehelper.make_outbound('test message')
yield worker.send_outbound_to_channel('testchannel', outbound)
[message] = yield self.workerhelper.wait_for_dispatched_outbound(
connector_name='testchannel')
self.assertEqual(message, outbound)
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
class DispatcherHelper(object):
"""
Test helper for dispatcher workers.
This helper construct and wraps several lower-level helpers and provides
higher-level functionality for dispatcher tests.
:param dispatcher_class:
The worker class for the dispatcher being tested.
:param bool use_riak:
Set to ``True`` if the test requires Riak. This is passed to the
underlying :class:`~vumi.tests.helpers.PersistenceHelper`.
:param \**msg_helper_args:
All other keyword params are passed to the underlying
:class:`~vumi.tests.helpers.MessageHelper`.
"""
implements(IHelper)
def __init__(self, dispatcher_class, use_riak=False, **msg_helper_args):
self.dispatcher_class = dispatcher_class
self.worker_helper = WorkerHelper()
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
def setup(self):
self.persistence_helper.setup()
self.worker_helper.setup()
@inlineCallbacks
def cleanup(self):
yield self.worker_helper.cleanup()
yield self.persistence_helper.cleanup()
def get_dispatcher(self, config, cls=None, start=True):
"""
Get an instance of a dispatcher class.
:param dict config: Config dict.
:param cls:
The transport class to instantiate. Defaults to
:attr:`dispatcher_class`
:param bool start:
``True`` to start the dispatcher (default), ``False`` otherwise.
"""
if cls is None:
cls = self.dispatcher_class
config = self.persistence_helper.mk_config(config)
return self.get_worker(cls, config, start)
def get_connector_helper(self, connector_name):
"""
Construct a :class:`~DispatcherConnectorHelper` for the provided
``connector_name``.
"""
return DispatcherConnectorHelper(self, connector_name)
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
self.worker = yield self.worker_helper.get_worker(
DummyWorker, {}, False)
def start_publisher(self, publisher):
client = WorkerHelper.get_fake_amqp_client(self.worker_helper.broker)
channel = yield client.get_channel()
publisher.start(channel)
class TransportHelper(object):
"""
Test helper for transport workers.
This helper construct and wraps several lower-level helpers and provides
higher-level functionality for transport tests.
:param transport_class:
The worker class for the transport being tested.
:param bool use_riak:
Set to ``True`` if the test requires Riak. This is passed to the
underlying :class:`~vumi.tests.helpers.PersistenceHelper`.
:param \**msg_helper_args:
All other keyword params are passed to the underlying
:class:`~vumi.tests.helpers.MessageHelper`.
"""
implements(IHelper)
def __init__(self, transport_class, use_riak=False, **msg_helper_args):
self.transport_class = transport_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(self.transport_name)
self.dispatch_helper = MessageDispatchHelper(
self.msg_helper, self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def setup(self):
self.persistence_helper.setup()
self.worker_helper.setup()
@inlineCallbacks
def cleanup(self):
yield self.worker_helper.cleanup()
yield self.persistence_helper.cleanup()
def get_transport(self, config, cls=None, start=True):
"""
Get an instance of a transport class.
:param config: Config dict.
:param cls: The transport class to instantiate.
Defaults to :attr:`transport_class`
:param start: True to start the transport (default), False otherwise.
Some default config values are helpfully provided in the
interests of reducing boilerplate:
* ``transport_name`` defaults to :attr:`self.transport_name`
"""
if cls is None:
cls = self.transport_class
config = self.mk_config(config)
config.setdefault('transport_name', self.transport_name)
return self.get_worker(cls, config, start)
def get_dispatched_failures(self, connector_name=None):
"""
Get failures dispatched by a transport.
:param str connector_name:
Connector name. If ``None``, the default connector name for the
helper instance will be used.
:returns:
A list of :class:`~vumi.transports.failures.FailureMessage`
instances.
"""
return self.get_dispatched(connector_name, 'failures', FailureMessage)
def start_manager_as_publisher(self, manager):
client = WorkerHelper.get_fake_amqp_client(self.worker_helper.broker)
channel = yield client.get_channel()
manager.start(channel)
self.add_cleanup(manager.stop)
class TestBaseRouterWorker(VumiTestCase, JunebugTestBase):
DEFAULT_ROUTER_WORKER_CONFIG = {
'inbound_ttl': 60,
'outbound_ttl': 60 * 60 * 24 * 2,
'metric_window': 1.0,
'destinations': [],
}
@inlineCallbacks
def setUp(self):
self.workerhelper = WorkerHelper()
self.addCleanup(self.workerhelper.cleanup)
self.persistencehelper = PersistenceHelper()
yield self.persistencehelper.setup()
self.addCleanup(self.persistencehelper.cleanup)
self.messagehelper = MessageHelper()
self.addCleanup(self.messagehelper.cleanup)
@inlineCallbacks
def get_router_worker(self, config=None):
if config is None:
config = {}
config = conjoin(
self.persistencehelper.mk_config(
self.DEFAULT_ROUTER_WORKER_CONFIG),
config)
TestRouter._create_worker = self.workerhelper.get_worker
worker = yield self.workerhelper.get_worker(TestRouter, config)
returnValue(worker)
@inlineCallbacks
def test_start_router_worker_no_destinations(self):
"""
If there are no destinations specified, no workers should be started.
The setup_router function should be called on the implementation.
"""
worker = yield self.get_router_worker()
self.assertEqual(len(worker.namedServices), 0)
self.assertTrue(worker.setup_called)
@inlineCallbacks
def test_start_router_with_destinations(self):
"""
If there are destinations specified, then a worker should be started
for every destination.
"""
worker = yield self.get_router_worker({
'destinations': [
{
'id': 'test-destination1',
},
{
'id': 'test-destination2',
},
],
})
self.assertTrue(worker.setup_called)
self.assertEqual(sorted(worker.namedServices.keys()), [
'test-destination1', 'test-destination2'])
for connector in worker.connectors.values():
self.assertFalse(connector.paused)
@inlineCallbacks
def test_teardown_router(self):
"""
Tearing down a router should pause all connectors, and call the
teardown method of the router implementation
"""
worker = yield self.get_router_worker({
'destinations': [{'id': 'test-destination1'}],
})
self.assertFalse(worker.teardown_called)
for connector in worker.connectors.values():
self.assertFalse(connector.paused)
yield worker.teardown_worker()
self.assertTrue(worker.teardown_called)
for connector in worker.connectors.values():
self.assertTrue(connector.paused)
@inlineCallbacks
def test_consume_channel(self):
"""
consume_channel should set up the appropriate connector, as well as
attach the specified callbacks for messages and events.
"""
worker = yield self.get_router_worker({})
messages = []
events = []
def message_callback(channelid, message):
assert channelid == 'testchannel'
messages.append(message)
def event_callback(channelid, event):
assert channelid == 'testchannel'
events.append(event)
yield worker.consume_channel(
'testchannel', message_callback, event_callback)
# Because this is only called in setup, and we're creating connectors
# after setup, we need to unpause them
worker.unpause_connectors()
self.assertEqual(messages, [])
inbound = self.messagehelper.make_inbound('test message')
yield self.workerhelper.dispatch_inbound(inbound, 'testchannel')
self.assertEqual(messages, [inbound])
self.assertEqual(events, [])
event = self.messagehelper.make_ack()
yield self.workerhelper.dispatch_event(event, 'testchannel')
self.assertEqual(events, [event])
@inlineCallbacks
def test_send_inbound_to_destination(self):
"""
send_inbound_to_destination should send the provided inbound message
to the specified destination worker
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination',
'amqp_queue': 'testqueue',
}],
})
inbound = self.messagehelper.make_inbound('test_message')
yield worker.send_inbound_to_destination('test-destination', inbound)
[message] = yield self.workerhelper.wait_for_dispatched_inbound(
connector_name='testqueue')
self.assertEqual(message, inbound)
@inlineCallbacks
def test_send_event_to_destination(self):
"""
send_event_to_destination should send the provided event message
to the specified destination worker
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination',
'amqp_queue': 'testqueue',
}],
})
ack = self.messagehelper.make_ack()
yield worker.send_event_to_destination('test-destination', ack)
[event] = yield self.workerhelper.wait_for_dispatched_events(
connector_name='testqueue')
self.assertEqual(event, ack)
@inlineCallbacks
def test_consume_destination(self):
"""
If a callback is attached to a destination, then that callback should
be called when an outbound is sent from a destination
"""
worker = yield self.get_router_worker({
'destinations': [{
'id': 'test-destination',
'amqp_queue': 'testqueue',
}],
})
messages = []
def message_callback(destinationid, message):
assert destinationid == 'test-destination'
messages.append(message)
yield worker.consume_destination('test-destination', message_callback)
# Because this is only called in setup, and we're creating connectors
# after setup, we need to unpause them
worker.unpause_connectors()
self.assertEqual(messages, [])
msg = self.messagehelper.make_outbound('testmessage')
yield self.workerhelper.dispatch_outbound(msg, 'test-destination')
self.assertEqual(messages, [msg])
@inlineCallbacks
def test_send_outbound_to_channel(self):
"""
send_outbound_to_channel should send the provided outbound message to
the specified channel
"""
worker = yield self.get_router_worker({})
yield worker.consume_channel('testchannel', lambda m: m, lambda e: e)
outbound = self.messagehelper.make_outbound('test message')
yield worker.send_outbound_to_channel('testchannel', outbound)
[message] = yield self.workerhelper.wait_for_dispatched_outbound(
connector_name='testchannel')
self.assertEqual(message, outbound)
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
self.worker_helper = self.add_helper(WorkerHelper())
self.msg_helper = self.add_helper(MessageHelper())
