class TestMessageStoreAPI(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
try:
from vumi.components.message_store_api import (
MatchResource, MessageStoreAPIWorker)
except ImportError, e:
import_skip(e, 'riakasaurus', 'riakasaurus.riak')
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
self.match_resource = MatchResource
self.base_path = '/api/v1/'
self.worker = yield self.worker_helper.get_worker(
MessageStoreAPIWorker,
self.persistence_helper.mk_config({
'web_path': self.base_path,
'web_port': 0,
'health_path': '/health/',
}))
self.store = self.worker.store
self.addr = self.worker.webserver.getHost()
self.url = 'http://%s:%s%s' % (self.addr.host, self.addr.port,
self.base_path)
self.tag = ("pool", "tag")
self.batch_id = yield self.store.batch_start([self.tag])
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
if riak_import_error is not None:
import_skip(riak_import_error, 'riak')
self.manager = self.persistence_helper.get_riak_manager()
self.msg_helper = self.add_helper(MessageHelper())
class TestMessageStoreResource(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
try:
from vumi.components.message_store_resource import (
MessageStoreResourceWorker)
except ImportError, e:
import_skip(e, 'riakasaurus', 'riakasaurus.riak')
self.worker_helper = self.add_helper(WorkerHelper())
config = self.persistence_helper.mk_config({
'twisted_endpoint':
'tcp:0',
'web_path':
'/resource_path/',
})
worker = yield self.worker_helper.get_worker(
MessageStoreResourceWorker, config)
yield worker.startService()
port = yield worker.services[0]._waitingForPort
addr = port.getHost()
self.msg_helper = self.add_helper(MessageHelper())
self.url = 'http://%s:%s' % (addr.host, addr.port)
self.store = worker.store
self.addCleanup(self.stop_server, port)
def __init__(self, vumi_helper=None, **kw):
self._msg_helper = MessageHelper(**kw)
self.transport_name = self._msg_helper.transport_name
self._vumi_helper = vumi_helper
self.mdb = None
if self._vumi_helper is not None:
self.mdb = self._vumi_helper.get_vumi_api().mdb
def test_rebuild_cache(self):
"""
Rebuilding the info cache for a batch will clear all cached data and
rebuild it from the given QueryMessageStore.
"""
msg_helper = self.add_helper(MessageHelper())
batch_info_cache = self.batch_manager.batch_info_cache
# Fill the message store backend with the data we want in the rebuilt
# cache.
yield self.backend.add_inbound_message(msg_helper.make_inbound("in 1"),
batch_ids=["mybatch"])
yield self.backend.add_outbound_message(
msg_helper.make_outbound("out 1"), batch_ids=["mybatch"])
yield self.backend.add_outbound_message(
msg_helper.make_outbound("out 2"), batch_ids=["mybatch"])
# Fill the cache with some nonsense that we want to throw out when
# rebuilding.
yield batch_info_cache.add_inbound_message_key("mybatch", "in1", 12345)
yield batch_info_cache.add_inbound_message_key("mybatch", "in2", 12345)
old_in = yield batch_info_cache.get_inbound_message_count("mybatch")
old_out = yield batch_info_cache.get_outbound_message_count("mybatch")
self.assertEqual((old_in, old_out), (2, 0))
# Rebuild the cache.
qms = QueryMessageStore(self.manager, self.redis)
yield self.batch_manager.rebuild_cache("mybatch", qms)
new_in = yield batch_info_cache.get_inbound_message_count("mybatch")
new_out = yield batch_info_cache.get_outbound_message_count("mybatch")
self.assertEqual((new_in, new_out), (1, 2))
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
self.worker_helper = self.add_helper(WorkerHelper())
self.msg_helper = self.add_helper(MessageHelper())
riak, redis = yield self.create_managers()
self.operational_store = OperationalMessageStore(riak, redis)
self.batch_manager = MessageStoreBatchManager(riak, redis)
def set_up_tests(self, manager):
"""
This should be called from .setUp().
"""
self.manager = manager
self.backend = MessageStoreRiakBackend(self.manager)
self.msg_helper = self.add_helper(MessageHelper())
self.msg_seq_helper = self.add_helper(
MessageSequenceHelper(self.backend, self.msg_helper))
def setUp(self):
self.workerhelper = WorkerHelper()
self.addCleanup(self.workerhelper.cleanup)
self.persistencehelper = PersistenceHelper()
yield self.persistencehelper.setup()
self.addCleanup(self.persistencehelper.cleanup)
self.messagehelper = MessageHelper()
self.addCleanup(self.messagehelper.cleanup)
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
self.manager = self.persistence_helper.get_riak_manager()
self.add_cleanup(self.manager.close_manager)
self.redis = yield self.persistence_helper.get_redis_manager()
self.store = OperationalMessageStore(self.manager, self.redis)
self.backend = self.store.riak_backend
self.bi_cache = self.store.batch_info_cache
self.msg_helper = self.add_helper(MessageHelper())
def __init__(self, dispatcher_class, use_riak=False, **msg_helper_args):
self.dispatcher_class = dispatcher_class
self.worker_helper = WorkerHelper()
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
def setUp(self):
self.config = {
'transport_names': ['transport1'],
'exposed_names': ['app1', 'app2'],
'toaddr_mappings': {
'app1': 'to:.*:1',
'app2': 'to:app2',
},
}
self.dispatcher = DummyDispatcher(self.config)
self.router = ToAddrRouter(self.dispatcher, self.config)
yield self.router.setup_routing()
self.msg_helper = self.add_helper(MessageHelper())
class TestMessageStoreBase(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
try:
from vumi.components.message_store import MessageStore
except ImportError, e:
import_skip(e, 'riak')
self.redis = yield self.persistence_helper.get_redis_manager()
self.manager = self.persistence_helper.get_riak_manager()
self.store = MessageStore(self.manager, self.redis)
self.msg_helper = self.add_helper(MessageHelper())
def __init__(self, application_class, use_riak=False, **msg_helper_args):
self.application_class = application_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(self.msg_helper.transport_name)
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def setUp(self):
self.clock = Clock()
MessengerTransport.clock = self.clock
self.remote_server = MockHttpServer(lambda _: 'OK')
yield self.remote_server.start()
self.addCleanup(self.remote_server.stop)
self.tx_helper = self.add_helper(
HttpRpcTransportHelper(PatchedMessengerTransport))
self.msg_helper = self.add_helper(MessageHelper())
connection_pool = HTTPConnectionPool(reactor, persistent=False)
treq._utils.set_global_pool(connection_pool)
def test_vumimessage_field_excludes_cache(self):
msg_helper = self.add_helper(MessageHelper())
msg_model = self.manager.proxy(self.VumiMessageModel)
cache_attr = TransportUserMessage._CACHE_ATTRIBUTE
msg = msg_helper.make_inbound("foo", extra="bar")
msg.cache["cache"] = "me"
self.assertEqual(msg[cache_attr], {"cache": "me"})
m1 = msg_model("foo", msg=msg)
self.assertTrue(cache_attr not in m1.msg)
yield m1.save()
m2 = yield msg_model.load("foo")
self.assertTrue(cache_attr not in m2.msg)
self.assertEqual(m2.msg, m1.msg)
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True, is_sync=False))
self.msg_helper = self.add_helper(MessageHelper())
# Since we're never loading the actual objects, we can't detect
# tombstones. Therefore, each test needs its own bucket prefix.
self.expected_bucket_prefix = "bucket-%s" % (uuid4().hex,)
self.riak_manager = self.persistence_helper.get_riak_manager({
"bucket_prefix": self.expected_bucket_prefix,
})
self.add_cleanup(self.riak_manager.close_manager)
self.redis_manager = yield self.persistence_helper.get_redis_manager()
self.mdb = MessageStore(self.riak_manager, self.redis_manager)
self.default_args = [
"-b", self.expected_bucket_prefix,
]
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True, is_sync=False))
self.msg_helper = self.add_helper(MessageHelper())
# Since we're never loading the actual objects, we can't detect
# tombstones. Therefore, each test needs its own bucket prefix.
config = self.persistence_helper.mk_config({})["riak_manager"].copy()
config["bucket_prefix"] = "%s-%s" % (
uuid4().hex, config["bucket_prefix"])
self.riak_manager = self.persistence_helper.get_riak_manager(config)
self.redis_manager = yield self.persistence_helper.get_redis_manager()
self.mdb = MessageStore(self.riak_manager, self.redis_manager)
self.expected_bucket_prefix = "bucket"
self.default_args = [
"-b", self.expected_bucket_prefix,
]
def test_vumimessage_field(self):
msg_helper = self.add_helper(MessageHelper())
msg_model = self.manager.proxy(self.VumiMessageModel)
msg = msg_helper.make_inbound("foo", extra="bar")
m1 = msg_model("foo", msg=msg)
yield m1.save()
m2 = yield msg_model.load("foo")
self.assertEqual(m1.msg, m2.msg)
self.assertEqual(m2.msg, msg)
self.assertRaises(ValidationError, setattr, m1, "msg", "foo")
# test extra keys are removed
msg2 = msg_helper.make_inbound("foo")
m1.msg = msg2
self.assertTrue("extra" not in m1.msg)
def __init__(self, transport_class, use_riak=False, **msg_helper_args):
self.transport_class = transport_class
self.persistence_helper = PersistenceHelper(use_riak=use_riak)
self.msg_helper = MessageHelper(**msg_helper_args)
self.transport_name = self.msg_helper.transport_name
self.worker_helper = WorkerHelper(connector_name=self.transport_name,
status_connector_name="%s.status" %
(self.transport_name, ))
self.dispatch_helper = MessageDispatchHelper(self.msg_helper,
self.worker_helper)
# Proxy methods from our helpers.
generate_proxies(self, self.msg_helper)
generate_proxies(self, self.worker_helper)
generate_proxies(self, self.dispatch_helper)
generate_proxies(self, self.persistence_helper)
def setUp(self):
config = {
"transport_names": [
"transport_1",
"transport_2",
],
"exposed_names": ["round_robin"],
"router_class": ("vumi.dispatchers.load_balancer."
"LoadBalancingRouter"),
}
if self.reply_affinity is not None:
config['reply_affinity'] = self.reply_affinity
if self.rewrite_transport_names is not None:
config['rewrite_transport_names'] = self.rewrite_transport_names
self.dispatcher = DummyDispatcher(config)
self.router = LoadBalancingRouter(self.dispatcher, config)
self.add_cleanup(self.router.teardown_routing)
yield self.router.setup_routing()
self.msg_helper = self.add_helper(MessageHelper())
class MessageStoreCacheTestCase(VumiTestCase):
start_batch = True
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
try:
from vumi.components.message_store import MessageStore
except ImportError, e:
import_skip(e, 'riak')
self.redis = yield self.persistence_helper.get_redis_manager()
self.manager = yield self.persistence_helper.get_riak_manager()
self.store = yield MessageStore(self.manager, self.redis)
self.cache = self.store.cache
self.batch_id = 'a-batch-id'
if self.start_batch:
yield self.cache.batch_start(self.batch_id)
self.msg_helper = self.add_helper(MessageHelper())
def setUp(self):
config = {
"transport_names": [
"transport_1",
"transport_2",
"transport_3",
],
"exposed_names": ["muxed"],
"router_class": "vumi.dispatchers.base.FromAddrMultiplexRouter",
"fromaddr_mappings": {
"thing1@muxme": "transport_1",
"thing2@muxme": "transport_2",
"thing3@muxme": "transport_3",
},
}
self.dispatcher = DummyDispatcher(config)
self.router = FromAddrMultiplexRouter(self.dispatcher, config)
self.add_cleanup(self.router.teardown_routing)
yield self.router.setup_routing()
self.msg_helper = self.add_helper(MessageHelper())
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
self.worker = yield self.worker_helper.get_worker(
DummyWorker, {}, False)
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
self.clear_events()
self.add_cleanup(self.clear_events)
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.worker_helper = self.add_helper(WorkerHelper())
def setUp(self):
self.persistence_helper = self.add_helper(
PersistenceHelper(use_riak=True))
self.worker_helper = self.add_helper(WorkerHelper())
self.msg_helper = self.add_helper(MessageHelper())
def setUp(self):
self.r_server = FakeRedis()
self.scheduler = Scheduler(self.r_server, self._scheduler_callback)
self.add_cleanup(self.stop_scheduler)
self._delivery_history = []
self.msg_helper = self.add_helper(MessageHelper())
def setUp(self):
self.msg_helper = self.add_helper(MessageHelper())
self.request = DummyRequest([''])
self.formatter = CsvFormatter()
def setUp(self):
self.vumi_helper = yield self.add_helper(VumiApiHelper())
self.user_helper = yield self.vumi_helper.make_user(u'user')
self.msg_helper = self.add_helper(MessageHelper())
