class CommonRiakManagerTests(object):
"""Common tests for Riak managers.
Tests assume self.manager is set to a suitable Riak
manager.
"""
def mkdummy(self, key, data=None, dummy_class=DummyModel):
dummy = dummy_class(self.manager, key)
dummy.set_riak(self.manager.riak_object(dummy, key))
if data is not None:
dummy.set_data(data)
return dummy
@inlineCallbacks
def purge_txriak(self):
"""
Creates a manager, purges all the riak data with it, then closes it.
"""
manager = self.create_txriak_manager()
yield manager.purge_all()
yield manager.close_manager()
def create_txriak_manager(self):
"""
Creates and returns a TxRiakManager, handling cleanup.
"""
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak', 'riak')
self.add_cleanup(self.purge_txriak)
return TxRiakManager.from_config({'bucket_prefix': 'test.'})
def setup_middleware(self):
store_prefix = self.config.store_prefix
r_config = self.config.redis_manager
self.redis = yield TxRedisManager.from_config(r_config)
manager = TxRiakManager.from_config(self.config.riak_manager)
self.store = MessageStore(manager,
self.redis.sub_manager(store_prefix))
self.store_on_consume = self.config.store_on_consume
def setup_middleware(self):
store_prefix = self.config.get('store_prefix', 'message_store')
r_config = self.config.get('redis_manager', {})
self.redis = yield TxRedisManager.from_config(r_config)
manager = TxRiakManager.from_config(self.config.get('riak_manager'))
self.store = MessageStore(manager,
self.redis.sub_manager(store_prefix))
self.store_on_consume = self.config.get('store_on_consume', True)
def setup_middleware(self):
store_prefix = self.config.store_prefix
r_config = self.config.redis_manager
self.redis = yield TxRedisManager.from_config(r_config)
manager = TxRiakManager.from_config(self.config.riak_manager)
self.store = MessageStore(manager,
self.redis.sub_manager(store_prefix))
self.store_on_consume = self.config.store_on_consume
def from_config_async(cls, config, command_publisher=None,
metric_publisher=None):
# Note: This takes a publisher rather than a client to avoid leaking
# AMQP channels by making our own transient publishers.
riak_config, redis_config = cls._parse_config(config)
manager = TxRiakManager.from_config(riak_config)
redis = yield TxRedisManager.from_config(redis_config)
sender = AsyncMessageSender(command_publisher)
returnValue(cls(manager, redis, sender, metric_publisher))
class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase):
@inlineCallbacks
def setUp(self):
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak', 'riak')
self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(self.manager.purge_all)
yield self.manager.purge_all()
class TestModelOnTxRiak(VumiTestCase, ModelTestMixin):
@inlineCallbacks
def setUp(self):
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak')
self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(self.cleanup_manager)
yield self.manager.purge_all()
def from_config_async(cls,
config,
command_publisher=None,
metric_publisher=None):
# Note: This takes a publisher rather than a client to avoid leaking
# AMQP channels by making our own transient publishers.
riak_config, redis_config = cls._parse_config(config)
manager = TxRiakManager.from_config(riak_config)
redis = yield TxRedisManager.from_config(redis_config)
sender = AsyncMessageSender(command_publisher)
returnValue(cls(manager, redis, sender, metric_publisher))
def setup_worker(self):
config = self.get_static_config()
self._riak = yield TxRiakManager.from_config(config.riak_manager)
redis = yield TxRedisManager.from_config(config.redis_manager)
self.store = MessageStore(self._riak, redis)
site = build_web_site({
config.web_path: MessageStoreResource(self.store),
config.health_path: httprpc.HttpRpcHealthResource(self),
})
self.addService(
StreamServerEndpointService(config.twisted_endpoint, site))
def setup_worker(self):
config = self.get_static_config()
riak = yield TxRiakManager.from_config(config.riak_manager)
redis = yield TxRedisManager.from_config(config.redis_manager)
self.store = MessageStore(riak, redis)
site = build_web_site({
config.web_path: MessageStoreResource(self.store),
config.health_path: httprpc.HttpRpcHealthResource(self),
})
self.addService(
StreamServerEndpointService(config.twisted_endpoint, site))
def startWorker(self):
web_path = self.config['web_path']
web_port = int(self.config['web_port'])
health_path = self.config['health_path']
riak = yield TxRiakManager.from_config(self.config['riak_manager'])
redis = yield TxRedisManager.from_config(self.config['redis_manager'])
self.store = MessageStore(riak, redis)
self.webserver = self.start_web_resources([
(MessageStoreAPI(self.store), web_path),
(httprpc.HttpRpcHealthResource(self), health_path),
], web_port)
class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase):
@inlineCallbacks
def setUp(self):
try:
from vumi.persist.txriak_manager import TxRiakManager
from riakasaurus import transport
except ImportError, e:
import_skip(e, 'riakasaurus', 'riakasaurus.riak')
self.pbc_transport = transport.PBCTransport
self.http_transport = transport.HTTPTransport
self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(self.manager.purge_all)
yield self.manager.purge_all()
def startWorker(self):
web_path = self.config['web_path']
web_port = int(self.config['web_port'])
health_path = self.config['health_path']
riak = yield TxRiakManager.from_config(self.config['riak_manager'])
redis = yield TxRedisManager.from_config(self.config['redis_manager'])
self.store = MessageStore(riak, redis)
self.webserver = self.start_web_resources([
(MessageStoreAPI(self.store), web_path),
(httprpc.HttpRpcHealthResource(self), health_path),
], web_port)
class TestModelOnTxRiak(VumiTestCase):
# TODO: all copies of mkmsg must be unified!
def mkmsg(self, **kw):
kw.setdefault("transport_name", "sphex")
kw.setdefault("transport_type", "sphex_type")
kw.setdefault("to_addr", "1234")
kw.setdefault("from_addr", "5678")
return TransportUserMessage(**kw)
@inlineCallbacks
def setUp(self):
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riakasaurus', 'riakasaurus.riak')
self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(self.manager.purge_all)
yield self.manager.purge_all()
def run(self):
manager = TxRiakManager.from_config({'bucket_prefix': 'test.bench.'})
model = manager.proxy(MessageModel)
yield manager.purge_all()
msg_batches = self.make_batches()
start = time.time()
for batch in msg_batches:
yield self.write_batch(model, batch)
write_done = time.time()
write_time = write_done - start
print "Write took %.2f seconds (%.2f msgs/s)" % (
write_time, self.messages / write_time)
result_batches = []
for batch in msg_batches:
r = yield self.read_batch(model, batch)
result_batches.append(r)
read_done = time.time()
read_time = read_done - write_done
print "Read took %.2f seconds (%.2f msgs/s)" % (
read_time, self.messages / read_time)
for batch, result_batch in zip(msg_batches, result_batches):
for msg, (good, stored_msg) in zip(batch, result_batch):
if not good or stored_msg is None:
raise RuntimeError("Failed to retrieve message (%s)" %
msg['content'])
if not (msg == stored_msg.msg): # TODO: fix message !=
raise RuntimeError("Message %r does not equal stored"
" message %r" % (msg, stored_msg.msg))
print "Messages retrieved successfully."
yield manager.purge_all()
print "Messages purged."
def run(self):
manager = TxRiakManager.from_config({'bucket_prefix': 'test.bench.'})
model = manager.proxy(MessageModel)
yield manager.purge_all()
msg_batches = self.make_batches()
start = time.time()
for batch in msg_batches:
yield self.write_batch(model, batch)
write_done = time.time()
write_time = write_done - start
print "Write took %.2f seconds (%.2f msgs/s)" % (
write_time, self.messages / write_time)
result_batches = []
for batch in msg_batches:
r = yield self.read_batch(model, batch)
result_batches.append(r)
read_done = time.time()
read_time = read_done - write_done
print "Read took %.2f seconds (%.2f msgs/s)" % (
read_time, self.messages / read_time)
for batch, result_batch in zip(msg_batches, result_batches):
for msg, (good, stored_msg) in zip(batch, result_batch):
if not good or stored_msg is None:
raise RuntimeError("Failed to retrieve message (%s)"
% msg['content'])
if not(msg == stored_msg.msg): # TODO: fix message !=
raise RuntimeError("Message %r does not equal stored"
" message %r" % (msg, stored_msg.msg))
print "Messages retrieved successfully."
yield manager.purge_all()
print "Messages purged."
def get_riak_manager(self, riak_config):
return TxRiakManager.from_config(riak_config)
def get_riak_manager(self, config=None):
if config is None:
config = self._persist_config['riak_manager'].copy()
if self.sync_persistence:
return self._get_sync_riak_manager(config)
return self._get_async_riak_manager(config)
def _get_async_riak_manager(self, config):
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak')
return TxRiakManager.from_config(config)
def _get_sync_riak_manager(self, config):
try:
from vumi.persist.riak_manager import RiakManager
except ImportError, e:
import_skip(e, 'riak')
return RiakManager.from_config(config)
def get_redis_manager(self, config=None):
if config is None:
config = self._persist_config['redis_manager'].copy()
if self.sync_persistence:
return self._get_sync_redis_manager(config)
except ImportError, e:
import_skip(e, 'riak')
self.manager = RiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(deco.cleanup_manager, self)
self.manager.purge_all()
@inlineCallbacks
def setup_txriak(deco, self):
"""
Set up a TxRiakManager on the given test class.
"""
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak')
self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(deco.cleanup_manager, self)
yield self.manager.purge_all()
@inlineCallbacks
def cleanup_manager(deco, self):
"""
Clean up the Riak manager on the given test class.
"""
yield self.manager.purge_all()
yield self.manager.close_manager()
model_field_tests = ModelFieldTestsDecorator()
def get_riak_manager(self, config=None):
if config is None:
config = self._persist_config['riak_manager'].copy()
if self.sync_persistence:
return self._get_sync_riak_manager(config)
return self._get_async_riak_manager(config)
def _get_async_riak_manager(self, config):
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak')
return TxRiakManager.from_config(config)
def _get_sync_riak_manager(self, config):
try:
from vumi.persist.riak_manager import RiakManager
except ImportError, e:
import_skip(e, 'riak')
return RiakManager.from_config(config)
def get_redis_manager(self, config=None):
if config is None:
config = self._persist_config['redis_manager'].copy()
if self.sync_persistence:
return self._get_sync_redis_manager(config)
def _get_riak_manager(self, config):
try:
return self.riak_manager
except AttributeError:
self.riak_manager = TxRiakManager.from_config(config.riak_manager)
return self.riak_manager
def get_riak_manager(self, riak_config):
return TxRiakManager.from_config(riak_config)
def from_config(cls, config):
riak_manager = TxRiakManager.from_config(config)
return cls(riak_manager)
except ImportError, e:
import_skip(e, 'riak')
self.manager = RiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(deco.cleanup_manager, self)
self.manager.purge_all()
@inlineCallbacks
def setup_txriak(deco, self):
"""
Set up a TxRiakManager on the given test class.
"""
try:
from vumi.persist.txriak_manager import TxRiakManager
except ImportError, e:
import_skip(e, 'riak')
self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
self.add_cleanup(deco.cleanup_manager, self)
yield self.manager.purge_all()
@inlineCallbacks
def cleanup_manager(deco, self):
"""
Clean up the Riak manager on the given test class.
"""
yield self.manager.purge_all()
yield self.manager.close_manager()
model_field_tests = ModelFieldTestsDecorator()
