class CommonRiakManagerTests(object): """Common tests for Riak managers. Tests assume self.manager is set to a suitable Riak manager. """ def mkdummy(self, key, data=None, dummy_class=DummyModel): dummy = dummy_class(self.manager, key) dummy.set_riak(self.manager.riak_object(dummy, key)) if data is not None: dummy.set_data(data) return dummy @inlineCallbacks def purge_txriak(self): """ Creates a manager, purges all the riak data with it, then closes it. """ manager = self.create_txriak_manager() yield manager.purge_all() yield manager.close_manager() def create_txriak_manager(self): """ Creates and returns a TxRiakManager, handling cleanup. """ try: from vumi.persist.txriak_manager import TxRiakManager except ImportError, e: import_skip(e, 'riak', 'riak') self.add_cleanup(self.purge_txriak) return TxRiakManager.from_config({'bucket_prefix': 'test.'})
def setup_middleware(self): store_prefix = self.config.store_prefix r_config = self.config.redis_manager self.redis = yield TxRedisManager.from_config(r_config) manager = TxRiakManager.from_config(self.config.riak_manager) self.store = MessageStore(manager, self.redis.sub_manager(store_prefix)) self.store_on_consume = self.config.store_on_consume
def setup_middleware(self): store_prefix = self.config.get('store_prefix', 'message_store') r_config = self.config.get('redis_manager', {}) self.redis = yield TxRedisManager.from_config(r_config) manager = TxRiakManager.from_config(self.config.get('riak_manager')) self.store = MessageStore(manager, self.redis.sub_manager(store_prefix)) self.store_on_consume = self.config.get('store_on_consume', True)
def from_config_async(cls, config, command_publisher=None, metric_publisher=None): # Note: This takes a publisher rather than a client to avoid leaking # AMQP channels by making our own transient publishers. riak_config, redis_config = cls._parse_config(config) manager = TxRiakManager.from_config(riak_config) redis = yield TxRedisManager.from_config(redis_config) sender = AsyncMessageSender(command_publisher) returnValue(cls(manager, redis, sender, metric_publisher))
class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase): @inlineCallbacks def setUp(self): try: from vumi.persist.txriak_manager import TxRiakManager except ImportError, e: import_skip(e, 'riak', 'riak') self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'}) self.add_cleanup(self.manager.purge_all) yield self.manager.purge_all()
class TestModelOnTxRiak(VumiTestCase, ModelTestMixin): @inlineCallbacks def setUp(self): try: from vumi.persist.txriak_manager import TxRiakManager except ImportError, e: import_skip(e, 'riak') self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'}) self.add_cleanup(self.cleanup_manager) yield self.manager.purge_all()
def setup_worker(self): config = self.get_static_config() self._riak = yield TxRiakManager.from_config(config.riak_manager) redis = yield TxRedisManager.from_config(config.redis_manager) self.store = MessageStore(self._riak, redis) site = build_web_site({ config.web_path: MessageStoreResource(self.store), config.health_path: httprpc.HttpRpcHealthResource(self), }) self.addService( StreamServerEndpointService(config.twisted_endpoint, site))
def setup_worker(self): config = self.get_static_config() riak = yield TxRiakManager.from_config(config.riak_manager) redis = yield TxRedisManager.from_config(config.redis_manager) self.store = MessageStore(riak, redis) site = build_web_site({ config.web_path: MessageStoreResource(self.store), config.health_path: httprpc.HttpRpcHealthResource(self), }) self.addService( StreamServerEndpointService(config.twisted_endpoint, site))
def startWorker(self): web_path = self.config['web_path'] web_port = int(self.config['web_port']) health_path = self.config['health_path'] riak = yield TxRiakManager.from_config(self.config['riak_manager']) redis = yield TxRedisManager.from_config(self.config['redis_manager']) self.store = MessageStore(riak, redis) self.webserver = self.start_web_resources([ (MessageStoreAPI(self.store), web_path), (httprpc.HttpRpcHealthResource(self), health_path), ], web_port)
class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase): @inlineCallbacks def setUp(self): try: from vumi.persist.txriak_manager import TxRiakManager from riakasaurus import transport except ImportError, e: import_skip(e, 'riakasaurus', 'riakasaurus.riak') self.pbc_transport = transport.PBCTransport self.http_transport = transport.HTTPTransport self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'}) self.add_cleanup(self.manager.purge_all) yield self.manager.purge_all()
class TestModelOnTxRiak(VumiTestCase): # TODO: all copies of mkmsg must be unified! def mkmsg(self, **kw): kw.setdefault("transport_name", "sphex") kw.setdefault("transport_type", "sphex_type") kw.setdefault("to_addr", "1234") kw.setdefault("from_addr", "5678") return TransportUserMessage(**kw) @inlineCallbacks def setUp(self): try: from vumi.persist.txriak_manager import TxRiakManager except ImportError, e: import_skip(e, 'riakasaurus', 'riakasaurus.riak') self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'}) self.add_cleanup(self.manager.purge_all) yield self.manager.purge_all()
def run(self): manager = TxRiakManager.from_config({'bucket_prefix': 'test.bench.'}) model = manager.proxy(MessageModel) yield manager.purge_all() msg_batches = self.make_batches() start = time.time() for batch in msg_batches: yield self.write_batch(model, batch) write_done = time.time() write_time = write_done - start print "Write took %.2f seconds (%.2f msgs/s)" % ( write_time, self.messages / write_time) result_batches = [] for batch in msg_batches: r = yield self.read_batch(model, batch) result_batches.append(r) read_done = time.time() read_time = read_done - write_done print "Read took %.2f seconds (%.2f msgs/s)" % ( read_time, self.messages / read_time) for batch, result_batch in zip(msg_batches, result_batches): for msg, (good, stored_msg) in zip(batch, result_batch): if not good or stored_msg is None: raise RuntimeError("Failed to retrieve message (%s)" % msg['content']) if not (msg == stored_msg.msg): # TODO: fix message != raise RuntimeError("Message %r does not equal stored" " message %r" % (msg, stored_msg.msg)) print "Messages retrieved successfully." yield manager.purge_all() print "Messages purged."
def run(self): manager = TxRiakManager.from_config({'bucket_prefix': 'test.bench.'}) model = manager.proxy(MessageModel) yield manager.purge_all() msg_batches = self.make_batches() start = time.time() for batch in msg_batches: yield self.write_batch(model, batch) write_done = time.time() write_time = write_done - start print "Write took %.2f seconds (%.2f msgs/s)" % ( write_time, self.messages / write_time) result_batches = [] for batch in msg_batches: r = yield self.read_batch(model, batch) result_batches.append(r) read_done = time.time() read_time = read_done - write_done print "Read took %.2f seconds (%.2f msgs/s)" % ( read_time, self.messages / read_time) for batch, result_batch in zip(msg_batches, result_batches): for msg, (good, stored_msg) in zip(batch, result_batch): if not good or stored_msg is None: raise RuntimeError("Failed to retrieve message (%s)" % msg['content']) if not(msg == stored_msg.msg): # TODO: fix message != raise RuntimeError("Message %r does not equal stored" " message %r" % (msg, stored_msg.msg)) print "Messages retrieved successfully." yield manager.purge_all() print "Messages purged."
def get_riak_manager(self, riak_config): return TxRiakManager.from_config(riak_config)
def get_riak_manager(self, config=None): if config is None: config = self._persist_config['riak_manager'].copy() if self.sync_persistence: return self._get_sync_riak_manager(config) return self._get_async_riak_manager(config) def _get_async_riak_manager(self, config): try: from vumi.persist.txriak_manager import TxRiakManager except ImportError, e: import_skip(e, 'riak') return TxRiakManager.from_config(config) def _get_sync_riak_manager(self, config): try: from vumi.persist.riak_manager import RiakManager except ImportError, e: import_skip(e, 'riak') return RiakManager.from_config(config) def get_redis_manager(self, config=None): if config is None: config = self._persist_config['redis_manager'].copy() if self.sync_persistence: return self._get_sync_redis_manager(config)
except ImportError, e: import_skip(e, 'riak') self.manager = RiakManager.from_config({'bucket_prefix': 'test.'}) self.add_cleanup(deco.cleanup_manager, self) self.manager.purge_all() @inlineCallbacks def setup_txriak(deco, self): """ Set up a TxRiakManager on the given test class. """ try: from vumi.persist.txriak_manager import TxRiakManager except ImportError, e: import_skip(e, 'riak') self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'}) self.add_cleanup(deco.cleanup_manager, self) yield self.manager.purge_all() @inlineCallbacks def cleanup_manager(deco, self): """ Clean up the Riak manager on the given test class. """ yield self.manager.purge_all() yield self.manager.close_manager() model_field_tests = ModelFieldTestsDecorator()
def _get_riak_manager(self, config): try: return self.riak_manager except AttributeError: self.riak_manager = TxRiakManager.from_config(config.riak_manager) return self.riak_manager
def from_config(cls, config): riak_manager = TxRiakManager.from_config(config) return cls(riak_manager)