Beispiel #1
0
class CommonRiakManagerTests(object):
    """Common tests for Riak managers.

    Tests assume self.manager is set to a suitable Riak
    manager.
    """
    def mkdummy(self, key, data=None, dummy_class=DummyModel):
        dummy = dummy_class(self.manager, key)
        dummy.set_riak(self.manager.riak_object(dummy, key))
        if data is not None:
            dummy.set_data(data)
        return dummy

    @inlineCallbacks
    def purge_txriak(self):
        """
        Creates a manager, purges all the riak data with it, then closes it.
        """
        manager = self.create_txriak_manager()
        yield manager.purge_all()
        yield manager.close_manager()

    def create_txriak_manager(self):
        """
        Creates and returns a TxRiakManager, handling cleanup.
        """
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak', 'riak')
        self.add_cleanup(self.purge_txriak)
        return TxRiakManager.from_config({'bucket_prefix': 'test.'})
Beispiel #2
0
 def setup_middleware(self):
     store_prefix = self.config.store_prefix
     r_config = self.config.redis_manager
     self.redis = yield TxRedisManager.from_config(r_config)
     manager = TxRiakManager.from_config(self.config.riak_manager)
     self.store = MessageStore(manager,
                               self.redis.sub_manager(store_prefix))
     self.store_on_consume = self.config.store_on_consume
Beispiel #3
0
 def setup_middleware(self):
     store_prefix = self.config.get('store_prefix', 'message_store')
     r_config = self.config.get('redis_manager', {})
     self.redis = yield TxRedisManager.from_config(r_config)
     manager = TxRiakManager.from_config(self.config.get('riak_manager'))
     self.store = MessageStore(manager,
                               self.redis.sub_manager(store_prefix))
     self.store_on_consume = self.config.get('store_on_consume', True)
Beispiel #4
0
 def setup_middleware(self):
     store_prefix = self.config.store_prefix
     r_config = self.config.redis_manager
     self.redis = yield TxRedisManager.from_config(r_config)
     manager = TxRiakManager.from_config(self.config.riak_manager)
     self.store = MessageStore(manager,
                               self.redis.sub_manager(store_prefix))
     self.store_on_consume = self.config.store_on_consume
Beispiel #5
0
 def from_config_async(cls, config, command_publisher=None,
                       metric_publisher=None):
     # Note: This takes a publisher rather than a client to avoid leaking
     #       AMQP channels by making our own transient publishers.
     riak_config, redis_config = cls._parse_config(config)
     manager = TxRiakManager.from_config(riak_config)
     redis = yield TxRedisManager.from_config(redis_config)
     sender = AsyncMessageSender(command_publisher)
     returnValue(cls(manager, redis, sender, metric_publisher))
class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase):
    @inlineCallbacks
    def setUp(self):
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak', 'riak')
        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(self.manager.purge_all)
        yield self.manager.purge_all()
Beispiel #7
0
class TestModelOnTxRiak(VumiTestCase, ModelTestMixin):

    @inlineCallbacks
    def setUp(self):
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak')
        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(self.cleanup_manager)
        yield self.manager.purge_all()
Beispiel #8
0
 def from_config_async(cls,
                       config,
                       command_publisher=None,
                       metric_publisher=None):
     # Note: This takes a publisher rather than a client to avoid leaking
     #       AMQP channels by making our own transient publishers.
     riak_config, redis_config = cls._parse_config(config)
     manager = TxRiakManager.from_config(riak_config)
     redis = yield TxRedisManager.from_config(redis_config)
     sender = AsyncMessageSender(command_publisher)
     returnValue(cls(manager, redis, sender, metric_publisher))
    def setup_worker(self):
        config = self.get_static_config()
        self._riak = yield TxRiakManager.from_config(config.riak_manager)
        redis = yield TxRedisManager.from_config(config.redis_manager)
        self.store = MessageStore(self._riak, redis)

        site = build_web_site({
            config.web_path: MessageStoreResource(self.store),
            config.health_path: httprpc.HttpRpcHealthResource(self),
        })
        self.addService(
            StreamServerEndpointService(config.twisted_endpoint, site))
Beispiel #10
0
    def setup_worker(self):
        config = self.get_static_config()
        riak = yield TxRiakManager.from_config(config.riak_manager)
        redis = yield TxRedisManager.from_config(config.redis_manager)
        self.store = MessageStore(riak, redis)

        site = build_web_site({
            config.web_path: MessageStoreResource(self.store),
            config.health_path: httprpc.HttpRpcHealthResource(self),
        })
        self.addService(
            StreamServerEndpointService(config.twisted_endpoint, site))
Beispiel #11
0
    def startWorker(self):
        web_path = self.config['web_path']
        web_port = int(self.config['web_port'])
        health_path = self.config['health_path']

        riak = yield TxRiakManager.from_config(self.config['riak_manager'])
        redis = yield TxRedisManager.from_config(self.config['redis_manager'])
        self.store = MessageStore(riak, redis)

        self.webserver = self.start_web_resources([
            (MessageStoreAPI(self.store), web_path),
            (httprpc.HttpRpcHealthResource(self), health_path),
            ], web_port)
Beispiel #12
0
class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase):
    @inlineCallbacks
    def setUp(self):
        try:
            from vumi.persist.txriak_manager import TxRiakManager
            from riakasaurus import transport
        except ImportError, e:
            import_skip(e, 'riakasaurus', 'riakasaurus.riak')
        self.pbc_transport = transport.PBCTransport
        self.http_transport = transport.HTTPTransport
        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(self.manager.purge_all)
        yield self.manager.purge_all()
Beispiel #13
0
    def startWorker(self):
        web_path = self.config['web_path']
        web_port = int(self.config['web_port'])
        health_path = self.config['health_path']

        riak = yield TxRiakManager.from_config(self.config['riak_manager'])
        redis = yield TxRedisManager.from_config(self.config['redis_manager'])
        self.store = MessageStore(riak, redis)

        self.webserver = self.start_web_resources([
            (MessageStoreAPI(self.store), web_path),
            (httprpc.HttpRpcHealthResource(self), health_path),
        ], web_port)
Beispiel #14
0
class TestModelOnTxRiak(VumiTestCase):

    # TODO: all copies of mkmsg must be unified!
    def mkmsg(self, **kw):
        kw.setdefault("transport_name", "sphex")
        kw.setdefault("transport_type", "sphex_type")
        kw.setdefault("to_addr", "1234")
        kw.setdefault("from_addr", "5678")
        return TransportUserMessage(**kw)

    @inlineCallbacks
    def setUp(self):
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riakasaurus', 'riakasaurus.riak')
        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(self.manager.purge_all)
        yield self.manager.purge_all()
Beispiel #15
0
    def run(self):
        manager = TxRiakManager.from_config({'bucket_prefix': 'test.bench.'})
        model = manager.proxy(MessageModel)
        yield manager.purge_all()

        msg_batches = self.make_batches()

        start = time.time()

        for batch in msg_batches:
            yield self.write_batch(model, batch)

        write_done = time.time()
        write_time = write_done - start
        print "Write took %.2f seconds (%.2f msgs/s)" % (
            write_time, self.messages / write_time)

        result_batches = []
        for batch in msg_batches:
            r = yield self.read_batch(model, batch)
            result_batches.append(r)

        read_done = time.time()
        read_time = read_done - write_done
        print "Read took %.2f seconds (%.2f msgs/s)" % (
            read_time, self.messages / read_time)

        for batch, result_batch in zip(msg_batches, result_batches):
            for msg, (good, stored_msg) in zip(batch, result_batch):
                if not good or stored_msg is None:
                    raise RuntimeError("Failed to retrieve message (%s)" %
                                       msg['content'])
                if not (msg == stored_msg.msg):  # TODO: fix message !=
                    raise RuntimeError("Message %r does not equal stored"
                                       " message %r" % (msg, stored_msg.msg))

        print "Messages retrieved successfully."

        yield manager.purge_all()
        print "Messages purged."
Beispiel #16
0
    def run(self):
        manager = TxRiakManager.from_config({'bucket_prefix': 'test.bench.'})
        model = manager.proxy(MessageModel)
        yield manager.purge_all()

        msg_batches = self.make_batches()

        start = time.time()

        for batch in msg_batches:
            yield self.write_batch(model, batch)

        write_done = time.time()
        write_time = write_done - start
        print "Write took %.2f seconds (%.2f msgs/s)" % (
                write_time, self.messages / write_time)

        result_batches = []
        for batch in msg_batches:
            r = yield self.read_batch(model, batch)
            result_batches.append(r)

        read_done = time.time()
        read_time = read_done - write_done
        print "Read took %.2f seconds (%.2f msgs/s)" % (
                read_time, self.messages / read_time)

        for batch, result_batch in zip(msg_batches, result_batches):
            for msg, (good, stored_msg) in zip(batch, result_batch):
                if not good or stored_msg is None:
                    raise RuntimeError("Failed to retrieve message (%s)"
                                       % msg['content'])
                if not(msg == stored_msg.msg):  # TODO: fix message !=
                    raise RuntimeError("Message %r does not equal stored"
                                       " message %r" % (msg, stored_msg.msg))

        print "Messages retrieved successfully."

        yield manager.purge_all()
        print "Messages purged."
Beispiel #17
0
 def get_riak_manager(self, riak_config):
     return TxRiakManager.from_config(riak_config)
Beispiel #18
0
    def get_riak_manager(self, config=None):
        if config is None:
            config = self._persist_config['riak_manager'].copy()

        if self.sync_persistence:
            return self._get_sync_riak_manager(config)
        return self._get_async_riak_manager(config)

    def _get_async_riak_manager(self, config):
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak')

        return TxRiakManager.from_config(config)

    def _get_sync_riak_manager(self, config):
        try:
            from vumi.persist.riak_manager import RiakManager
        except ImportError, e:
            import_skip(e, 'riak')

        return RiakManager.from_config(config)

    def get_redis_manager(self, config=None):
        if config is None:
            config = self._persist_config['redis_manager'].copy()

        if self.sync_persistence:
            return self._get_sync_redis_manager(config)
Beispiel #19
0
        except ImportError, e:
            import_skip(e, 'riak')
        self.manager = RiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(deco.cleanup_manager, self)
        self.manager.purge_all()

    @inlineCallbacks
    def setup_txriak(deco, self):
        """
        Set up a TxRiakManager on the given test class.
        """
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak')
        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(deco.cleanup_manager, self)
        yield self.manager.purge_all()

    @inlineCallbacks
    def cleanup_manager(deco, self):
        """
        Clean up the Riak manager on the given test class.
        """
        yield self.manager.purge_all()
        yield self.manager.close_manager()


model_field_tests = ModelFieldTestsDecorator()

Beispiel #20
0
    def get_riak_manager(self, config=None):
        if config is None:
            config = self._persist_config['riak_manager'].copy()

        if self.sync_persistence:
            return self._get_sync_riak_manager(config)
        return self._get_async_riak_manager(config)

    def _get_async_riak_manager(self, config):
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak')

        return TxRiakManager.from_config(config)

    def _get_sync_riak_manager(self, config):
        try:
            from vumi.persist.riak_manager import RiakManager
        except ImportError, e:
            import_skip(e, 'riak')

        return RiakManager.from_config(config)

    def get_redis_manager(self, config=None):
        if config is None:
            config = self._persist_config['redis_manager'].copy()

        if self.sync_persistence:
            return self._get_sync_redis_manager(config)
Beispiel #21
0
 def _get_riak_manager(self, config):
     try:
         return self.riak_manager
     except AttributeError:
         self.riak_manager = TxRiakManager.from_config(config.riak_manager)
         return self.riak_manager
Beispiel #22
0
 def get_riak_manager(self, riak_config):
     return TxRiakManager.from_config(riak_config)
Beispiel #23
0
 def from_config(cls, config):
     riak_manager = TxRiakManager.from_config(config)
     return cls(riak_manager)
Beispiel #24
0
        except ImportError, e:
            import_skip(e, 'riak')
        self.manager = RiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(deco.cleanup_manager, self)
        self.manager.purge_all()

    @inlineCallbacks
    def setup_txriak(deco, self):
        """
        Set up a TxRiakManager on the given test class.
        """
        try:
            from vumi.persist.txriak_manager import TxRiakManager
        except ImportError, e:
            import_skip(e, 'riak')
        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
        self.add_cleanup(deco.cleanup_manager, self)
        yield self.manager.purge_all()

    @inlineCallbacks
    def cleanup_manager(deco, self):
        """
        Clean up the Riak manager on the given test class.
        """
        yield self.manager.purge_all()
        yield self.manager.close_manager()


model_field_tests = ModelFieldTestsDecorator()