Пример #1
0
 def setup_application(self):
     yield super(BulkMessageApplication, self).setup_application()
     wm_redis = self.redis.sub_manager('%s:window_manager' %
                                       (self.worker_name, ))
     self.window_manager = WindowManager(wm_redis,
                                         window_size=self.max_ack_window,
                                         flight_lifetime=self.max_ack_wait)
     self.window_manager.monitor(self.on_window_key_ready,
                                 interval=self.monitor_interval,
                                 cleanup=self.monitor_window_cleanup,
                                 cleanup_callback=self.on_window_cleanup)
Пример #2
0
    def setUp(self):
        self.persistence_helper = self.add_helper(PersistenceHelper())
        redis = yield self.persistence_helper.get_redis_manager()
        self.window_id = 'window_id'

        # Patch the count_waiting so we can fake the race condition
        self.clock = Clock()
        self.patch(WindowManager, 'count_waiting', lambda _, window_id: 100)

        self.wm = WindowManager(redis, window_size=10, flight_lifetime=10)
        self.add_cleanup(self.wm.stop)
        yield self.wm.create_window(self.window_id)
        self.redis = self.wm.redis
Пример #3
0
    def setUp(self):
        self.persistence_helper = self.add_helper(PersistenceHelper())
        redis = yield self.persistence_helper.get_redis_manager()
        self.window_id = 'window_id'

        # Patch the clock so we can control time
        self.clock = Clock()
        self.patch(WindowManager, 'get_clock', lambda _: self.clock)

        self.wm = WindowManager(redis, window_size=10, flight_lifetime=10)
        self.add_cleanup(self.wm.stop)
        yield self.wm.create_window(self.window_id)
        self.redis = self.wm.redis
Пример #4
0
 def setup_application(self):
     yield super(BulkMessageApplication, self).setup_application()
     wm_redis = self.redis.sub_manager('%s:window_manager' % (
         self.worker_name,))
     self.window_manager = WindowManager(wm_redis,
         window_size=self.max_ack_window,
         flight_lifetime=self.max_ack_wait)
     self.window_manager.monitor(self.on_window_key_ready,
         interval=self.monitor_interval,
         cleanup=self.monitor_window_cleanup,
         cleanup_callback=self.on_window_cleanup)
Пример #5
0
class TestConcurrentWindowManager(VumiTestCase):

    @inlineCallbacks
    def setUp(self):
        self.persistence_helper = self.add_helper(PersistenceHelper())
        redis = yield self.persistence_helper.get_redis_manager()
        self.window_id = 'window_id'

        # Patch the count_waiting so we can fake the race condition
        self.clock = Clock()
        self.patch(WindowManager, 'count_waiting', lambda _, window_id: 100)

        self.wm = WindowManager(redis, window_size=10, flight_lifetime=10)
        self.add_cleanup(self.wm.stop)
        yield self.wm.create_window(self.window_id)
        self.redis = self.wm.redis

    @inlineCallbacks
    def test_race_condition(self):
        """
        A race condition can occur when multiple window managers try and
        access the same window at the same time.

        A LoopingCall loops over the available windows, for those windows
        it tries to get a next key. It does that by checking how many are
        waiting to be sent out and adding however many it can still carry
        to its own flight.

        Since there are concurrent workers, between the time of checking how
        many are available and how much room it has available, a different
        window manager may have already beaten it to it.

        If this happens Redis' `rpoplpush` method will return None since
        there are no more available keys for the given window.
        """
        yield self.wm.add(self.window_id, 1)
        yield self.wm.add(self.window_id, 2)
        yield self.wm._monitor_windows(lambda *a: True, True)
        self.assertEqual((yield self.wm.get_next_key(self.window_id)), None)
Пример #6
0
class BulkMessageApplication(GoApplicationWorker):
    """
    Application that accepts 'send message' commands and does exactly that.
    """
    worker_name = 'bulk_message_application'
    max_ack_window = 100
    max_ack_wait = 100
    monitor_interval = 20
    monitor_window_cleanup = True

    @inlineCallbacks
    def setup_application(self):
        yield super(BulkMessageApplication, self).setup_application()
        wm_redis = self.redis.sub_manager('%s:window_manager' % (
            self.worker_name,))
        self.window_manager = WindowManager(wm_redis,
            window_size=self.max_ack_window,
            flight_lifetime=self.max_ack_wait)
        self.window_manager.monitor(self.on_window_key_ready,
            interval=self.monitor_interval,
            cleanup=self.monitor_window_cleanup,
            cleanup_callback=self.on_window_cleanup)

    @inlineCallbacks
    def teardown_application(self):
        yield super(BulkMessageApplication, self).teardown_application()
        self.window_manager.stop()

    @inlineCallbacks
    def on_window_key_ready(self, window_id, flight_key):
        data = yield self.window_manager.get_data(window_id, flight_key)
        to_addr = data['to_addr']
        content = data['content']
        msg_options = data['msg_options']
        msg = yield self.send_to(
            to_addr, content, endpoint='default', **msg_options)
        yield self.window_manager.set_external_id(window_id, flight_key,
            msg['message_id'])

    def on_window_cleanup(self, window_id):
        log.info('Finished window %s, removing.' % (window_id,))

    def get_window_id(self, conversation_key, batch_id):
        return ':'.join([conversation_key, batch_id])

    @inlineCallbacks
    def send_message_via_window(self, conv, window_id, batch_id, to_addr,
                                msg_options, content):
        yield self.window_manager.create_window(window_id, strict=False)
        yield self.window_manager.add(window_id, {
            'batch_id': batch_id,
            'to_addr': to_addr,
            'content': content,
            'msg_options': msg_options,
            })

    @inlineCallbacks
    def process_command_bulk_send(self, user_account_key, conversation_key,
                                  batch_id, msg_options, content, dedupe,
                                  delivery_class, **extra_params):

        conv = yield self.get_conversation(user_account_key, conversation_key)
        if conv is None:
            log.warning("Cannot find conversation '%s' for user '%s'." % (
                conversation_key, user_account_key))
            return

        to_addresses = []
        for contacts_batch in (
                yield conv.get_opted_in_contact_bunches(delivery_class)):
            for contact in (yield contacts_batch):
                to_addresses.append(contact.addr_for(delivery_class))
        if dedupe:
            to_addresses = set(to_addresses)

        self.add_conv_to_msg_options(conv, msg_options)
        window_id = self.get_window_id(conversation_key, batch_id)
        for to_addr in to_addresses:
            yield self.send_message_via_window(
                conv, window_id, batch_id, to_addr, msg_options, content)

    def consume_ack(self, event):
        return self.handle_event(event)

    def consume_nack(self, event):
        return self.handle_event(event)

    @inlineCallbacks
    def handle_event(self, event):
        message = yield self.find_message_for_event(event)
        if message is None:
            log.error('Unable to find message for %s, user_message_id: %s' % (
                event['event_type'], event.get('user_message_id')))
            return

        msg_mdh = self.get_metadata_helper(message)
        conv = yield msg_mdh.get_conversation()
        if conv:
            window_id = self.get_window_id(conv.key, conv.batch.key)
            flight_key = yield self.window_manager.get_internal_id(window_id,
                                message['message_id'])
            yield self.window_manager.remove_key(window_id, flight_key)

    @inlineCallbacks
    def process_command_initial_action_hack(self, user_account_key,
                                            conversation_key, **kwargs):
        # HACK: This lets us do whatever we used to do when we got a `start'
        # message without having horrible app-specific view logic.
        # TODO: Remove this when we've decoupled the various conversation
        # actions from the lifecycle.

        conv = yield self.get_conversation(user_account_key, conversation_key)
        if conv is None:
            log.warning("Cannot find conversation '%s' for user '%s'." % (
                user_account_key, conversation_key))
            return

        kwargs.setdefault('content', conv.description)
        kwargs.setdefault('dedupe', False)
        yield self.process_command_bulk_send(
            user_account_key=user_account_key,
            conversation_key=conversation_key,
            **kwargs)
Пример #7
0
class BulkMessageApplication(GoApplicationWorker):
    """
    Application that accepts 'send message' commands and does exactly that.
    """
    worker_name = 'bulk_message_application'
    max_ack_window = 100
    max_ack_wait = 100
    monitor_interval = 20
    monitor_window_cleanup = True

    @inlineCallbacks
    def setup_application(self):
        yield super(BulkMessageApplication, self).setup_application()
        wm_redis = self.redis.sub_manager('%s:window_manager' %
                                          (self.worker_name, ))
        self.window_manager = WindowManager(wm_redis,
                                            window_size=self.max_ack_window,
                                            flight_lifetime=self.max_ack_wait)
        self.window_manager.monitor(self.on_window_key_ready,
                                    interval=self.monitor_interval,
                                    cleanup=self.monitor_window_cleanup,
                                    cleanup_callback=self.on_window_cleanup)

    @inlineCallbacks
    def teardown_application(self):
        yield super(BulkMessageApplication, self).teardown_application()
        self.window_manager.stop()

    @inlineCallbacks
    def on_window_key_ready(self, window_id, flight_key):
        data = yield self.window_manager.get_data(window_id, flight_key)
        to_addr = data['to_addr']
        content = data['content']
        msg_options = data['msg_options']
        msg = yield self.send_to(to_addr,
                                 content,
                                 endpoint='default',
                                 **msg_options)
        yield self.window_manager.set_external_id(window_id, flight_key,
                                                  msg['message_id'])

    def on_window_cleanup(self, window_id):
        log.info('Finished window %s, removing.' % (window_id, ))

    def get_window_id(self, conversation_key, batch_id):
        return ':'.join([conversation_key, batch_id])

    @inlineCallbacks
    def send_message_via_window(self, conv, window_id, batch_id, to_addr,
                                msg_options, content):
        yield self.window_manager.create_window(window_id, strict=False)
        yield self.window_manager.add(
            window_id, {
                'batch_id': batch_id,
                'to_addr': to_addr,
                'content': content,
                'msg_options': msg_options,
            })

    @inlineCallbacks
    def process_command_bulk_send(self, user_account_key, conversation_key,
                                  batch_id, msg_options, content, dedupe,
                                  delivery_class, **extra_params):

        conv = yield self.get_conversation(user_account_key, conversation_key)
        if conv is None:
            log.warning("Cannot find conversation '%s' for user '%s'." %
                        (conversation_key, user_account_key))
            return

        to_addresses = []
        for contacts_batch in (
                yield conv.get_opted_in_contact_bunches(delivery_class)):
            for contact in (yield contacts_batch):
                to_addresses.append(contact.addr_for(delivery_class))
        if dedupe:
            to_addresses = set(to_addresses)

        self.add_conv_to_msg_options(conv, msg_options)
        window_id = self.get_window_id(conversation_key, batch_id)
        for to_addr in to_addresses:
            yield self.send_message_via_window(conv, window_id, batch_id,
                                               to_addr, msg_options, content)

    def consume_ack(self, event):
        return self.handle_event(event)

    def consume_nack(self, event):
        return self.handle_event(event)

    @inlineCallbacks
    def handle_event(self, event):
        message = yield self.find_message_for_event(event)
        if message is None:
            log.error('Unable to find message for %s, user_message_id: %s' %
                      (event['event_type'], event.get('user_message_id')))
            return

        msg_mdh = self.get_metadata_helper(message)
        conv = yield msg_mdh.get_conversation()
        if conv:
            window_id = self.get_window_id(conv.key, conv.batch.key)
            flight_key = yield self.window_manager.get_internal_id(
                window_id, message['message_id'])
            yield self.window_manager.remove_key(window_id, flight_key)

    @inlineCallbacks
    def process_command_initial_action_hack(self, user_account_key,
                                            conversation_key, **kwargs):
        # HACK: This lets us do whatever we used to do when we got a `start'
        # message without having horrible app-specific view logic.
        # TODO: Remove this when we've decoupled the various conversation
        # actions from the lifecycle.

        conv = yield self.get_conversation(user_account_key, conversation_key)
        if conv is None:
            log.warning("Cannot find conversation '%s' for user '%s'." %
                        (user_account_key, conversation_key))
            return

        kwargs.setdefault('content', conv.description)
        kwargs.setdefault('dedupe', False)
        yield self.process_command_bulk_send(user_account_key=user_account_key,
                                             conversation_key=conversation_key,
                                             **kwargs)
Пример #8
0
class TestWindowManager(VumiTestCase):

    @inlineCallbacks
    def setUp(self):
        self.persistence_helper = self.add_helper(PersistenceHelper())
        redis = yield self.persistence_helper.get_redis_manager()
        self.window_id = 'window_id'

        # Patch the clock so we can control time
        self.clock = Clock()
        self.patch(WindowManager, 'get_clock', lambda _: self.clock)

        self.wm = WindowManager(redis, window_size=10, flight_lifetime=10)
        self.add_cleanup(self.wm.stop)
        yield self.wm.create_window(self.window_id)
        self.redis = self.wm.redis

    @inlineCallbacks
    def test_windows(self):
        windows = yield self.wm.get_windows()
        self.assertTrue(self.window_id in windows)

    def test_strict_window_recreation(self):
        return self.assertFailure(
            self.wm.create_window(self.window_id, strict=True),
                                    WindowException)

    @inlineCallbacks
    def test_window_recreation(self):
        orig_clock_time = self.clock.seconds()
        clock_time = yield self.wm.create_window(self.window_id)
        self.assertEqual(clock_time, orig_clock_time)

    @inlineCallbacks
    def test_window_removal(self):
        yield self.wm.add(self.window_id, 1)
        yield self.assertFailure(self.wm.remove_window(self.window_id),
            WindowException)
        key = yield self.wm.get_next_key(self.window_id)
        item = yield self.wm.get_data(self.window_id, key)
        self.assertEqual(item, 1)
        self.assertEqual((yield self.wm.remove_window(self.window_id)), None)

    @inlineCallbacks
    def test_adding_to_window(self):
        for i in range(10):
            yield self.wm.add(self.window_id, i)
        window_key = self.wm.window_key(self.window_id)
        window_members = yield self.redis.llen(window_key)
        self.assertEqual(window_members, 10)

    @inlineCallbacks
    def test_fetching_from_window(self):
        for i in range(12):
            yield self.wm.add(self.window_id, i)

        flight_keys = []
        for i in range(10):
            flight_key = yield self.wm.get_next_key(self.window_id)
            self.assertTrue(flight_key)
            flight_keys.append(flight_key)

        out_of_window_flight = yield self.wm.get_next_key(self.window_id)
        self.assertEqual(out_of_window_flight, None)

        # We should get data out in the order we put it in
        for i, flight_key in enumerate(flight_keys):
            data = yield self.wm.get_data(self.window_id, flight_key)
            self.assertEqual(data, i)

        # Removing one should allow for space for the next to fill up
        yield self.wm.remove_key(self.window_id, flight_keys[0])
        next_flight_key = yield self.wm.get_next_key(self.window_id)
        self.assertTrue(next_flight_key)

    @inlineCallbacks
    def test_set_and_external_id(self):
        yield self.wm.set_external_id(self.window_id, "flight_key",
                                      "external_id")
        self.assertEqual(
            (yield self.wm.get_external_id(self.window_id, "flight_key")),
            "external_id")
        self.assertEqual(
            (yield self.wm.get_internal_id(self.window_id, "external_id")),
            "flight_key")

    @inlineCallbacks
    def test_remove_key_removes_external_and_internal_id(self):
        yield self.wm.set_external_id(self.window_id, "flight_key",
                                      "external_id")
        yield self.wm.remove_key(self.window_id, "flight_key")
        self.assertEqual(
            (yield self.wm.get_external_id(self.window_id, "flight_key")),
            None)
        self.assertEqual(
            (yield self.wm.get_internal_id(self.window_id, "external_id")),
            None)

    @inlineCallbacks
    def assert_count_waiting(self, window_id, amount):
        self.assertEqual((yield self.wm.count_waiting(window_id)), amount)

    @inlineCallbacks
    def assert_expired_keys(self, window_id, amount):
        # Stuff has taken too long and so we should get 10 expired keys
        expired_keys = yield self.wm.get_expired_flight_keys(window_id)
        self.assertEqual(len(expired_keys), amount)

    @inlineCallbacks
    def assert_in_flight(self, window_id, amount):
        self.assertEqual((yield self.wm.count_in_flight(window_id)),
            amount)

    @inlineCallbacks
    def slide_window(self, limit=10):
        for i in range(limit):
            yield self.wm.get_next_key(self.window_id)

    @inlineCallbacks
    def test_expiry_of_acks(self):

        def mock_clock_time(self):
            return self._clocktime

        self.patch(WindowManager, 'get_clocktime', mock_clock_time)
        self.wm._clocktime = 0

        for i in range(30):
            yield self.wm.add(self.window_id, i)

        # We're manually setting the clock instead of using clock.advance()
        # so we can wait for the deferreds to finish before continuing to the
        # next clear_expired_flight_keys run since LoopingCall() will only fire
        # again if the previous run has completed.
        yield self.slide_window()
        self.wm._clocktime = 10
        yield self.wm.clear_expired_flight_keys()
        self.assert_expired_keys(self.window_id, 10)

        yield self.slide_window()
        self.wm._clocktime = 20
        yield self.wm.clear_expired_flight_keys()
        self.assert_expired_keys(self.window_id, 20)

        yield self.slide_window()
        self.wm._clocktime = 30
        yield self.wm.clear_expired_flight_keys()
        self.assert_expired_keys(self.window_id, 30)

        self.assert_in_flight(self.window_id, 0)
        self.assert_count_waiting(self.window_id, 0)

    @inlineCallbacks
    def test_monitor_windows(self):
        yield self.wm.remove_window(self.window_id)

        window_ids = ['window_id_1', 'window_id_2']
        for window_id in window_ids:
            yield self.wm.create_window(window_id)
            for i in range(20):
                yield self.wm.add(window_id, i)

        key_callbacks = {}

        def callback(window_id, key):
            key_callbacks.setdefault(window_id, []).append(key)

        cleanup_callbacks = []

        def cleanup_callback(window_id):
            cleanup_callbacks.append(window_id)

        yield self.wm._monitor_windows(callback, False)

        self.assertEqual(set(key_callbacks.keys()), set(window_ids))
        self.assertEqual(len(key_callbacks.values()[0]), 10)
        self.assertEqual(len(key_callbacks.values()[1]), 10)

        yield self.wm._monitor_windows(callback, False)

        # Nothing should've changed since we haven't removed anything.
        self.assertEqual(len(key_callbacks.values()[0]), 10)
        self.assertEqual(len(key_callbacks.values()[1]), 10)

        for window_id, keys in key_callbacks.items():
            for key in keys:
                yield self.wm.remove_key(window_id, key)

        yield self.wm._monitor_windows(callback, False)
        # Everything should've been processed now
        self.assertEqual(len(key_callbacks.values()[0]), 20)
        self.assertEqual(len(key_callbacks.values()[1]), 20)

        # Now run again but cleanup the empty windows
        self.assertEqual(set((yield self.wm.get_windows())), set(window_ids))
        for window_id, keys in key_callbacks.items():
            for key in keys:
                yield self.wm.remove_key(window_id, key)

        yield self.wm._monitor_windows(callback, True, cleanup_callback)
        self.assertEqual(len(key_callbacks.values()[0]), 20)
        self.assertEqual(len(key_callbacks.values()[1]), 20)
        self.assertEqual((yield self.wm.get_windows()), [])
        self.assertEqual(set(cleanup_callbacks), set(window_ids))