def test_new_crash_reprocessing_queue(self): """ Tests queue with reprocessing, standard items; no priority items """ config = self._setup_config() crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = 'socorro.normal' crash_store.rabbitmq.config.reprocessing_queue_name = \ 'socorro.reprocessing' crash_store.rabbitmq.config.priority_queue_name = 'socorro.priority' test_queue = [ (None, None, None), ('1', '1', 'normal_crash_id'), (None, None, None), ('1', '1', 'reprocessing_crash_id'), (None, None, None), ] def basic_get(queue='socorro.priority'): if len(test_queue) == 0: return (None, None, None) if queue == 'socorro.priority': return test_queue.pop() elif queue == 'socorro.reprocessing': return test_queue.pop() elif queue == 'socorro.normal': return test_queue.pop() crash_store.rabbitmq.connection.return_value.channel.basic_get = \ MagicMock(side_effect=basic_get) expected = ['normal_crash_id', 'reprocessing_crash_id'] for result in crash_store.new_crashes(): eq_(expected.pop(), result)
def test_new_crash_reprocessing_queue(self): """ Tests queue with reprocessing, standard items; no priority items """ config = self._setup_config() config.transaction_executor_class = TransactionExecutor crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = 'socorro.normal' crash_store.rabbitmq.config.reprocessing_queue_name = \ 'socorro.reprocessing' crash_store.rabbitmq.config.priority_queue_name = 'socorro.priority' test_queue = [ (None, None, None), ('1', '1', 'normal_crash_id'), (None, None, None), ('1', '1', 'reprocessing_crash_id'), (None, None, None), ] def basic_get(queue): if len(test_queue) == 0: raise StopIteration return test_queue.pop() crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=basic_get) expected = ['normal_crash_id', 'reprocessing_crash_id'] for result in crash_store.new_crashes(): eq_(expected.pop(), result)
def test_new_crash_with_fail_retry_then_permanent_fail(self): config = self._setup_config() config.transaction_executor_class = \ TransactionExecutorWithInfiniteBackoff crash_store = RabbitMQCrashStorage(config) class MyException(Exception): pass iterable = [('1', '1', 'crash_id'), MyException(), timeout(), ('2', '2', 'other_id')] def an_iterator(queue): item = iterable.pop() if isinstance(item, Exception): raise item return item crash_store.rabbitmq.operational_exceptions = (timeout, ) crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=an_iterator) expected = ('other_id', ) count = 0 try: for expected, result in zip(expected, crash_store.new_crashes()): count += 1 if count == 1: eq_(expected, result) eq_(count, 1, 'looped too far') except MyException: eq_(count, 1)
def test_new_crash_standard_queue(self): """ Tests queue with standard queue items only """ config = self._setup_config() crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = 'socorro.normal' crash_store.rabbitmq.config.reprocessing_queue_name = 'socorro.reprocessing' crash_store.rabbitmq.config.priority_queue_name = 'socorro.priority' test_queue = [ ('1', '1', 'normal_crash_id'), (None, None, None), (None, None, None), ] def basic_get(queue): if len(test_queue) == 0: return if queue == 'socorro.priority': return (None, None, None) elif queue == 'socorro.reprocessing': return (None, None, None) elif queue == 'socorro.normal': return test_queue.pop() crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=basic_get) expected = ['normal_crash_id'] for result in crash_store.new_crashes(): assert expected.pop() == result
def test_new_crash_reprocessing_queue(self): """ Tests queue with reprocessing, standard items; no priority items """ config = self._setup_config() config.transaction_executor_class = TransactionExecutor crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = "socorro.normal" crash_store.rabbitmq.config.reprocessing_queue_name = "socorro.reprocessing" crash_store.rabbitmq.config.priority_queue_name = "socorro.priority" test_queue = [ (None, None, None), ("1", "1", "normal_crash_id"), (None, None, None), ("1", "1", "reprocessing_crash_id"), (None, None, None), ] def basic_get(queue): if len(test_queue) == 0: raise StopIteration return test_queue.pop() crash_store.rabbitmq.return_value.__enter__.return_value.channel.basic_get = MagicMock(side_effect=basic_get) expected = ["normal_crash_id", "reprocessing_crash_id"] for result in crash_store.new_crashes(): eq_(expected.pop(), result)
def test_new_crash_with_fail_retry(self): config = self._setup_config() config.transaction_executor_class = \ TransactionExecutorWithInfiniteBackoff crash_store = RabbitMQCrashStorage(config) iterable = [('1', '1', 'crash_id'), timeout(), timeout(), ('2', '2', 'other_id')] def an_iterator(queue): item = iterable.pop() if isinstance(item, Exception): raise item return item crash_store.rabbitmq.operational_exceptions = (timeout, ) crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=an_iterator) expected = ( 'other_id', 'crash_id', ) for expected, result in zip(expected, crash_store.new_crashes()): assert expected == result
def test_new_crash_duplicate_discovered(self): """ Tests queue with standard queue items only """ config = self._setup_config() config.transaction_executor_class = TransactionExecutor crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = "socorro.normal" crash_store.rabbitmq.config.reprocessing_queue_name = "socorro.reprocessing" crash_store.rabbitmq.config.priority_queue_name = "socorro.priority" faked_methodframe = DotDict() faked_methodframe.delivery_tag = "delivery_tag" test_queue = [(None, None, None), (faked_methodframe, "1", "normal_crash_id"), (None, None, None)] def basic_get(queue="socorro.priority"): if len(test_queue) == 0: raise StopIteration return test_queue.pop() crash_store.rabbitmq.return_value.__enter__.return_value.channel.basic_get = MagicMock(side_effect=basic_get) transaction_connection = crash_store.transaction.db_conn_context_source.return_value.__enter__.return_value # load the cache as if this crash had alredy been seen crash_store.acknowledgement_token_cache["normal_crash_id"] = faked_methodframe for result in crash_store.new_crashes(): # new crash should be suppressed eq_(None, result) # we should ack the new crash even though we did use it for processing transaction_connection.channel.basic_ack.assert_called_with(delivery_tag=faked_methodframe.delivery_tag)
def test_new_crash_with_fail_retry_then_permanent_fail(self): config = self._setup_config() config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff crash_store = RabbitMQCrashStorage(config) class MyException(Exception): pass iterable = [("1", "1", "crash_id"), MyException(), timeout(), ("2", "2", "other_id")] def an_iterator(queue): item = iterable.pop() if isinstance(item, Exception): raise item return item crash_store.rabbitmq.operational_exceptions = (timeout,) crash_store.rabbitmq.return_value.__enter__.return_value.channel.basic_get = MagicMock(side_effect=an_iterator) expected = ("other_id",) count = 0 try: for expected, result in zip(expected, crash_store.new_crashes()): count += 1 if count == 1: eq_(expected, result) eq_(count, 1, "looped too far") except MyException: eq_(count, 1)
def test_new_crash_standard_queue(self): """ Tests queue with standard queue items only """ config = self._setup_config() config.transaction_executor_class = TransactionExecutor crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = 'socorro.normal' crash_store.rabbitmq.config.reprocessing_queue_name = \ 'socorro.reprocessing' crash_store.rabbitmq.config.priority_queue_name = 'socorro.priority' test_queue = [ ('1', '1', 'normal_crash_id'), (None, None, None), (None, None, None), ] def basic_get(queue): if len(test_queue) == 0: raise StopIteration if queue == 'socorro.priority': return (None, None, None) elif queue == 'socorro.reprocessing': return (None, None, None) elif queue == 'socorro.normal': return test_queue.pop() crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=basic_get) expected = ['normal_crash_id'] for result in crash_store.new_crashes(): assert expected.pop() == result
def test_new_crash_with_fail_retry(self): config = self._setup_config() config.transaction_executor_class = \ TransactionExecutorWithInfiniteBackoff crash_store = RabbitMQCrashStorage(config) iterable = [ ('1', '1', 'crash_id'), timeout(), timeout(), ('2', '2', 'other_id') ] def an_iterator(queue): item = iterable.pop() if isinstance(item, Exception): raise item return item crash_store.rabbitmq.operational_exceptions = ( timeout, ) crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=an_iterator) expected = ('other_id', 'crash_id', ) for expected, result in zip(expected, crash_store.new_crashes()): assert expected == result
def test_new_crash(self): config = self._setup_config() crash_store = RabbitMQCrashStorage(config) iterable = (('1', '1', 'crash_id'),) crash_store.rabbitmq.connection.return_value.channel.basic_get = \ MagicMock(side_effect=iterable) expected = 'crash_id' for result in crash_store.new_crashes(): eq_(expected, result)
def test_new_crash(self): config = self._setup_config() crash_store = RabbitMQCrashStorage(config) iterable = (('1', '1', 'crash_id'), ) crash_store.rabbitmq.connection.return_value.channel.basic_get = \ MagicMock(side_effect=iterable) expected = 'crash_id' for result in crash_store.new_crashes(): eq_(expected, result)
def test_new_crash_duplicate_discovered(self): """ Tests queue with standard queue items only """ config = self._setup_config() config.transaction_executor_class = TransactionExecutor crash_store = RabbitMQCrashStorage(config) crash_store.rabbitmq.config.standard_queue_name = 'socorro.normal' crash_store.rabbitmq.config.reprocessing_queue_name = \ 'socorro.reprocessing' crash_store.rabbitmq.config.priority_queue_name = 'socorro.priority' faked_methodframe = DotDict() faked_methodframe.delivery_tag = 'delivery_tag' test_queue = [ (None, None, None), (faked_methodframe, '1', 'normal_crash_id'), (None, None, None), ] def basic_get(queue='socorro.priority'): if len(test_queue) == 0: raise StopIteration return test_queue.pop() crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=basic_get) transaction_connection = crash_store.transaction.db_conn_context_source \ .return_value.__enter__.return_value # load the cache as if this crash had alredy been seen crash_store.acknowledgement_token_cache['normal_crash_id'] = \ faked_methodframe for result in crash_store.new_crashes(): # new crash should be suppressed eq_(None, result) # we should ack the new crash even though we did use it for processing transaction_connection.channel.basic_ack \ .assert_called_with( delivery_tag=faked_methodframe.delivery_tag )
def test_new_crash(self): config = self._setup_config() config.transaction_executor_class = TransactionExecutorWithInfiniteBackoff crash_store = RabbitMQCrashStorage(config) iterable = [StopIteration(), ("1", "1", "crash_id")] def iter_the_iterable(queue): item = iterable.pop() if isinstance(item, Exception): raise item return item crash_store.rabbitmq.return_value.__enter__.return_value.channel.basic_get = MagicMock( side_effect=iter_the_iterable ) expected = "crash_id" for result in crash_store.new_crashes(): eq_(expected, result)
def test_new_crash(self): config = self._setup_config() config.transaction_executor_class = \ TransactionExecutorWithInfiniteBackoff crash_store = RabbitMQCrashStorage(config) iterable = [ StopIteration(), ('1', '1', 'crash_id'), ] def iter_the_iterable(queue): item = iterable.pop() if isinstance(item, Exception): raise item return item crash_store.rabbitmq.return_value.__enter__.return_value \ .channel.basic_get = MagicMock(side_effect=iter_the_iterable) expected = 'crash_id' for result in crash_store.new_crashes(): eq_(expected, result)