def save_raw_crash(self, raw_crash, dumps, crash_id):
try:
this_crash_should_be_queued = (
not self.config.filter_on_legacy_processing or
raw_crash.legacy_processing == 0
)
except KeyError:
self.logger.debug(
'RabbitMQCrashStorage legacy_processing key absent in crash '
'%s', crash_id
)
return
if this_crash_should_be_queued:
self.logger.debug('RabbitMQCrashStorage saving crash %s', crash_id)
retry(
self.rabbitmq,
self.quit_check,
self._save_raw_crash,
crash_id=crash_id
)
return True
else:
self.logger.debug(
'RabbitMQCrashStorage not saving crash %s, legacy processing '
'flag is %s', crash_id, raw_crash.legacy_processing
)
def test_retry_once(self):
with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock:
sleep_mock.return_value = None
conn = object()
connection_context = mock.MagicMock()
connection_context.return_value.__enter__.return_value = conn
quit_check = mock.MagicMock()
exc = Exception('omg!')
def fun(conn, call_count):
# The first time this is called, raise an exception; use
# call_count to maintain state between calls
if len(call_count) == 0:
call_count.append(1)
raise exc
call_count.append(1)
return 1
call_count = []
retry(connection_context, quit_check, fun, call_count=call_count)
# Assert fun was called twice
assert len(call_count) == 2
# Assert quit_check was called once
assert quit_check.call_count == 1
def test_retry_once(self):
with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock:
sleep_mock.return_value = None
conn = object()
connection_context = mock.MagicMock()
connection_context.return_value.__enter__.return_value = conn
quit_check = mock.MagicMock()
exc = Exception('omg!')
def fun(conn, call_count):
# The first time this is called, raise an exception; use
# call_count to maintain state between calls
if len(call_count) == 0:
call_count.append(1)
raise exc
call_count.append(1)
return 1
call_count = []
retry(
connection_context,
quit_check,
fun,
call_count=call_count
)
# Assert fun was called twice
assert len(call_count) == 2
# Assert quit_check was called once
assert quit_check.call_count == 1
def save_raw_crash(self, raw_crash, dumps, crash_id):
retry(self.connection_source,
self.quit_check,
self.do_save_raw_crash,
raw_crash=raw_crash,
dumps=dumps,
crash_id=crash_id)
def _consume_acknowledgement_queue(self):
"""The acknowledgement of the processing of each crash_id yielded
from the 'new_crashes' method must take place on the same connection
that the crash_id came from. The crash_ids are queued in the
'acknowledgment_queue'. That queue is consumed by the QueuingThread"""
try:
while True:
crash_id_to_be_acknowledged = self.acknowledgment_queue.get_nowait(
)
try:
acknowledgement_token = self.acknowledgement_token_cache[
crash_id_to_be_acknowledged]
retry(connection_context=self.rabbitmq,
quit_check=self.quit_check_callback,
fun=self._ack_crash,
crash_id=crash_id_to_be_acknowledged,
acknowledgement_token=acknowledgement_token)
del self.acknowledgement_token_cache[
crash_id_to_be_acknowledged]
except KeyError:
self.logger.warning(
'RabbitMQCrashQueue tried to acknowledge crash %s, which was not in cache',
crash_id_to_be_acknowledged,
exc_info=True)
except Exception:
self.logger.error(
'RabbitMQCrashQueue unexpected failure on %s',
crash_id_to_be_acknowledged,
exc_info=True)
except Empty:
pass # nothing to do with an empty queue
def save_processed(self, processed_crash):
retry(
self.connection_source,
self.quit_check,
self._do_save_processed,
processed_crash=processed_crash,
)
def test_retry_and_die(self):
with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock:
sleep_mock.return_value = None
conn = object()
connection_context = mock.MagicMock()
connection_context.return_value.__enter__.return_value = conn
quit_check = mock.MagicMock()
exc = Exception('omg!')
def fun(conn, call_count):
# Raise exceptions to simulate failing; use call_count to keep
# track
call_count.append(1)
raise exc
call_count = []
with pytest.raises(Exception) as exc_info:
retry(
connection_context,
quit_check,
fun,
call_count=call_count
)
# Assert retry runs out of backoffs and throws the last error
assert exc_info.value == exc
# Assert fun was called six times
assert len(call_count) == 6
# quit_check gets called for every backoff, so 6 times
assert quit_check.call_count == 6
def save_processed(self, processed_crash):
retry(
self.connection_source,
self.quit_check,
self._do_save_processed,
processed_crash=processed_crash
)
def test_retry_and_die(self):
with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock:
sleep_mock.return_value = None
conn = object()
connection_context = mock.MagicMock()
connection_context.return_value.__enter__.return_value = conn
quit_check = mock.MagicMock()
exc = Exception('omg!')
def fun(conn, call_count):
# Raise exceptions to simulate failing; use call_count to keep
# track
call_count.append(1)
raise exc
call_count = []
with pytest.raises(Exception) as exc_info:
retry(connection_context,
quit_check,
fun,
call_count=call_count)
# Assert retry runs out of backoffs and throws the last error
assert exc_info.value == exc
# Assert fun was called six times
assert len(call_count) == 6
# quit_check gets called for every backoff, so 6 times
assert quit_check.call_count == 6
def save_raw_crash(self, raw_crash, dumps, crash_id):
retry(
self.connection_source,
self.quit_check,
self.do_save_raw_crash,
raw_crash=raw_crash,
dumps=dumps,
crash_id=crash_id
)
def test_fine(self):
conn = object()
connection_context = mock.MagicMock()
connection_context.return_value.__enter__.return_value = conn
quit_check = mock.MagicMock()
fun = mock.MagicMock()
retry(connection_context, quit_check, fun)
# Assert fun was called with the connection as the first argument
fun.assert_called_with(conn)
# Assert quit_check was not called at all
assert not quit_check.called
def _suppress_duplicate_jobs(self, crash_id, acknowledgement_token):
"""if this crash is in the cache, then it is already in progress
and this is a duplicate. Acknowledge it, then return to True
to let the caller know to skip on to the next crash."""
if crash_id in self.acknowledgement_token_cache:
# reject this crash - it's already being processsed
self.logger.info('duplicate job: %s is already in progress',
crash_id)
# ack this
retry(connection_context=self.rabbitmq,
quit_check=self.quit_check_callback,
fun=self._ack_crash,
crash_id=crash_id,
acknowledgement_token=acknowledgement_token)
return True
return False
def get_raw_crash(self, crash_id):
return retry(
self.connection_source,
self.quit_check,
self.do_get_raw_crash,
crash_id=crash_id,
json_object_hook=self.config.json_object_hook
)
def get_unredacted_processed(self, crash_id):
return retry(
self.connection_source,
self.quit_check,
self._do_get_unredacted_processed,
crash_id=crash_id,
json_object_hook=self.config.json_object_hook,
)
def get_raw_dump(self, crash_id, name=None):
return retry(
self.connection_source,
self.quit_check,
self.do_get_raw_dump,
crash_id=crash_id,
name=name,
)
def get_raw_dump(self, crash_id, name=None):
return retry(
self.connection_source,
self.quit_check,
self.do_get_raw_dump,
crash_id=crash_id,
name=name
)
def _suppress_duplicate_jobs(self, crash_id, acknowledgement_token):
"""if this crash is in the cache, then it is already in progress
and this is a duplicate. Acknowledge it, then return to True
to let the caller know to skip on to the next crash."""
if crash_id in self.acknowledgement_token_cache:
# reject this crash - it's already being processsed
self.logger.info('duplicate job: %s is already in progress', crash_id)
# ack this
retry(
connection_context=self.rabbitmq,
quit_check=self.quit_check_callback,
fun=self._ack_crash,
crash_id=crash_id,
acknowledgement_token=acknowledgement_token
)
return True
return False
def get_raw_crash(self, crash_id):
return retry(
self.connection_source,
self.quit_check,
self.do_get_raw_crash,
crash_id=crash_id,
json_object_hook=self.config.json_object_hook,
)
def get_unredacted_processed(self, crash_id):
return retry(
self.connection_source,
self.quit_check,
self._do_get_unredacted_processed,
crash_id=crash_id,
json_object_hook=self.config.json_object_hook
)
def test_fine(self):
conn = object()
connection_context = mock.MagicMock()
connection_context.return_value.__enter__.return_value = conn
quit_check = mock.MagicMock()
fun = mock.MagicMock()
retry(
connection_context,
quit_check,
fun
)
# Assert fun was called with the connection as the first argument
fun.assert_called_with(conn)
# Assert quit_check was not called at all
assert not quit_check.called
def get_raw_dumps(self, crash_id):
"""Fetch raw dumps
:returns: MemoryDumpsMapping
"""
return retry(self.connection_source,
self.quit_check,
self.do_get_raw_dumps,
crash_id=crash_id)
def get_raw_dumps(self, crash_id):
"""Fetch raw dumps
:returns: MemoryDumpsMapping
"""
return retry(
self.connection_source,
self.quit_check,
self.do_get_raw_dumps,
crash_id=crash_id
)
def _consume_acknowledgement_queue(self):
"""The acknowledgement of the processing of each crash_id yielded
from the 'new_crashes' method must take place on the same connection
that the crash_id came from. The crash_ids are queued in the
'acknowledgment_queue'. That queue is consumed by the QueuingThread"""
try:
while True:
crash_id_to_be_acknowledged = self.acknowledgment_queue.get_nowait()
try:
acknowledgement_token = self.acknowledgement_token_cache[
crash_id_to_be_acknowledged
]
retry(
connection_context=self.rabbitmq,
quit_check=self.quit_check_callback,
fun=self._ack_crash,
crash_id=crash_id_to_be_acknowledged,
acknowledgement_token=acknowledgement_token
)
del self.acknowledgement_token_cache[crash_id_to_be_acknowledged]
except KeyError:
self.logger.warning(
'RabbitMQCrashQueue tried to acknowledge crash %s, which was not in cache',
crash_id_to_be_acknowledged,
exc_info=True
)
except Exception:
self.logger.error(
'RabbitMQCrashQueue unexpected failure on %s',
crash_id_to_be_acknowledged,
exc_info=True
)
except Empty:
pass # nothing to do with an empty queue
def __iter__(self):
"""Return an iterator over crashes from RabbitMQ.
Each crash is a tuple of the ``(args, kwargs)`` variety. The lone arg
is a crash ID, and the kwargs contain only a callback function which
the FTS app will call to send an ack to Rabbit after processing is
complete.
"""
self._consume_acknowledgement_queue()
queues = [
self.rabbitmq.config.priority_queue_name,
self.rabbitmq.config.standard_queue_name,
self.rabbitmq.config.reprocessing_queue_name,
self.rabbitmq.config.priority_queue_name,
]
while True:
for queue in queues:
method_frame, header_frame, body = retry(
connection_context=self.rabbitmq,
quit_check=self.quit_check_callback,
fun=self._basic_get,
queue=queue
)
# The body is always a string, so convert it to a string
if body:
body = body.decode('utf-8')
if method_frame and self._suppress_duplicate_jobs(body, method_frame):
continue
if method_frame:
break
# must consume ack queue before testing for end of iterator
# or the last job won't get ack'd
self._consume_acknowledgement_queue()
if not method_frame:
# there was nothing in the queue - leave the iterator
return
self.acknowledgement_token_cache[body] = method_frame
yield (
(body,),
{'finished_func': partial(self.ack_crash, body)}
)
queues.reverse()
def __iter__(self):
"""Return an iterator over crashes from RabbitMQ.
Each crash is a tuple of the ``(args, kwargs)`` variety. The lone arg
is a crash ID, and the kwargs contain only a callback function which
the FTS app will call to send an ack to Rabbit after processing is
complete.
"""
self._consume_acknowledgement_queue()
queues = [
self.rabbitmq.config.priority_queue_name,
self.rabbitmq.config.standard_queue_name,
self.rabbitmq.config.reprocessing_queue_name,
self.rabbitmq.config.priority_queue_name,
]
while True:
for queue in queues:
method_frame, header_frame, body = retry(
connection_context=self.rabbitmq,
quit_check=self.quit_check_callback,
fun=self._basic_get,
queue=queue)
# The body is always a string, so convert it to a string
if body:
body = body.decode('utf-8')
if method_frame and self._suppress_duplicate_jobs(
body, method_frame):
continue
if method_frame:
break
# must consume ack queue before testing for end of iterator
# or the last job won't get ack'd
self._consume_acknowledgement_queue()
if not method_frame:
# there was nothing in the queue - leave the iterator
return
self.acknowledgement_token_cache[body] = method_frame
yield ((body, ), {'finished_func': partial(self.ack_crash, body)})
queues.reverse()
def new_crashes(self):
"""This generator fetches crash_ids from RabbitMQ."""
# We've set up RabbitMQ to require acknowledgement of processing of a
# crash_id from this generator. It is the responsibility of the
# consumer of the crash_id to tell this instance of the class when has
# completed its work on the crash_id. That is done with the call to
# 'ack_crash' below. Because RabbitMQ connections are not thread safe,
# only the thread that read the crash may acknowledge it. 'ack_crash'
# queues the crash_id. The '_consume_acknowledgement_queue' function
# is run to send acknowledgments back to RabbitMQ
self._consume_acknowledgement_queue()
queues = [
self.rabbitmq.config.priority_queue_name,
self.rabbitmq.config.standard_queue_name,
self.rabbitmq.config.reprocessing_queue_name,
self.rabbitmq.config.priority_queue_name,
]
while True:
for queue in queues:
method_frame, header_frame, body = retry(self.rabbitmq,
self.quit_check,
self._basic_get,
queue=queue)
# The body is always a string, so convert it to a string
if body:
body = body.decode('utf-8')
if method_frame and self._suppress_duplicate_jobs(
body, method_frame):
continue
if method_frame:
break
# must consume ack queue before testing for end of iterator
# or the last job won't get ack'd
self._consume_acknowledgement_queue()
if not method_frame:
# there was nothing in the queue - leave the iterator
return
self.acknowledgement_token_cache[body] = method_frame
yield body
queues.reverse()
