def save_raw_crash(self, raw_crash, dumps, crash_id): try: this_crash_should_be_queued = ( not self.config.filter_on_legacy_processing or raw_crash.legacy_processing == 0 ) except KeyError: self.logger.debug( 'RabbitMQCrashStorage legacy_processing key absent in crash ' '%s', crash_id ) return if this_crash_should_be_queued: self.logger.debug('RabbitMQCrashStorage saving crash %s', crash_id) retry( self.rabbitmq, self.quit_check, self._save_raw_crash, crash_id=crash_id ) return True else: self.logger.debug( 'RabbitMQCrashStorage not saving crash %s, legacy processing ' 'flag is %s', crash_id, raw_crash.legacy_processing )
def test_retry_once(self): with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock: sleep_mock.return_value = None conn = object() connection_context = mock.MagicMock() connection_context.return_value.__enter__.return_value = conn quit_check = mock.MagicMock() exc = Exception('omg!') def fun(conn, call_count): # The first time this is called, raise an exception; use # call_count to maintain state between calls if len(call_count) == 0: call_count.append(1) raise exc call_count.append(1) return 1 call_count = [] retry(connection_context, quit_check, fun, call_count=call_count) # Assert fun was called twice assert len(call_count) == 2 # Assert quit_check was called once assert quit_check.call_count == 1
def test_retry_once(self): with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock: sleep_mock.return_value = None conn = object() connection_context = mock.MagicMock() connection_context.return_value.__enter__.return_value = conn quit_check = mock.MagicMock() exc = Exception('omg!') def fun(conn, call_count): # The first time this is called, raise an exception; use # call_count to maintain state between calls if len(call_count) == 0: call_count.append(1) raise exc call_count.append(1) return 1 call_count = [] retry( connection_context, quit_check, fun, call_count=call_count ) # Assert fun was called twice assert len(call_count) == 2 # Assert quit_check was called once assert quit_check.call_count == 1
def save_raw_crash(self, raw_crash, dumps, crash_id): retry(self.connection_source, self.quit_check, self.do_save_raw_crash, raw_crash=raw_crash, dumps=dumps, crash_id=crash_id)
def _consume_acknowledgement_queue(self): """The acknowledgement of the processing of each crash_id yielded from the 'new_crashes' method must take place on the same connection that the crash_id came from. The crash_ids are queued in the 'acknowledgment_queue'. That queue is consumed by the QueuingThread""" try: while True: crash_id_to_be_acknowledged = self.acknowledgment_queue.get_nowait( ) try: acknowledgement_token = self.acknowledgement_token_cache[ crash_id_to_be_acknowledged] retry(connection_context=self.rabbitmq, quit_check=self.quit_check_callback, fun=self._ack_crash, crash_id=crash_id_to_be_acknowledged, acknowledgement_token=acknowledgement_token) del self.acknowledgement_token_cache[ crash_id_to_be_acknowledged] except KeyError: self.logger.warning( 'RabbitMQCrashQueue tried to acknowledge crash %s, which was not in cache', crash_id_to_be_acknowledged, exc_info=True) except Exception: self.logger.error( 'RabbitMQCrashQueue unexpected failure on %s', crash_id_to_be_acknowledged, exc_info=True) except Empty: pass # nothing to do with an empty queue
def save_processed(self, processed_crash): retry( self.connection_source, self.quit_check, self._do_save_processed, processed_crash=processed_crash, )
def test_retry_and_die(self): with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock: sleep_mock.return_value = None conn = object() connection_context = mock.MagicMock() connection_context.return_value.__enter__.return_value = conn quit_check = mock.MagicMock() exc = Exception('omg!') def fun(conn, call_count): # Raise exceptions to simulate failing; use call_count to keep # track call_count.append(1) raise exc call_count = [] with pytest.raises(Exception) as exc_info: retry( connection_context, quit_check, fun, call_count=call_count ) # Assert retry runs out of backoffs and throws the last error assert exc_info.value == exc # Assert fun was called six times assert len(call_count) == 6 # quit_check gets called for every backoff, so 6 times assert quit_check.call_count == 6
def save_processed(self, processed_crash): retry( self.connection_source, self.quit_check, self._do_save_processed, processed_crash=processed_crash )
def test_retry_and_die(self): with mock.patch('socorro.lib.transaction.time.sleep') as sleep_mock: sleep_mock.return_value = None conn = object() connection_context = mock.MagicMock() connection_context.return_value.__enter__.return_value = conn quit_check = mock.MagicMock() exc = Exception('omg!') def fun(conn, call_count): # Raise exceptions to simulate failing; use call_count to keep # track call_count.append(1) raise exc call_count = [] with pytest.raises(Exception) as exc_info: retry(connection_context, quit_check, fun, call_count=call_count) # Assert retry runs out of backoffs and throws the last error assert exc_info.value == exc # Assert fun was called six times assert len(call_count) == 6 # quit_check gets called for every backoff, so 6 times assert quit_check.call_count == 6
def save_raw_crash(self, raw_crash, dumps, crash_id): retry( self.connection_source, self.quit_check, self.do_save_raw_crash, raw_crash=raw_crash, dumps=dumps, crash_id=crash_id )
def test_fine(self): conn = object() connection_context = mock.MagicMock() connection_context.return_value.__enter__.return_value = conn quit_check = mock.MagicMock() fun = mock.MagicMock() retry(connection_context, quit_check, fun) # Assert fun was called with the connection as the first argument fun.assert_called_with(conn) # Assert quit_check was not called at all assert not quit_check.called
def _suppress_duplicate_jobs(self, crash_id, acknowledgement_token): """if this crash is in the cache, then it is already in progress and this is a duplicate. Acknowledge it, then return to True to let the caller know to skip on to the next crash.""" if crash_id in self.acknowledgement_token_cache: # reject this crash - it's already being processsed self.logger.info('duplicate job: %s is already in progress', crash_id) # ack this retry(connection_context=self.rabbitmq, quit_check=self.quit_check_callback, fun=self._ack_crash, crash_id=crash_id, acknowledgement_token=acknowledgement_token) return True return False
def get_raw_crash(self, crash_id): return retry( self.connection_source, self.quit_check, self.do_get_raw_crash, crash_id=crash_id, json_object_hook=self.config.json_object_hook )
def get_unredacted_processed(self, crash_id): return retry( self.connection_source, self.quit_check, self._do_get_unredacted_processed, crash_id=crash_id, json_object_hook=self.config.json_object_hook, )
def get_raw_dump(self, crash_id, name=None): return retry( self.connection_source, self.quit_check, self.do_get_raw_dump, crash_id=crash_id, name=name, )
def get_raw_dump(self, crash_id, name=None): return retry( self.connection_source, self.quit_check, self.do_get_raw_dump, crash_id=crash_id, name=name )
def _suppress_duplicate_jobs(self, crash_id, acknowledgement_token): """if this crash is in the cache, then it is already in progress and this is a duplicate. Acknowledge it, then return to True to let the caller know to skip on to the next crash.""" if crash_id in self.acknowledgement_token_cache: # reject this crash - it's already being processsed self.logger.info('duplicate job: %s is already in progress', crash_id) # ack this retry( connection_context=self.rabbitmq, quit_check=self.quit_check_callback, fun=self._ack_crash, crash_id=crash_id, acknowledgement_token=acknowledgement_token ) return True return False
def get_raw_crash(self, crash_id): return retry( self.connection_source, self.quit_check, self.do_get_raw_crash, crash_id=crash_id, json_object_hook=self.config.json_object_hook, )
def get_unredacted_processed(self, crash_id): return retry( self.connection_source, self.quit_check, self._do_get_unredacted_processed, crash_id=crash_id, json_object_hook=self.config.json_object_hook )
def test_fine(self): conn = object() connection_context = mock.MagicMock() connection_context.return_value.__enter__.return_value = conn quit_check = mock.MagicMock() fun = mock.MagicMock() retry( connection_context, quit_check, fun ) # Assert fun was called with the connection as the first argument fun.assert_called_with(conn) # Assert quit_check was not called at all assert not quit_check.called
def get_raw_dumps(self, crash_id): """Fetch raw dumps :returns: MemoryDumpsMapping """ return retry(self.connection_source, self.quit_check, self.do_get_raw_dumps, crash_id=crash_id)
def get_raw_dumps(self, crash_id): """Fetch raw dumps :returns: MemoryDumpsMapping """ return retry( self.connection_source, self.quit_check, self.do_get_raw_dumps, crash_id=crash_id )
def _consume_acknowledgement_queue(self): """The acknowledgement of the processing of each crash_id yielded from the 'new_crashes' method must take place on the same connection that the crash_id came from. The crash_ids are queued in the 'acknowledgment_queue'. That queue is consumed by the QueuingThread""" try: while True: crash_id_to_be_acknowledged = self.acknowledgment_queue.get_nowait() try: acknowledgement_token = self.acknowledgement_token_cache[ crash_id_to_be_acknowledged ] retry( connection_context=self.rabbitmq, quit_check=self.quit_check_callback, fun=self._ack_crash, crash_id=crash_id_to_be_acknowledged, acknowledgement_token=acknowledgement_token ) del self.acknowledgement_token_cache[crash_id_to_be_acknowledged] except KeyError: self.logger.warning( 'RabbitMQCrashQueue tried to acknowledge crash %s, which was not in cache', crash_id_to_be_acknowledged, exc_info=True ) except Exception: self.logger.error( 'RabbitMQCrashQueue unexpected failure on %s', crash_id_to_be_acknowledged, exc_info=True ) except Empty: pass # nothing to do with an empty queue
def __iter__(self): """Return an iterator over crashes from RabbitMQ. Each crash is a tuple of the ``(args, kwargs)`` variety. The lone arg is a crash ID, and the kwargs contain only a callback function which the FTS app will call to send an ack to Rabbit after processing is complete. """ self._consume_acknowledgement_queue() queues = [ self.rabbitmq.config.priority_queue_name, self.rabbitmq.config.standard_queue_name, self.rabbitmq.config.reprocessing_queue_name, self.rabbitmq.config.priority_queue_name, ] while True: for queue in queues: method_frame, header_frame, body = retry( connection_context=self.rabbitmq, quit_check=self.quit_check_callback, fun=self._basic_get, queue=queue ) # The body is always a string, so convert it to a string if body: body = body.decode('utf-8') if method_frame and self._suppress_duplicate_jobs(body, method_frame): continue if method_frame: break # must consume ack queue before testing for end of iterator # or the last job won't get ack'd self._consume_acknowledgement_queue() if not method_frame: # there was nothing in the queue - leave the iterator return self.acknowledgement_token_cache[body] = method_frame yield ( (body,), {'finished_func': partial(self.ack_crash, body)} ) queues.reverse()
def __iter__(self): """Return an iterator over crashes from RabbitMQ. Each crash is a tuple of the ``(args, kwargs)`` variety. The lone arg is a crash ID, and the kwargs contain only a callback function which the FTS app will call to send an ack to Rabbit after processing is complete. """ self._consume_acknowledgement_queue() queues = [ self.rabbitmq.config.priority_queue_name, self.rabbitmq.config.standard_queue_name, self.rabbitmq.config.reprocessing_queue_name, self.rabbitmq.config.priority_queue_name, ] while True: for queue in queues: method_frame, header_frame, body = retry( connection_context=self.rabbitmq, quit_check=self.quit_check_callback, fun=self._basic_get, queue=queue) # The body is always a string, so convert it to a string if body: body = body.decode('utf-8') if method_frame and self._suppress_duplicate_jobs( body, method_frame): continue if method_frame: break # must consume ack queue before testing for end of iterator # or the last job won't get ack'd self._consume_acknowledgement_queue() if not method_frame: # there was nothing in the queue - leave the iterator return self.acknowledgement_token_cache[body] = method_frame yield ((body, ), {'finished_func': partial(self.ack_crash, body)}) queues.reverse()
def new_crashes(self): """This generator fetches crash_ids from RabbitMQ.""" # We've set up RabbitMQ to require acknowledgement of processing of a # crash_id from this generator. It is the responsibility of the # consumer of the crash_id to tell this instance of the class when has # completed its work on the crash_id. That is done with the call to # 'ack_crash' below. Because RabbitMQ connections are not thread safe, # only the thread that read the crash may acknowledge it. 'ack_crash' # queues the crash_id. The '_consume_acknowledgement_queue' function # is run to send acknowledgments back to RabbitMQ self._consume_acknowledgement_queue() queues = [ self.rabbitmq.config.priority_queue_name, self.rabbitmq.config.standard_queue_name, self.rabbitmq.config.reprocessing_queue_name, self.rabbitmq.config.priority_queue_name, ] while True: for queue in queues: method_frame, header_frame, body = retry(self.rabbitmq, self.quit_check, self._basic_get, queue=queue) # The body is always a string, so convert it to a string if body: body = body.decode('utf-8') if method_frame and self._suppress_duplicate_jobs( body, method_frame): continue if method_frame: break # must consume ack queue before testing for end of iterator # or the last job won't get ack'd self._consume_acknowledgement_queue() if not method_frame: # there was nothing in the queue - leave the iterator return self.acknowledgement_token_cache[body] = method_frame yield body queues.reverse()