def _connect_retrieval_manager_to_cookie_jar(retrieval_manager: RetrievalManager, cookie_jar: CookieJar, number_of_threads: int=None, logger: Logger=PythonLoggingLogger()): """ Connect the given retrieval manager to the given cookie jar. :param retrieval_manager: the retrieval manager :param cookie_jar: the cookie jar to connect to :param number_of_threads: the number of threads to use when putting cookies into the jar """ still_to_enrich = 0 still_to_enrich_lock = Lock() thread_pool = ThreadPoolExecutor(max_workers=number_of_threads) def timed_enrichment(target: str, enrichment: Enrichment): nonlocal still_to_enrich logging.debug("Enriching \"%s\" with: %s" % (target, enrichment)) started_at = time.monotonic() cookie_jar.enrich_cookie(target, enrichment) time_taken = time.monotonic() - started_at logger.record(MEASUREMENT_ENRICH_TIME, time_taken) logging.info("Took %f seconds (wall time) to enrich cookie with path \"%s\"" % (time_taken, target)) with still_to_enrich_lock: still_to_enrich -= 1 logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich) def put_updates_in_cookie_jar(update_collection: UpdateCollection): nonlocal still_to_enrich for update in update_collection: enrichment = Enrichment("irods_update", datetime.now(), update.metadata) with still_to_enrich_lock: still_to_enrich += 1 logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich) thread_pool.submit(timed_enrichment, update.target, enrichment) retrieval_manager.add_listener(put_updates_in_cookie_jar)
def _connect_retrieval_manager_to_cookie_jar(retrieval_manager: RetrievalManager, cookie_jar: CookieJar, number_of_threads: int=None, logger: Logger=PythonLoggingLogger()): """ Connect the given retrieval manager to the given cookie jar. :param retrieval_manager: the retrieval manager :param cookie_jar: the cookie jar to connect to :param number_of_threads: the number of threads to use when putting cookies into the jar """ still_to_enrich = 0 still_to_enrich_lock = Lock() thread_pool = ThreadPoolExecutor(max_workers=number_of_threads) def timed_enrichment(target: str, enrichment: Enrichment): nonlocal still_to_enrich logging.debug("Enriching \"%s\" with: %s" % (target, enrichment)) started_at = time.monotonic() try: # Let's leave this here, as it's very important that the # enrichment succeeds and we need to know about it in detail # if/when it doesn't! cookie_jar.enrich_cookie(target, enrichment) except: logging.exception("Enrichment of \"%s\" failed!!", target) raise time_taken = time.monotonic() - started_at logger.record(MEASUREMENT_ENRICH_TIME, time_taken) logging.info("Took %f seconds (wall time) to enrich cookie with path \"%s\"" % (time_taken, target)) with still_to_enrich_lock: still_to_enrich -= 1 logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich) def put_updates_in_cookie_jar(update_collection: UpdateCollection): nonlocal still_to_enrich for update in update_collection: enrichment = Enrichment(IRODS_UPDATE_ENRICHMENT, update.timestamp, update.metadata) with still_to_enrich_lock: still_to_enrich += 1 logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich) thread_pool.submit(timed_enrichment, update.target, enrichment) retrieval_manager.add_listener(put_updates_in_cookie_jar)
def _connect_retrieval_manager_to_since_file(retrieval_manager: RetrievalManager, config_location: str): """ Connect the given retrieval manager to the "since" file @param retrieval_manager The retrieval manager @param config_location Directory containing since file """ since_file = os.path.join(config_location, "since") def update_since_file(update_collection: UpdateCollection): nonlocal since_file if update_collection: last_retrieval_time = update_collection.get_most_recent()[0].timestamp with open(since_file, "w") as f: f.write(str(int(last_retrieval_time.timestamp()))) retrieval_manager.add_listener(update_since_file)
def setUp(self): super().setUp() # Create retrieval manager self.retrieval_manager = RetrievalManager(self.update_mapper, self.logger)
class TestRetrievalManager(_BaseRetrievalManagerTest): """ Test cases for `RetrievalManager`. """ def setUp(self): super().setUp() # Create retrieval manager self.retrieval_manager = RetrievalManager(self.update_mapper, self.logger) def test_run_with_updates(self): # Setup listener = MagicMock() self.retrieval_manager.add_listener(listener) self.logger.add = MagicMock() # Call SUT method self.retrieval_manager.run(SINCE) # Assert that retrieves updates from source self.update_mapper.get_all_since.assert_called_once_with(SINCE) # Assert that updates listeners are called listener.assert_called_once_with(self.updates) # Assert that retrieval is logged self._assert_logged_updated(self.updates) def test_run_without_updates(self): # Setup listener = MagicMock() self.retrieval_manager.add_listener(listener) self.logger.add = MagicMock() self.updates.clear() self.retrieval_manager._retrieved_updates_since = CURRENT_MONOTONIC_TIME - (24 * 60 * 60) # Call SUT method self.retrieval_manager.run(SINCE) # Assert that retrieves updates from source self.update_mapper.get_all_since.assert_called_once_with(SINCE) # Assert that updates listener has not been called given that there are no updates listener.assert_not_called() # Assert that retrieval is logged self._assert_logged_updated(self.updates) def _assert_logged_updated(self, updates: UpdateCollection): """ TODO :param updates: :return: """ self.assertEqual(self.logger.record.call_count, 1) args = self.logger.record.call_args[0] self.assertEqual(args[0], MEASURED_RETRIEVAL) self.assertEqual(DatetimeISOFormatJSONDecoder().decode(args[1][MEASURED_RETRIEVAL_STARTED_AT]), CURRENT_CLOCK_TIME) self.assertGreaterEqual(args[1][MEASURED_RETRIEVAL_DURATION], TIME_TAKEN_TO_DO_RETRIEVE) self.assertEqual(args[1][MEASURED_RETRIEVAL_UPDATE_COUNT], len(updates)) logged_most_recent_retrived = args[1][MEASURED_RETRIEVAL_MOST_RECENT_RETRIEVED] if len(updates) > 0: self.assertEqual(DatetimeISOFormatJSONDecoder().decode(logged_most_recent_retrived), updates.get_most_recent()[0].timestamp) else: self.assertIsNone(logged_most_recent_retrived)
def do_query(*args): RetrievalManager._get_monotonic_time = MagicMock( return_value=RetrievalManager._get_monotonic_time() + RETRIEVAL_PERIOD + TIME_TAKEN_TO_DO_RETRIEVE) return self.updates