def _connect_retrieval_manager_to_cookie_jar(retrieval_manager: RetrievalManager, cookie_jar: CookieJar,
number_of_threads: int=None, logger: Logger=PythonLoggingLogger()):
"""
Connect the given retrieval manager to the given cookie jar.
:param retrieval_manager: the retrieval manager
:param cookie_jar: the cookie jar to connect to
:param number_of_threads: the number of threads to use when putting cookies into the jar
"""
still_to_enrich = 0
still_to_enrich_lock = Lock()
thread_pool = ThreadPoolExecutor(max_workers=number_of_threads)
def timed_enrichment(target: str, enrichment: Enrichment):
nonlocal still_to_enrich
logging.debug("Enriching \"%s\" with: %s" % (target, enrichment))
started_at = time.monotonic()
cookie_jar.enrich_cookie(target, enrichment)
time_taken = time.monotonic() - started_at
logger.record(MEASUREMENT_ENRICH_TIME, time_taken)
logging.info("Took %f seconds (wall time) to enrich cookie with path \"%s\"" % (time_taken, target))
with still_to_enrich_lock:
still_to_enrich -= 1
logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich)
def put_updates_in_cookie_jar(update_collection: UpdateCollection):
nonlocal still_to_enrich
for update in update_collection:
enrichment = Enrichment("irods_update", datetime.now(), update.metadata)
with still_to_enrich_lock:
still_to_enrich += 1
logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich)
thread_pool.submit(timed_enrichment, update.target, enrichment)
retrieval_manager.add_listener(put_updates_in_cookie_jar)
def _connect_retrieval_manager_to_cookie_jar(retrieval_manager: RetrievalManager, cookie_jar: CookieJar,
number_of_threads: int=None, logger: Logger=PythonLoggingLogger()):
"""
Connect the given retrieval manager to the given cookie jar.
:param retrieval_manager: the retrieval manager
:param cookie_jar: the cookie jar to connect to
:param number_of_threads: the number of threads to use when putting cookies into the jar
"""
still_to_enrich = 0
still_to_enrich_lock = Lock()
thread_pool = ThreadPoolExecutor(max_workers=number_of_threads)
def timed_enrichment(target: str, enrichment: Enrichment):
nonlocal still_to_enrich
logging.debug("Enriching \"%s\" with: %s" % (target, enrichment))
started_at = time.monotonic()
try:
# Let's leave this here, as it's very important that the
# enrichment succeeds and we need to know about it in detail
# if/when it doesn't!
cookie_jar.enrich_cookie(target, enrichment)
except:
logging.exception("Enrichment of \"%s\" failed!!", target)
raise
time_taken = time.monotonic() - started_at
logger.record(MEASUREMENT_ENRICH_TIME, time_taken)
logging.info("Took %f seconds (wall time) to enrich cookie with path \"%s\"" % (time_taken, target))
with still_to_enrich_lock:
still_to_enrich -= 1
logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich)
def put_updates_in_cookie_jar(update_collection: UpdateCollection):
nonlocal still_to_enrich
for update in update_collection:
enrichment = Enrichment(IRODS_UPDATE_ENRICHMENT, update.timestamp, update.metadata)
with still_to_enrich_lock:
still_to_enrich += 1
logger.record(MEASUREMENT_STILL_TO_ENRICH, still_to_enrich)
thread_pool.submit(timed_enrichment, update.target, enrichment)
retrieval_manager.add_listener(put_updates_in_cookie_jar)
def _connect_retrieval_manager_to_since_file(retrieval_manager: RetrievalManager,
config_location: str):
"""
Connect the given retrieval manager to the "since" file
@param retrieval_manager The retrieval manager
@param config_location Directory containing since file
"""
since_file = os.path.join(config_location, "since")
def update_since_file(update_collection: UpdateCollection):
nonlocal since_file
if update_collection:
last_retrieval_time = update_collection.get_most_recent()[0].timestamp
with open(since_file, "w") as f:
f.write(str(int(last_retrieval_time.timestamp())))
retrieval_manager.add_listener(update_since_file)
def setUp(self):
super().setUp()
# Create retrieval manager
self.retrieval_manager = RetrievalManager(self.update_mapper, self.logger)
class TestRetrievalManager(_BaseRetrievalManagerTest):
"""
Test cases for `RetrievalManager`.
"""
def setUp(self):
super().setUp()
# Create retrieval manager
self.retrieval_manager = RetrievalManager(self.update_mapper, self.logger)
def test_run_with_updates(self):
# Setup
listener = MagicMock()
self.retrieval_manager.add_listener(listener)
self.logger.add = MagicMock()
# Call SUT method
self.retrieval_manager.run(SINCE)
# Assert that retrieves updates from source
self.update_mapper.get_all_since.assert_called_once_with(SINCE)
# Assert that updates listeners are called
listener.assert_called_once_with(self.updates)
# Assert that retrieval is logged
self._assert_logged_updated(self.updates)
def test_run_without_updates(self):
# Setup
listener = MagicMock()
self.retrieval_manager.add_listener(listener)
self.logger.add = MagicMock()
self.updates.clear()
self.retrieval_manager._retrieved_updates_since = CURRENT_MONOTONIC_TIME - (24 * 60 * 60)
# Call SUT method
self.retrieval_manager.run(SINCE)
# Assert that retrieves updates from source
self.update_mapper.get_all_since.assert_called_once_with(SINCE)
# Assert that updates listener has not been called given that there are no updates
listener.assert_not_called()
# Assert that retrieval is logged
self._assert_logged_updated(self.updates)
def _assert_logged_updated(self, updates: UpdateCollection):
"""
TODO
:param updates:
:return:
"""
self.assertEqual(self.logger.record.call_count, 1)
args = self.logger.record.call_args[0]
self.assertEqual(args[0], MEASURED_RETRIEVAL)
self.assertEqual(DatetimeISOFormatJSONDecoder().decode(args[1][MEASURED_RETRIEVAL_STARTED_AT]),
CURRENT_CLOCK_TIME)
self.assertGreaterEqual(args[1][MEASURED_RETRIEVAL_DURATION], TIME_TAKEN_TO_DO_RETRIEVE)
self.assertEqual(args[1][MEASURED_RETRIEVAL_UPDATE_COUNT], len(updates))
logged_most_recent_retrived = args[1][MEASURED_RETRIEVAL_MOST_RECENT_RETRIEVED]
if len(updates) > 0:
self.assertEqual(DatetimeISOFormatJSONDecoder().decode(logged_most_recent_retrived),
updates.get_most_recent()[0].timestamp)
else:
self.assertIsNone(logged_most_recent_retrived)
def do_query(*args):
RetrievalManager._get_monotonic_time = MagicMock(
return_value=RetrievalManager._get_monotonic_time() + RETRIEVAL_PERIOD + TIME_TAKEN_TO_DO_RETRIEVE)
return self.updates
