def _test_loaded_in_correct_order(
self, enrichment_manager: EnrichmentManager, enrichment_loaders: Iterable[EnrichmentLoader]
):
"""
Tests that the given enrichment manager applies enrichments defined be the given loaders in the correct order.
:param enrichment_manager: enrichment manager
:param enrichment_loaders: enrichment loaders
"""
logging.root.setLevel(logging.CRITICAL)
cookie = Cookie("the_identifier")
enrichment_loaders_priority_queue = PriorityQueue()
for enrichment_loader in enrichment_loaders:
if enrichment_loader.can_enrich(cookie):
enrichment_loaders_priority_queue.put(enrichment_loader)
enrichment = enrichment_manager.next_enrichment(cookie)
while enrichment is not None:
expected_enrichment_loader = enrichment_loaders_priority_queue.get() # type: EnrichmentLoader
expected_enrichment = expected_enrichment_loader.load_enrichment(cookie)
self.assertEqual(enrichment, expected_enrichment)
cookie.enrich(enrichment)
expected_enrichment_loader.can_enrich = MagicMock(return_value=False)
enrichment = enrichment_manager.next_enrichment(cookie)
self.assertTrue(enrichment_loaders_priority_queue.empty())
class TestCookie(unittest.TestCase):
"""
Tests for `Cookie`.
"""
_IDENTIFIER = "id"
def setUp(self):
self._cookie = Cookie(TestCookie._IDENTIFIER)
def test_enrich(self):
enrichment = Enrichment("source", datetime(1, 1, 1), Metadata())
self._cookie.enrich(enrichment)
self.assertCountEqual(self._cookie.enrichments, [enrichment])
def _get_cookie(self, identifier: str) -> Optional[Cookie]:
"""
This method *actually* fetches the Cookie, but is not targeted
by the rate limiter
"""
_, doc = self._queue.get_by_identifier(identifier) or (None, None)
if doc is None:
return None
cookie = Cookie(identifier)
cookie.enrichments = EnrichmentCollection(self._metadata.get_metadata(identifier))
return cookie
def _load_cookie(self, identifier: str) -> Cookie:
"""
Loads a cookie with the given identifier.
:param identifier: the identifier of the cookie to load
:return: the loaded cookie
"""
# FIXME: There should be a `CookieJar` to do all this (see issue:
# https://github.com/wtsi-hgi/cookie-monster/issues/44)
url = "%s?identifier=%s" % (self.api_location, quote(identifier))
request = Request(url, None, {"Accept": "application/json"})
cookie_as_json = json.loads(urlopen(request).read().decode("utf-8"))
# FIXME: There should be a `CookieJSONDecoder`
cookie = Cookie(identifier)
for enrichment in _CookieLoadingDict._ENRICHMENT_JSON_DECODER.decode_parsed(cookie_as_json["enrichments"]):
assert isinstance(enrichment, Enrichment)
cookie.enrich(enrichment)
return cookie
def setUp(self):
self._cookie = Cookie(TestCookie._IDENTIFIER)
class TestCookie(unittest.TestCase):
"""
Tests for `Cookie`.
"""
_IDENTIFIER = "id"
def setUp(self):
self._cookie = Cookie(TestCookie._IDENTIFIER)
def test_enrich_with_no_previous(self):
enrichment = Enrichment("source", datetime(1, 2, 3), Metadata())
self._cookie.enrich(enrichment)
self.assertEqual(self._cookie.enrichments, [enrichment])
def test_enrich_with_older_previous(self):
older_enrichment = Enrichment("source", datetime(1, 1, 1), Metadata())
self._cookie.enrich(older_enrichment)
enrichment = Enrichment("source", datetime(2, 2, 2), Metadata())
self._cookie.enrich(enrichment)
self.assertEqual(self._cookie.enrichments, [older_enrichment, enrichment])
def test_enrich_with_newer_previous(self):
newer_enrichment = Enrichment("source", datetime(3, 3, 3), Metadata())
self._cookie.enrich(newer_enrichment)
enrichment = Enrichment("source", datetime(2, 2, 2), Metadata())
self._cookie.enrich(enrichment)
self.assertEqual(self._cookie.enrichments, [enrichment, newer_enrichment])
def test_enrich_with_older_and_newer_previous(self):
older_enrichment = Enrichment("source", datetime(1, 1, 1), Metadata())
self._cookie.enrich(older_enrichment)
newer_enrichment = Enrichment("source", datetime(3, 3, 3), Metadata())
self._cookie.enrich(newer_enrichment)
enrichment = Enrichment("source", datetime(2, 2, 2), Metadata())
self._cookie.enrich(enrichment)
self.assertEqual(self._cookie.enrichments, [older_enrichment, enrichment, newer_enrichment])
def test_get_most_recent_enrichment_from_source_when_none_from_source(self):
self.assertIsNone(self._cookie.get_most_recent_enrichment_from_source("source"))
def test_get_most_recent_enrichment_from_source_when_multiple_from_source(self):
older_enrichment = Enrichment("source", datetime(1, 1, 1), Metadata())
self._cookie.enrich(older_enrichment)
newer_enrichment = Enrichment("source", datetime(2, 2, 2), Metadata())
self._cookie.enrich(newer_enrichment)
other_enrichment = Enrichment("other", datetime(3, 3, 3), Metadata())
self._cookie.enrich(other_enrichment)
self.assertEqual(self._cookie.get_most_recent_enrichment_from_source("source"), newer_enrichment)
def test_get_enrichment_sources_when_no_enrichments(self):
self.assertEqual(len(self._cookie.get_enrichment_sources()), 0)
def test_get_enrichment_sources_when_enrichments(self):
self._cookie.enrich(Enrichment("source_1", datetime(1, 1, 1), Metadata()))
self._cookie.enrich(Enrichment("source_1", datetime(2, 1, 1), Metadata()))
self._cookie.enrich(Enrichment("source_2", datetime(1, 1, 1), Metadata()))
self.assertCountEqual(self._cookie.get_enrichment_sources(), ["source_1", "source_2"])
def test_enrichment_no_diff(self):
self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata()))
self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata()))
diffs = self._cookie.get_enrichment_changes_from_source('source')
self.assertEqual(len(diffs), 0)
def test_enrichment_diff(self):
self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789})))
self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999})))
diffs = self._cookie.get_enrichment_changes_from_source('source')
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertIsInstance(diff, EnrichmentDiff)
self.assertEqual(diff.source, 'source')
self.assertEqual(diff.timestamp, datetime(2, 2, 2))
self.assertTrue(diff.is_different())
self.assertEqual(diff.additions, Metadata({'xyz': 123, 'quux': 999}))
self.assertEqual(diff.deletions, Metadata({'foo': 123, 'quux': 789}))
def test_enrichment_diff_by_key(self):
self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789})))
self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999})))
diffs = self._cookie.get_enrichment_changes_from_source('source', 'foo')
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertEqual(diff.additions, Metadata())
self.assertEqual(diff.deletions, Metadata({'foo': 123}))
def test_enrichment_diff_by_keys(self):
self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789})))
self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999})))
diffs = self._cookie.get_enrichment_changes_from_source('source', ['bar', 'quux'])
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertEqual(diff.additions, Metadata({'quux': 999}))
self.assertEqual(diff.deletions, Metadata({'quux': 789}))
def test_enrichment_diff_from_timestamp(self):
self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata()))
self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'foo': 123})))
self._cookie.enrich(Enrichment('source', datetime(3, 3, 3), Metadata({'bar': 123})))
all_diffs = self._cookie.get_enrichment_changes_from_source('source')
self.assertEqual(len(all_diffs), 2)
since_diffs = self._cookie.get_enrichment_changes_from_source('source', since=datetime(3, 3, 3))
self.assertEqual(len(since_diffs), 1)
diff = since_diffs[0]
self.assertEqual(diff.timestamp, datetime(3, 3, 3))
self.assertEqual(diff.additions, Metadata({'bar': 123}))
self.assertEqual(diff.deletions, Metadata({'foo': 123}))
def _matches(cookie: Cookie, context: Context) -> bool:
enrichment_from_source = cookie.get_most_recent_enrichment_from_source(HASH_ENRICHMENT_LOADER_ID)
if enrichment_from_source is None:
return False
return KEY in enrichment_from_source.metadata
def record_fail_if_changed(cookie: Cookie) -> bool:
nonlocal change_detected_in_next_rule
if source in cookie.get_enrichment_sources():
change_detected_in_next_rule = True
def cookie_changer(cookie: Cookie) -> bool:
enrichment = Enrichment(source, datetime(year=2000, month=1, day=1), Metadata())
cookie.enrich(enrichment)
return False
def _matches(cookie: Cookie, context: HgiContext) -> bool:
if IRODS_SOURCE not in cookie.get_enrichment_sources():
return False
return "not_interesting" in cookie.identifier
