def _test_loaded_in_correct_order( self, enrichment_manager: EnrichmentManager, enrichment_loaders: Iterable[EnrichmentLoader] ): """ Tests that the given enrichment manager applies enrichments defined be the given loaders in the correct order. :param enrichment_manager: enrichment manager :param enrichment_loaders: enrichment loaders """ logging.root.setLevel(logging.CRITICAL) cookie = Cookie("the_identifier") enrichment_loaders_priority_queue = PriorityQueue() for enrichment_loader in enrichment_loaders: if enrichment_loader.can_enrich(cookie): enrichment_loaders_priority_queue.put(enrichment_loader) enrichment = enrichment_manager.next_enrichment(cookie) while enrichment is not None: expected_enrichment_loader = enrichment_loaders_priority_queue.get() # type: EnrichmentLoader expected_enrichment = expected_enrichment_loader.load_enrichment(cookie) self.assertEqual(enrichment, expected_enrichment) cookie.enrich(enrichment) expected_enrichment_loader.can_enrich = MagicMock(return_value=False) enrichment = enrichment_manager.next_enrichment(cookie) self.assertTrue(enrichment_loaders_priority_queue.empty())
class TestCookie(unittest.TestCase): """ Tests for `Cookie`. """ _IDENTIFIER = "id" def setUp(self): self._cookie = Cookie(TestCookie._IDENTIFIER) def test_enrich(self): enrichment = Enrichment("source", datetime(1, 1, 1), Metadata()) self._cookie.enrich(enrichment) self.assertCountEqual(self._cookie.enrichments, [enrichment])
def _get_cookie(self, identifier: str) -> Optional[Cookie]: """ This method *actually* fetches the Cookie, but is not targeted by the rate limiter """ _, doc = self._queue.get_by_identifier(identifier) or (None, None) if doc is None: return None cookie = Cookie(identifier) cookie.enrichments = EnrichmentCollection(self._metadata.get_metadata(identifier)) return cookie
def _load_cookie(self, identifier: str) -> Cookie: """ Loads a cookie with the given identifier. :param identifier: the identifier of the cookie to load :return: the loaded cookie """ # FIXME: There should be a `CookieJar` to do all this (see issue: # https://github.com/wtsi-hgi/cookie-monster/issues/44) url = "%s?identifier=%s" % (self.api_location, quote(identifier)) request = Request(url, None, {"Accept": "application/json"}) cookie_as_json = json.loads(urlopen(request).read().decode("utf-8")) # FIXME: There should be a `CookieJSONDecoder` cookie = Cookie(identifier) for enrichment in _CookieLoadingDict._ENRICHMENT_JSON_DECODER.decode_parsed(cookie_as_json["enrichments"]): assert isinstance(enrichment, Enrichment) cookie.enrich(enrichment) return cookie
def setUp(self): self._cookie = Cookie(TestCookie._IDENTIFIER)
class TestCookie(unittest.TestCase): """ Tests for `Cookie`. """ _IDENTIFIER = "id" def setUp(self): self._cookie = Cookie(TestCookie._IDENTIFIER) def test_enrich_with_no_previous(self): enrichment = Enrichment("source", datetime(1, 2, 3), Metadata()) self._cookie.enrich(enrichment) self.assertEqual(self._cookie.enrichments, [enrichment]) def test_enrich_with_older_previous(self): older_enrichment = Enrichment("source", datetime(1, 1, 1), Metadata()) self._cookie.enrich(older_enrichment) enrichment = Enrichment("source", datetime(2, 2, 2), Metadata()) self._cookie.enrich(enrichment) self.assertEqual(self._cookie.enrichments, [older_enrichment, enrichment]) def test_enrich_with_newer_previous(self): newer_enrichment = Enrichment("source", datetime(3, 3, 3), Metadata()) self._cookie.enrich(newer_enrichment) enrichment = Enrichment("source", datetime(2, 2, 2), Metadata()) self._cookie.enrich(enrichment) self.assertEqual(self._cookie.enrichments, [enrichment, newer_enrichment]) def test_enrich_with_older_and_newer_previous(self): older_enrichment = Enrichment("source", datetime(1, 1, 1), Metadata()) self._cookie.enrich(older_enrichment) newer_enrichment = Enrichment("source", datetime(3, 3, 3), Metadata()) self._cookie.enrich(newer_enrichment) enrichment = Enrichment("source", datetime(2, 2, 2), Metadata()) self._cookie.enrich(enrichment) self.assertEqual(self._cookie.enrichments, [older_enrichment, enrichment, newer_enrichment]) def test_get_most_recent_enrichment_from_source_when_none_from_source(self): self.assertIsNone(self._cookie.get_most_recent_enrichment_from_source("source")) def test_get_most_recent_enrichment_from_source_when_multiple_from_source(self): older_enrichment = Enrichment("source", datetime(1, 1, 1), Metadata()) self._cookie.enrich(older_enrichment) newer_enrichment = Enrichment("source", datetime(2, 2, 2), Metadata()) self._cookie.enrich(newer_enrichment) other_enrichment = Enrichment("other", datetime(3, 3, 3), Metadata()) self._cookie.enrich(other_enrichment) self.assertEqual(self._cookie.get_most_recent_enrichment_from_source("source"), newer_enrichment) def test_get_enrichment_sources_when_no_enrichments(self): self.assertEqual(len(self._cookie.get_enrichment_sources()), 0) def test_get_enrichment_sources_when_enrichments(self): self._cookie.enrich(Enrichment("source_1", datetime(1, 1, 1), Metadata())) self._cookie.enrich(Enrichment("source_1", datetime(2, 1, 1), Metadata())) self._cookie.enrich(Enrichment("source_2", datetime(1, 1, 1), Metadata())) self.assertCountEqual(self._cookie.get_enrichment_sources(), ["source_1", "source_2"]) def test_enrichment_no_diff(self): self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata())) self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata())) diffs = self._cookie.get_enrichment_changes_from_source('source') self.assertEqual(len(diffs), 0) def test_enrichment_diff(self): self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789}))) self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999}))) diffs = self._cookie.get_enrichment_changes_from_source('source') self.assertEqual(len(diffs), 1) diff = diffs[0] self.assertIsInstance(diff, EnrichmentDiff) self.assertEqual(diff.source, 'source') self.assertEqual(diff.timestamp, datetime(2, 2, 2)) self.assertTrue(diff.is_different()) self.assertEqual(diff.additions, Metadata({'xyz': 123, 'quux': 999})) self.assertEqual(diff.deletions, Metadata({'foo': 123, 'quux': 789})) def test_enrichment_diff_by_key(self): self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789}))) self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999}))) diffs = self._cookie.get_enrichment_changes_from_source('source', 'foo') self.assertEqual(len(diffs), 1) diff = diffs[0] self.assertEqual(diff.additions, Metadata()) self.assertEqual(diff.deletions, Metadata({'foo': 123})) def test_enrichment_diff_by_keys(self): self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789}))) self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999}))) diffs = self._cookie.get_enrichment_changes_from_source('source', ['bar', 'quux']) self.assertEqual(len(diffs), 1) diff = diffs[0] self.assertEqual(diff.additions, Metadata({'quux': 999})) self.assertEqual(diff.deletions, Metadata({'quux': 789})) def test_enrichment_diff_from_timestamp(self): self._cookie.enrich(Enrichment('source', datetime(1, 1, 1), Metadata())) self._cookie.enrich(Enrichment('source', datetime(2, 2, 2), Metadata({'foo': 123}))) self._cookie.enrich(Enrichment('source', datetime(3, 3, 3), Metadata({'bar': 123}))) all_diffs = self._cookie.get_enrichment_changes_from_source('source') self.assertEqual(len(all_diffs), 2) since_diffs = self._cookie.get_enrichment_changes_from_source('source', since=datetime(3, 3, 3)) self.assertEqual(len(since_diffs), 1) diff = since_diffs[0] self.assertEqual(diff.timestamp, datetime(3, 3, 3)) self.assertEqual(diff.additions, Metadata({'bar': 123})) self.assertEqual(diff.deletions, Metadata({'foo': 123}))
def _matches(cookie: Cookie, context: Context) -> bool: enrichment_from_source = cookie.get_most_recent_enrichment_from_source(HASH_ENRICHMENT_LOADER_ID) if enrichment_from_source is None: return False return KEY in enrichment_from_source.metadata
def record_fail_if_changed(cookie: Cookie) -> bool: nonlocal change_detected_in_next_rule if source in cookie.get_enrichment_sources(): change_detected_in_next_rule = True
def cookie_changer(cookie: Cookie) -> bool: enrichment = Enrichment(source, datetime(year=2000, month=1, day=1), Metadata()) cookie.enrich(enrichment) return False
def _matches(cookie: Cookie, context: HgiContext) -> bool: if IRODS_SOURCE not in cookie.get_enrichment_sources(): return False return "not_interesting" in cookie.identifier