def setUp(self):
self.cookie_jar = InMemoryCookieJar()
self.rules = [Rule(lambda *args: False, MagicMock(), RULE_IDENTIFIER) for _ in range(10)]
self.cookie_jar.enrich_cookie(
COOKIE_IDENTIFIER, Enrichment("first", datetime(year=1, month=1, day=1, tzinfo=timezone.utc), Metadata()))
self.cookie = self.cookie_jar.fetch_cookie(COOKIE_IDENTIFIER)
self.processor = BasicProcessor(self.cookie_jar, [], [])
class TestThreadsMonitor(unittest.TestCase):
"""
Tests for `ThreadsMonitor`.
"""
def setUp(self):
self._logger = MagicMock()
self._cookie_jar = InMemoryCookieJar()
self._monitor = CookieJarMonitor(self._logger, timedelta(microseconds=1), self._cookie_jar)
def test_do_log_record(self):
self._cookie_jar.enrich_cookie("test", Enrichment("test", datetime(1, 2, 3), Metadata()))
self._monitor.do_log_record()
self.assertEqual(self._logger.record.call_count, 1)
call_args = self._logger.record.call_args[0]
self.assertEqual(call_args[0], MEASURED_COOKIE_JAR_STATUS)
self.assertEqual(call_args[1], {MEASURED_COOKIE_JAR_TO_PROCESS: 1})
def test_call_to_non_adapted_cookie_jar_method(self):
self._cookie_jar = InMemoryCookieJar()
self._cookie_jar._composite_cookie_jar = self._cookie_jar
listener = lambda: None
self._cookie_jar.add_listener(listener)
assert self._cookie_jar.get_listeners() == [listener]
self.assertEqual(self._cookie_jar.get_listeners, self._cookie_jar.get_listeners)
self.assertEqual(self._cookie_jar.get_listeners(), [listener])
def _change_time(self, cookie_jar: InMemoryCookieJar, change_time_to: int):
cookie_jar._get_time = MagicMock(return_value=change_time_to)
for end_time in cookie_jar._timers.keys():
if end_time <= change_time_to:
while len(cookie_jar._timers[end_time]) > 0:
timer = cookie_jar._timers[end_time][0] # type: Timer
timer.interval = 0
timer.run()
def setUp(self):
self._logger = MagicMock()
self._cookie_jar = MagicMock()
self._composite_methods = dict() # type: Dict[str, MagicMock]
for method_name in CookieJar.__abstractmethods__:
method = getattr(self._cookie_jar, method_name)
self._composite_methods[method_name] = method
add_cookie_jar_logging(self._cookie_jar, self._logger)
def create_magic_mock_cookie_jar() -> CookieJar:
"""
Creates a magic mock CookieJar - has the implementation of a CookieJar all methods are implemented using magic mocks
and therefore their usage is recorded.
:return: the created magic mock
"""
cookie_jar = InMemoryCookieJar()
original_get_next_for_processing = cookie_jar.get_next_for_processing
original_enrich_cookie = cookie_jar.enrich_cookie
original_mark_as_failed = cookie_jar.mark_as_complete
original_mark_as_completed = cookie_jar.mark_as_complete
original_mark_as_reprocess = cookie_jar.mark_for_processing
cookie_jar.get_next_for_processing = MagicMock(side_effect=original_get_next_for_processing)
cookie_jar.enrich_cookie = MagicMock(side_effect=original_enrich_cookie)
cookie_jar.mark_as_failed = MagicMock(side_effect=original_mark_as_failed)
cookie_jar.mark_as_complete = MagicMock(side_effect=original_mark_as_completed)
cookie_jar.mark_for_processing = MagicMock(side_effect=original_mark_as_reprocess)
return cookie_jar
class TestAddCookieJarLogging(unittest.TestCase):
"""
Tests for `add_cookie_jar_logging` method.
"""
def setUp(self):
self._logger = MagicMock()
self._cookie_jar = MagicMock()
self._composite_methods = dict() # type: Dict[str, MagicMock]
for method_name in CookieJar.__abstractmethods__:
method = getattr(self._cookie_jar, method_name)
method.__name__ = method_name
self._composite_methods[method_name] = method
add_cookie_jar_logging(self._cookie_jar, self._logger)
def test_enrich_cookie(self):
source = "my_source"
enrichment = Enrichment("source", datetime.min, Metadata())
self._cookie_jar.enrich_cookie(source, enrichment)
self._composite_methods[CookieJar.enrich_cookie.__name__].assert_called_once_with(source, enrichment)
self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.enrich_cookie.__name__])
def test_mark_as_failed(self):
identifier = "identifier"
requeue_delay = timedelta(seconds=5)
self._cookie_jar.mark_as_failed(identifier, requeue_delay)
self._composite_methods[CookieJar.mark_as_failed.__name__].assert_called_once_with(identifier, requeue_delay)
self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.mark_as_failed.__name__])
def test_mark_as_complete(self):
identifier = "identifier"
self._cookie_jar.mark_as_complete(identifier)
self._composite_methods[CookieJar.mark_as_complete.__name__].assert_called_once_with(identifier)
self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.mark_as_complete.__name__])
def test_mark_for_processing(self):
identifier = "identifier"
self._cookie_jar.mark_for_processing(identifier)
self._composite_methods[CookieJar.mark_for_processing.__name__].assert_called_once_with(identifier)
self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.mark_for_processing.__name__])
def test_get_next_for_processing(self):
self._cookie_jar.get_next_for_processing()
self._composite_methods[CookieJar.get_next_for_processing.__name__].assert_called_once_with()
self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.get_next_for_processing.__name__])
def test_queue_length(self):
self._cookie_jar.queue_length()
self._composite_methods[CookieJar.queue_length.__name__].assert_called_once_with()
self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.queue_length.__name__])
def test_call_to_non_adapted_cookie_jar_method(self):
self._cookie_jar = InMemoryCookieJar()
self._cookie_jar._composite_cookie_jar = self._cookie_jar
listener = lambda: None
self._cookie_jar.add_listener(listener)
assert self._cookie_jar.get_listeners() == [listener]
self.assertEqual(self._cookie_jar.get_listeners, self._cookie_jar.get_listeners)
self.assertEqual(self._cookie_jar.get_listeners(), [listener])
def test_call_to_non_cookie_jar_method(self):
self._cookie_jar.some_method = MagicMock()
self._cookie_jar.some_method()
self._cookie_jar.some_method.assert_called_once_with()
def test_call_to_non_cookie_jar_property(self):
self._cookie_jar.some_property = 1
self.assertEqual(self._cookie_jar.some_property, 1)
def _assert_measured(self, measured: str):
calls = self._logger.record.mock_calls
self.assertEqual(len(calls), 1)
args, kwargs = self._logger.record.call_args
self.assertEqual(args[0], measured)
class TestBasicProcessor(unittest.TestCase):
"""
Tests for `BasicProcessor`.
"""
def setUp(self):
self.cookie_jar = InMemoryCookieJar()
self.rules = [Rule(lambda *args: False, MagicMock(), RULE_IDENTIFIER) for _ in range(10)]
self.cookie_jar.enrich_cookie(
COOKIE_IDENTIFIER, Enrichment("first", datetime(year=1, month=1, day=1, tzinfo=timezone.utc), Metadata()))
self.cookie = self.cookie_jar.fetch_cookie(COOKIE_IDENTIFIER)
self.processor = BasicProcessor(self.cookie_jar, [], [])
def test_evaluate_rules_with_cookie_when_no_rules(self):
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
self.assertFalse(halt)
def test_evaluate_rules_with_cookie_when_no_matched_rules(self):
self.processor.rules = self.rules
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
for rule in self.rules:
rule._action.assert_not_called()
self.assertFalse(halt)
def test_evaluate_rules_with_cookie_when_matched_rules_and_no_termination(self):
extra_rules = [
Rule(lambda *args: True, MagicMock(return_value=False), RULE_IDENTIFIER),
Rule(lambda *args: True, MagicMock(return_value=False), RULE_IDENTIFIER)
]
self.processor.rules = self.rules + extra_rules
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
for rule in self.rules:
rule._action.assert_not_called()
for rule in extra_rules:
self.assertEqual(rule._action.call_count, 1)
self.assertEqual(rule._action.call_args[0][0].identifier, self.cookie.identifier)
self.assertFalse(halt)
def test_evaluate_rules_with_cookie_when_matched_rules_and_termination(self):
production_to_be_called = MagicMock(return_value=True)
extra_rules = [
Rule(lambda *args: True, lambda *args: False, Priority.MIN_PRIORITY),
Rule(lambda *args: True, production_to_be_called,
Priority.get_lower_priority_value(Priority.MAX_PRIORITY)),
Rule(lambda *args: False, lambda *args: False, Priority.MAX_PRIORITY)
]
self.processor.rules = self.rules + extra_rules
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
total_call_count = 0
for rule in self.rules:
total_call_count += rule._action.call_count
self.assertEqual(total_call_count, 0)
self.assertEqual(production_to_be_called.call_count, 1)
self.assertTrue(halt)
def test_evaluate_rules_with_cookie_does_not_allow_rule_to_change_cookie_for_subsequent_rules(self):
source = "my_enrichment"
change_detected_in_next_rule = False
def cookie_changer(cookie: Cookie) -> bool:
enrichment = Enrichment(source, datetime(year=2000, month=1, day=1), Metadata())
cookie.enrich(enrichment)
return False
def record_fail_if_changed(cookie: Cookie) -> bool:
nonlocal change_detected_in_next_rule
if source in cookie.get_enrichment_sources():
change_detected_in_next_rule = True
self.processor.rules = [
Rule(cookie_changer, self.rules[0]._action, RULE_IDENTIFIER, priority=Priority.MAX_PRIORITY),
Rule(record_fail_if_changed, self.rules[0]._action, RULE_IDENTIFIER, priority=Priority.MIN_PRIORITY)
]
self.processor.process_cookie(self.cookie)
self.assertFalse(change_detected_in_next_rule)
def test_evaluate_rules_with_cookie_when_no_matched__does_not_enrich_with_rule_application_log(self):
self.processor.rules = self.rules
_ = self.processor.evaluate_rules_with_cookie(self.cookie)
cookie = self.cookie_jar.fetch_cookie(self.cookie.identifier)
self.assertEqual(len(cookie.enrichments), 1)
def test_evaluate_rules_with_cookie_when_matched_enriches_with_rule_application_log(self):
matched_rule_identifier = "matched_rule"
terminates = False
self.processor.rules = self.rules \
+ [Rule(lambda *args: True, MagicMock(return_value=terminates), matched_rule_identifier)]
_ = self.processor.evaluate_rules_with_cookie(self.cookie)
cookie = self.cookie_jar.fetch_cookie(self.cookie.identifier)
self.assertGreater(len(cookie.enrichments), 1)
last_enrichment = cookie.enrichments[-1]
self.assertEqual(last_enrichment.source, RULE_APPLICATION)
rule_application_log = RuleApplicationLogJSONDecoder().decode_parsed(last_enrichment.metadata) # type: RuleApplicationLog
self.assertEqual(rule_application_log.rule_id, matched_rule_identifier)
self.assertEqual(rule_application_log.terminated_processing, terminates)
def test_handle_cookie_enrichment_when_matching_enrichments(self):
self.processor.notification_receivers = [MagicMock()]
self.processor.enrichment_loaders = [EnrichmentLoader(
lambda *args: True, lambda *args: SAMPLE_ENRICHMENT, RULE_IDENTIFIER)]
self.processor.handle_cookie_enrichment(self.cookie)
self.processor.notification_receivers[0].receive.assert_not_called()
cookie = self.cookie_jar.get_next_for_processing()
self.assertIn(SAMPLE_ENRICHMENT, cookie.enrichments)
def setUp(self):
self.cookie_jar = InMemoryCookieJar()
self.rules = [Rule(lambda *args: False, MagicMock()) for _ in range(10)]
self.cookie = Cookie(COOKIE_IDENTIFIER)
self.processor = BasicProcessor(self.cookie_jar, [], [])
class TestBasicProcessor(unittest.TestCase):
"""
Tests for `BasicProcessor`.
"""
def setUp(self):
self.cookie_jar = InMemoryCookieJar()
self.rules = [Rule(lambda *args: False, MagicMock()) for _ in range(10)]
self.cookie = Cookie(COOKIE_IDENTIFIER)
self.processor = BasicProcessor(self.cookie_jar, [], [])
def test_evaluate_rules_with_cookie_when_no_rules(self):
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
self.assertFalse(halt)
def test_evaluate_rules_with_cookie_when_no_matched_rules(self):
self.processor.rules = self.rules
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
for rule in self.rules:
rule._action.assert_not_called()
self.assertFalse(halt)
def test_evaluate_rules_with_cookie_when_matched_rules_and_no_termination(self):
extra_rules = [
Rule(lambda *args: True, MagicMock(return_value=False)),
Rule(lambda *args: True, MagicMock(return_value=False))
]
self.processor.rules = self.rules + extra_rules
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
for rule in self.rules:
rule._action.assert_not_called()
self.assertFalse(halt)
def test_evaluate_rules_with_cookie_when_matched_rules_and_termination(self):
production_to_be_called = MagicMock(return_value=True)
extra_rules = [
Rule(lambda *args: True, lambda *args: False, Priority.MIN_PRIORITY),
Rule(lambda *args: True, production_to_be_called,
Priority.get_lower_priority_value(Priority.MAX_PRIORITY)),
Rule(lambda *args: False, lambda *args: False, Priority.MAX_PRIORITY)
]
self.processor.rules = self.rules + extra_rules
halt = self.processor.evaluate_rules_with_cookie(self.cookie)
total_call_count = 0
for rule in self.rules:
total_call_count += rule._action.call_count
self.assertEqual(total_call_count, 0)
self.assertEqual(production_to_be_called.call_count, 1)
self.assertTrue(halt)
def test_evaluate_rules_with_cookie_does_not_allow_rule_to_change_cookie_for_subsequent_rules(self):
source = "my_enrichment"
change_detected_in_next_rule = False
def cookie_changer(cookie: Cookie) -> bool:
enrichment = Enrichment(source, datetime(year=2000, month=1, day=1), Metadata())
cookie.enrich(enrichment)
return False
def record_fail_if_changed(cookie: Cookie) -> bool:
nonlocal change_detected_in_next_rule
if source in cookie.get_enrichment_sources():
change_detected_in_next_rule = True
self.processor.rules = [
Rule(cookie_changer, self.rules[0]._action, priority=Priority.MAX_PRIORITY),
Rule(record_fail_if_changed, self.rules[0]._action, priority=Priority.MIN_PRIORITY)
]
self.processor.process_cookie(self.cookie)
self.assertFalse(change_detected_in_next_rule)
def test_handle_cookie_enrichment_when_matching_enrichments(self):
self.processor.notification_receivers = [MagicMock()]
self.processor.enrichment_loaders = [EnrichmentLoader(lambda *args: True, lambda *args: SAMPLE_ENRICHMENT)]
self.processor.handle_cookie_enrichment(self.cookie)
self.processor.notification_receivers[0].receive.assert_not_called()
cookie = self.cookie_jar.get_next_for_processing()
self.assertIn(SAMPLE_ENRICHMENT, cookie.enrichments)
def setUp(self):
self._logger = MagicMock()
self._cookie_jar = InMemoryCookieJar()
self._monitor = CookieJarMonitor(self._logger, timedelta(microseconds=1), self._cookie_jar)