def setUp(self):
     self.cookie_jar = InMemoryCookieJar()
     self.rules = [Rule(lambda *args: False, MagicMock(), RULE_IDENTIFIER) for _ in range(10)]
     self.cookie_jar.enrich_cookie(
         COOKIE_IDENTIFIER, Enrichment("first", datetime(year=1, month=1, day=1, tzinfo=timezone.utc), Metadata()))
     self.cookie = self.cookie_jar.fetch_cookie(COOKIE_IDENTIFIER)
     self.processor = BasicProcessor(self.cookie_jar, [], [])
class TestThreadsMonitor(unittest.TestCase):
    """
    Tests for `ThreadsMonitor`.
    """
    def setUp(self):
        self._logger = MagicMock()
        self._cookie_jar = InMemoryCookieJar()
        self._monitor = CookieJarMonitor(self._logger, timedelta(microseconds=1), self._cookie_jar)

    def test_do_log_record(self):
        self._cookie_jar.enrich_cookie("test", Enrichment("test", datetime(1, 2, 3), Metadata()))
        self._monitor.do_log_record()
        self.assertEqual(self._logger.record.call_count, 1)
        call_args = self._logger.record.call_args[0]
        self.assertEqual(call_args[0], MEASURED_COOKIE_JAR_STATUS)
        self.assertEqual(call_args[1], {MEASURED_COOKIE_JAR_TO_PROCESS: 1})
    def test_call_to_non_adapted_cookie_jar_method(self):
        self._cookie_jar = InMemoryCookieJar()
        self._cookie_jar._composite_cookie_jar = self._cookie_jar

        listener = lambda: None
        self._cookie_jar.add_listener(listener)
        assert self._cookie_jar.get_listeners() == [listener]
        self.assertEqual(self._cookie_jar.get_listeners, self._cookie_jar.get_listeners)
        self.assertEqual(self._cookie_jar.get_listeners(), [listener])
示例#4
0
    def _change_time(self, cookie_jar: InMemoryCookieJar, change_time_to: int):
        cookie_jar._get_time = MagicMock(return_value=change_time_to)

        for end_time in cookie_jar._timers.keys():
            if end_time <= change_time_to:
                while len(cookie_jar._timers[end_time]) > 0:
                    timer = cookie_jar._timers[end_time][0]    # type: Timer
                    timer.interval = 0
                    timer.run()
    def setUp(self):
        self._logger = MagicMock()

        self._cookie_jar = MagicMock()
        self._composite_methods = dict()    # type: Dict[str, MagicMock]
        for method_name in CookieJar.__abstractmethods__:
            method = getattr(self._cookie_jar, method_name)
            self._composite_methods[method_name] = method

        add_cookie_jar_logging(self._cookie_jar, self._logger)
示例#6
0
def create_magic_mock_cookie_jar() -> CookieJar:
    """
    Creates a magic mock CookieJar - has the implementation of a CookieJar all methods are implemented using magic mocks
    and therefore their usage is recorded.
    :return: the created magic mock
    """
    cookie_jar = InMemoryCookieJar()
    original_get_next_for_processing = cookie_jar.get_next_for_processing
    original_enrich_cookie = cookie_jar.enrich_cookie
    original_mark_as_failed = cookie_jar.mark_as_complete
    original_mark_as_completed = cookie_jar.mark_as_complete
    original_mark_as_reprocess = cookie_jar.mark_for_processing
    cookie_jar.get_next_for_processing = MagicMock(side_effect=original_get_next_for_processing)
    cookie_jar.enrich_cookie = MagicMock(side_effect=original_enrich_cookie)
    cookie_jar.mark_as_failed = MagicMock(side_effect=original_mark_as_failed)
    cookie_jar.mark_as_complete = MagicMock(side_effect=original_mark_as_completed)
    cookie_jar.mark_for_processing = MagicMock(side_effect=original_mark_as_reprocess)
    return cookie_jar
class TestAddCookieJarLogging(unittest.TestCase):
    """
    Tests for `add_cookie_jar_logging` method.
    """
    def setUp(self):
        self._logger = MagicMock()

        self._cookie_jar = MagicMock()
        self._composite_methods = dict()    # type: Dict[str, MagicMock]
        for method_name in CookieJar.__abstractmethods__:
            method = getattr(self._cookie_jar, method_name)
            method.__name__ = method_name
            self._composite_methods[method_name] = method

        add_cookie_jar_logging(self._cookie_jar, self._logger)

    def test_enrich_cookie(self):
        source = "my_source"
        enrichment = Enrichment("source", datetime.min, Metadata())
        self._cookie_jar.enrich_cookie(source, enrichment)
        self._composite_methods[CookieJar.enrich_cookie.__name__].assert_called_once_with(source, enrichment)
        self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.enrich_cookie.__name__])

    def test_mark_as_failed(self):
        identifier = "identifier"
        requeue_delay = timedelta(seconds=5)
        self._cookie_jar.mark_as_failed(identifier, requeue_delay)
        self._composite_methods[CookieJar.mark_as_failed.__name__].assert_called_once_with(identifier, requeue_delay)
        self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.mark_as_failed.__name__])

    def test_mark_as_complete(self):
        identifier = "identifier"
        self._cookie_jar.mark_as_complete(identifier)
        self._composite_methods[CookieJar.mark_as_complete.__name__].assert_called_once_with(identifier)
        self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.mark_as_complete.__name__])

    def test_mark_for_processing(self):
        identifier = "identifier"
        self._cookie_jar.mark_for_processing(identifier)
        self._composite_methods[CookieJar.mark_for_processing.__name__].assert_called_once_with(identifier)
        self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.mark_for_processing.__name__])

    def test_get_next_for_processing(self):
        self._cookie_jar.get_next_for_processing()
        self._composite_methods[CookieJar.get_next_for_processing.__name__].assert_called_once_with()
        self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.get_next_for_processing.__name__])

    def test_queue_length(self):
        self._cookie_jar.queue_length()
        self._composite_methods[CookieJar.queue_length.__name__].assert_called_once_with()
        self._assert_measured(MEASUREMENT_QUERY_TIME[CookieJar.queue_length.__name__])

    def test_call_to_non_adapted_cookie_jar_method(self):
        self._cookie_jar = InMemoryCookieJar()
        self._cookie_jar._composite_cookie_jar = self._cookie_jar

        listener = lambda: None
        self._cookie_jar.add_listener(listener)
        assert self._cookie_jar.get_listeners() == [listener]
        self.assertEqual(self._cookie_jar.get_listeners, self._cookie_jar.get_listeners)
        self.assertEqual(self._cookie_jar.get_listeners(), [listener])

    def test_call_to_non_cookie_jar_method(self):
        self._cookie_jar.some_method = MagicMock()
        self._cookie_jar.some_method()
        self._cookie_jar.some_method.assert_called_once_with()

    def test_call_to_non_cookie_jar_property(self):
        self._cookie_jar.some_property = 1
        self.assertEqual(self._cookie_jar.some_property, 1)

    def _assert_measured(self, measured: str):
        calls = self._logger.record.mock_calls
        self.assertEqual(len(calls), 1)
        args, kwargs = self._logger.record.call_args
        self.assertEqual(args[0], measured)
class TestBasicProcessor(unittest.TestCase):
    """
    Tests for `BasicProcessor`.
    """
    def setUp(self):
        self.cookie_jar = InMemoryCookieJar()
        self.rules = [Rule(lambda *args: False, MagicMock(), RULE_IDENTIFIER) for _ in range(10)]
        self.cookie_jar.enrich_cookie(
            COOKIE_IDENTIFIER, Enrichment("first", datetime(year=1, month=1, day=1, tzinfo=timezone.utc), Metadata()))
        self.cookie = self.cookie_jar.fetch_cookie(COOKIE_IDENTIFIER)
        self.processor = BasicProcessor(self.cookie_jar, [], [])

    def test_evaluate_rules_with_cookie_when_no_rules(self):
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        self.assertFalse(halt)

    def test_evaluate_rules_with_cookie_when_no_matched_rules(self):
        self.processor.rules = self.rules
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        for rule in self.rules:
            rule._action.assert_not_called()
        self.assertFalse(halt)

    def test_evaluate_rules_with_cookie_when_matched_rules_and_no_termination(self):
        extra_rules = [
            Rule(lambda *args: True, MagicMock(return_value=False), RULE_IDENTIFIER),
            Rule(lambda *args: True, MagicMock(return_value=False), RULE_IDENTIFIER)
        ]
        self.processor.rules = self.rules + extra_rules
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        for rule in self.rules:
            rule._action.assert_not_called()
        for rule in extra_rules:
            self.assertEqual(rule._action.call_count, 1)
            self.assertEqual(rule._action.call_args[0][0].identifier, self.cookie.identifier)
        self.assertFalse(halt)

    def test_evaluate_rules_with_cookie_when_matched_rules_and_termination(self):
        production_to_be_called = MagicMock(return_value=True)
        extra_rules = [
            Rule(lambda *args: True, lambda *args: False, Priority.MIN_PRIORITY),
            Rule(lambda *args: True, production_to_be_called,
                 Priority.get_lower_priority_value(Priority.MAX_PRIORITY)),
            Rule(lambda *args: False, lambda *args: False, Priority.MAX_PRIORITY)
        ]
        self.processor.rules = self.rules + extra_rules
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        total_call_count = 0
        for rule in self.rules:
            total_call_count += rule._action.call_count
        self.assertEqual(total_call_count, 0)
        self.assertEqual(production_to_be_called.call_count, 1)
        self.assertTrue(halt)

    def test_evaluate_rules_with_cookie_does_not_allow_rule_to_change_cookie_for_subsequent_rules(self):
        source = "my_enrichment"
        change_detected_in_next_rule = False

        def cookie_changer(cookie: Cookie) -> bool:
            enrichment = Enrichment(source, datetime(year=2000, month=1, day=1), Metadata())
            cookie.enrich(enrichment)
            return False

        def record_fail_if_changed(cookie: Cookie) -> bool:
            nonlocal change_detected_in_next_rule
            if source in cookie.get_enrichment_sources():
                change_detected_in_next_rule = True

        self.processor.rules = [
            Rule(cookie_changer, self.rules[0]._action, RULE_IDENTIFIER, priority=Priority.MAX_PRIORITY),
            Rule(record_fail_if_changed, self.rules[0]._action, RULE_IDENTIFIER, priority=Priority.MIN_PRIORITY)
        ]
        self.processor.process_cookie(self.cookie)

        self.assertFalse(change_detected_in_next_rule)

    def test_evaluate_rules_with_cookie_when_no_matched__does_not_enrich_with_rule_application_log(self):
        self.processor.rules = self.rules
        _ = self.processor.evaluate_rules_with_cookie(self.cookie)

        cookie = self.cookie_jar.fetch_cookie(self.cookie.identifier)
        self.assertEqual(len(cookie.enrichments), 1)

    def test_evaluate_rules_with_cookie_when_matched_enriches_with_rule_application_log(self):
        matched_rule_identifier = "matched_rule"
        terminates = False
        self.processor.rules = self.rules \
                               + [Rule(lambda *args: True, MagicMock(return_value=terminates), matched_rule_identifier)]
        _ = self.processor.evaluate_rules_with_cookie(self.cookie)

        cookie = self.cookie_jar.fetch_cookie(self.cookie.identifier)
        self.assertGreater(len(cookie.enrichments), 1)
        last_enrichment = cookie.enrichments[-1]
        self.assertEqual(last_enrichment.source, RULE_APPLICATION)
        rule_application_log = RuleApplicationLogJSONDecoder().decode_parsed(last_enrichment.metadata)  # type: RuleApplicationLog
        self.assertEqual(rule_application_log.rule_id, matched_rule_identifier)
        self.assertEqual(rule_application_log.terminated_processing, terminates)

    def test_handle_cookie_enrichment_when_matching_enrichments(self):
        self.processor.notification_receivers = [MagicMock()]
        self.processor.enrichment_loaders = [EnrichmentLoader(
            lambda *args: True, lambda *args: SAMPLE_ENRICHMENT, RULE_IDENTIFIER)]

        self.processor.handle_cookie_enrichment(self.cookie)

        self.processor.notification_receivers[0].receive.assert_not_called()
        cookie = self.cookie_jar.get_next_for_processing()
        self.assertIn(SAMPLE_ENRICHMENT, cookie.enrichments)
 def setUp(self):
     self.cookie_jar = InMemoryCookieJar()
     self.rules = [Rule(lambda *args: False, MagicMock()) for _ in range(10)]
     self.cookie = Cookie(COOKIE_IDENTIFIER)
     self.processor = BasicProcessor(self.cookie_jar, [], [])
class TestBasicProcessor(unittest.TestCase):
    """
    Tests for `BasicProcessor`.
    """
    def setUp(self):
        self.cookie_jar = InMemoryCookieJar()
        self.rules = [Rule(lambda *args: False, MagicMock()) for _ in range(10)]
        self.cookie = Cookie(COOKIE_IDENTIFIER)
        self.processor = BasicProcessor(self.cookie_jar, [], [])

    def test_evaluate_rules_with_cookie_when_no_rules(self):
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        self.assertFalse(halt)

    def test_evaluate_rules_with_cookie_when_no_matched_rules(self):
        self.processor.rules = self.rules
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        for rule in self.rules:
            rule._action.assert_not_called()
        self.assertFalse(halt)

    def test_evaluate_rules_with_cookie_when_matched_rules_and_no_termination(self):
        extra_rules = [
            Rule(lambda *args: True, MagicMock(return_value=False)),
            Rule(lambda *args: True, MagicMock(return_value=False))
        ]
        self.processor.rules = self.rules + extra_rules
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        for rule in self.rules:
            rule._action.assert_not_called()
        self.assertFalse(halt)

    def test_evaluate_rules_with_cookie_when_matched_rules_and_termination(self):
        production_to_be_called = MagicMock(return_value=True)
        extra_rules = [
            Rule(lambda *args: True, lambda *args: False, Priority.MIN_PRIORITY),
            Rule(lambda *args: True, production_to_be_called,
                 Priority.get_lower_priority_value(Priority.MAX_PRIORITY)),
            Rule(lambda *args: False, lambda *args: False, Priority.MAX_PRIORITY)
        ]
        self.processor.rules = self.rules + extra_rules
        halt = self.processor.evaluate_rules_with_cookie(self.cookie)

        total_call_count = 0
        for rule in self.rules:
            total_call_count += rule._action.call_count
        self.assertEqual(total_call_count, 0)
        self.assertEqual(production_to_be_called.call_count, 1)
        self.assertTrue(halt)

    def test_evaluate_rules_with_cookie_does_not_allow_rule_to_change_cookie_for_subsequent_rules(self):
        source = "my_enrichment"
        change_detected_in_next_rule = False

        def cookie_changer(cookie: Cookie) -> bool:
            enrichment = Enrichment(source, datetime(year=2000, month=1, day=1), Metadata())
            cookie.enrich(enrichment)
            return False

        def record_fail_if_changed(cookie: Cookie) -> bool:
            nonlocal change_detected_in_next_rule
            if source in cookie.get_enrichment_sources():
                change_detected_in_next_rule = True

        self.processor.rules = [
            Rule(cookie_changer, self.rules[0]._action, priority=Priority.MAX_PRIORITY),
            Rule(record_fail_if_changed, self.rules[0]._action, priority=Priority.MIN_PRIORITY)
        ]
        self.processor.process_cookie(self.cookie)

        self.assertFalse(change_detected_in_next_rule)

    def test_handle_cookie_enrichment_when_matching_enrichments(self):
        self.processor.notification_receivers = [MagicMock()]
        self.processor.enrichment_loaders = [EnrichmentLoader(lambda *args: True, lambda *args: SAMPLE_ENRICHMENT)]

        self.processor.handle_cookie_enrichment(self.cookie)

        self.processor.notification_receivers[0].receive.assert_not_called()
        cookie = self.cookie_jar.get_next_for_processing()
        self.assertIn(SAMPLE_ENRICHMENT, cookie.enrichments)
 def setUp(self):
     self._logger = MagicMock()
     self._cookie_jar = InMemoryCookieJar()
     self._monitor = CookieJarMonitor(self._logger, timedelta(microseconds=1), self._cookie_jar)