def test1sorted_log_atoms(self):
"""In this test case multiple, SORTED LogAtoms of different sources are received by the class."""
description = "Test1SimpleMultisourceAtomSync"
sync_wait_time = 3
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector1,
description)
new_match_path_detector2 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector2,
description + "2")
simple_multisource_atom_sync = SimpleMultisourceAtomSync(
[new_match_path_detector1, new_match_path_detector2],
sync_wait_time)
t = time()
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element(
'match', match_context)
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector1)
log_atom2 = LogAtom(match_element.match_object,
ParserMatch(match_element), t + 1,
new_match_path_detector1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
sleep(sync_wait_time + 1)
# not of the same source, thus must not be accepted.
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
# logAtom1 is handled now, so logAtom2 is accepted.
self.reset_output_stream()
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 1).strftime(
self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description, 1,
self.match_path, self.calculation) + self.__expected_string %
(datetime.fromtimestamp(t + 1).strftime(
self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description + "2", 1,
self.match_path, self.calculation))
def test3normal_incomplete_overlong_line_stream_not_ended(self):
"""A incomplete, overlong line, with the stream NOT ended, is tested as Input of the Class."""
match_context_fixed_dme = MatchContext(self.illegal_access1)
any_dme = AnyByteDataModelElement('s')
_match_element_fixed_dme = any_dme.get_match_element("match1", match_context_fixed_dme)
byte_stream_line_atomizer = ByteStreamLineAtomizer(
any_dme, [], [self.stream_printer_event_handler], sys.getsizeof(match_context_fixed_dme.match_data) - 1, [])
self.assertGreater(byte_stream_line_atomizer.consume_data(self.illegal_access1, False), 0)
self.assertEqual(self.output_stream.getvalue(), 'Start of overlong line detected (1 lines)\n %s' % self.illegal_access2)
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated accordingly."""
data = b'abcdefghijklmnopqrstuvwxyz.!?'
match_context = DummyMatchContext(data)
any_dme = AnyByteDataModelElement("s0")
match_element = any_dme.get_match_element("any", match_context)
self.assertEqual(match_element.path, "any/s0")
self.assertEqual(match_element.match_string, data)
self.assertEqual(match_element.match_object, data)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, data)
def test3_parser_match_is_other_element(self):
"""In this test case the ParserMatch actually is no instance of ParserMatch. The atom should still be considered to be parsed."""
description = "Test3SimpleUnparsedAtomHandler"
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False)
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element('match', match_context)
log_atom = LogAtom(match_element.match_object, any_byte_data_model_element, time(), new_match_path_detector1)
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler([self.stream_printer_event_handler])
self.analysis_context.register_component(simple_unparsed_atom_handler, description)
self.assertTrue(not simple_unparsed_atom_handler.receive_atom(log_atom))
def test2_atom_is_parsed(self):
"""The atom in this test case has no ParserMatch."""
description = "Test2SimpleUnparsedAtomHandler"
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False)
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element('match', match_context)
log_atom = LogAtom(match_element.match_object, None, time(), new_match_path_detector1)
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler([self.stream_printer_event_handler])
self.analysis_context.register_component(simple_unparsed_atom_handler, description)
self.assertTrue(simple_unparsed_atom_handler.receive_atom(log_atom))
def test13value_dependent_delegated_match_rule(self):
"""This case unit the ValueDependentDelegatedMatchRule."""
description = "Test13Rules"
string_regex_match_rule = StringRegexMatchRule(self.match_any, re.compile(r'\w'), None)
self.analysis_context.register_component(string_regex_match_rule, description)
any_byte_date_me = AnyByteDataModelElement('any')
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule, description + "2")
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
value_dependent_delegated_match_rule = ValueDependentDelegatedMatchRule([
self.match_any, self.match_ipv4], {(self.alphabet, None): string_regex_match_rule,
(None, 3232235520): i_pv4_in_rfc1918_match_rule})
self.analysis_context.register_component(value_dependent_delegated_match_rule, description + "3")
match_context = MatchContext(self.alphabet)
match_element = any_byte_date_me.get_match_element('match', match_context)
log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), 1, value_dependent_delegated_match_rule)
self.assertTrue(value_dependent_delegated_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element('match', match_context)
log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), 1, value_dependent_delegated_match_rule)
self.assertTrue(value_dependent_delegated_match_rule.match(log_atom))
# not matching values
match_context = MatchContext('.There are 26 letters in the english alphabet')
match_element = any_byte_date_me.get_match_element('match', match_context)
log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), 1, value_dependent_delegated_match_rule)
self.assertTrue(not value_dependent_delegated_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.0.1')
match_element = ip_address_data_model_element.get_match_element('match', match_context)
log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), 1, value_dependent_delegated_match_rule)
self.assertTrue(not value_dependent_delegated_match_rule.match(log_atom))
def test4has_idle_source(self):
"""In this test case a source becomes idle and expires."""
description = "Test4SimpleMultisourceAtomSync"
sync_wait_time = 3
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector1,
description)
new_match_path_detector2 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector2,
description + "2")
simple_multisource_atom_sync = SimpleMultisourceAtomSync(
[new_match_path_detector1], sync_wait_time)
t = time()
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element(
'match', match_context)
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector1)
log_atom2 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector2)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
sleep(sync_wait_time + 1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
# log_atom1 is handled now, so new_match_path_detector1 should be deleted after waiting the sync_wait_time.
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
sleep(sync_wait_time + 1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
self.assertEqual(
simple_multisource_atom_sync.sources_dict, {
new_match_path_detector1: [log_atom1.get_timestamp(), None],
new_match_path_detector2:
[log_atom2.get_timestamp(), log_atom2]
})
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
sleep(sync_wait_time + 1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
simple_multisource_atom_sync.sources_dict, {
new_match_path_detector1: [log_atom1.get_timestamp(), None],
new_match_path_detector2:
[log_atom2.get_timestamp(), log_atom2]
})
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t + 1,
new_match_path_detector1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
simple_multisource_atom_sync.sources_dict, {
new_match_path_detector1:
[log_atom1.get_timestamp() - 1, log_atom1],
new_match_path_detector2:
[log_atom2.get_timestamp(), log_atom2]
})
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t - 1,
new_match_path_detector1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))