def test3_new_year_with_start_year_value(self):
"""This test case checks if a new year is learned successfully with the start year being set."""
start_year = 2017
match_context = MatchContext(b'07.02.2018 11:40:00: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m.%Y %H:%M:%S',
datetime.timezone.utc,
None, start_year)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02.2018 11:40:00 UTC+0000: it still works')
date_time_model_element = DateTimeModelElement(
'path', b'%d.%m.%Y %H:%M:%S %z', datetime.timezone.utc, None,
start_year)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02.2018 11:40:00 UTC+0001: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518007200)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02.2018 11:40:00 UTC+0000: it still works')
date_time_model_element = DateTimeModelElement(
'path', b'%d.%m.%Y %H:%M:%S %z', datetime.timezone.utc, None,
start_year)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02.2018 11:40:00 UTC-0001: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518000000)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02.2018 11:40:00 CET+1: it still works')
date_time_model_element = DateTimeModelElement(
'path', b'%d.%m.%Y %H:%M:%S %z', datetime.timezone.utc, None,
start_year)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02.2018 11:40:00 UTC+2: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518007200)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07/Feb/2017:11:40:00 +0000] "GET /login.php HTTP/1.1" 200 2532 "-" "Mozilla/5.0 (X11; Ubuntu; '
b'Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0')
date_time_model_element = DateTimeModelElement(
'path', b'%d/%b/%Y:%H:%M:%S %z', datetime.timezone.utc, None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(
match_context.match_data,
b'] "GET /login.php HTTP/1.1" 200 2532 "-" "Mozilla/5.0 (X11; Ubuntu; '
b'Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0')
match_context = MatchContext(
b'07.02.2018 11:40:00 UTC: it still works')
date_time_model_element = DateTimeModelElement(
'path', b'%d.%m.%Y %H:%M:%S %z', datetime.timezone.utc, None,
start_year)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
def test2do_timer(self):
"""In this test case the functionality of the timer is tested. The eventCollectTime must not be 0."""
description = "Test2DefaultMailNotificationEventHandler"
default_mail_notification_event_handler = DefaultMailNotificationEventHandler(
self.analysis_context)
self.analysis_context.register_component(self, description)
t = time()
match_context = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s3', self.pid)
match_element = fixed_dme.get_match_element("match", match_context)
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t,
self)
default_mail_notification_event_handler.receive_event(
self.test % self.__class__.__name__,
'New value for pathes %s: %s' %
('match/s3', repr(match_element.match_object)),
[log_atom.raw_data], None, log_atom, self)
t = 0
default_mail_notification_event_handler.do_timer(t)
# skipcq: PYL-W1510, BAN-B602
result = subprocess.run(self.mail_call,
shell=True,
stdout=subprocess.PIPE)
self.assertFalse(
self.__expected_string %
(match_element.get_path(), match_element.get_match_object(),
self.__class__.__name__, description, 1,
match_element.get_match_string().decode() +
"\n\n") in str(result.stdout, 'utf-8'))
t = time()
default_mail_notification_event_handler.next_alert_time = t + 500
default_mail_notification_event_handler.do_timer(t)
# skipcq: PYL-W1510, BAN-B602
result = subprocess.run(self.mail_call,
shell=True,
stdout=subprocess.PIPE)
self.assertFalse(
self.__expected_string %
(match_element.get_path(), match_element.get_match_object(),
self.__class__.__name__, description, 1,
match_element.get_match_string().decode() +
"\n\n") in str(result.stdout, 'utf-8'))
default_mail_notification_event_handler.next_alert_time = t
default_mail_notification_event_handler.do_timer(t)
sleep(2)
# skipcq: PYL-W1510, BAN-B602
result = subprocess.run(self.mail_call,
shell=True,
stdout=subprocess.PIPE)
# skipcq: PYL-W1510, BAN-B602
subprocess.run(self.mail_delete_call,
shell=True,
stdout=subprocess.PIPE)
if datetime.fromtimestamp(t).strftime(
self.datetime_format_string) not in str(
result.stdout, 'utf-8'):
print("ERROR: %s t not found in mail!" % description,
file=sys.stderr)
self.assertTrue(
self.__expected_string %
(match_element.get_path(), match_element.get_match_object(),
self.__class__.__name__, description, 1,
match_element.get_match_string().decode() + "\n\n")
in str(result.stdout, 'utf-8'),
msg="%s vs \n %s" %
(self.__expected_string %
(match_element.get_path(), match_element.get_match_object(),
self.__class__.__name__, description, 1,
match_element.get_match_string().decode() + "\n\n"),
str(result.stdout, 'utf-8')))
class NewMatchPathDetectorTest(TestBase):
"""Unittests for the NewMatchPathDetector."""
__expected_string = '%s New path(es) detected\n%s: "%s" (%d lines)\n %s\n%s\n\n'
match_path_s1 = "['/s1']"
match_path_d1 = "['/d1']"
datetime_format_string = '%Y-%m-%d %H:%M:%S'
analysis = 'Analysis.%s'
pid = " pid="
uid = " uid=2"
match_context_fixed_dme = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
match_element_fixed_dme = fixed_dme.get_match_element(
"", match_context_fixed_dme)
match_context_decimal_integer_value_me = MatchContext(b'25537 uid=2')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element_decimal_integer_value_me = decimal_integer_value_me.get_match_element(
"", match_context_decimal_integer_value_me)
def test1_log_atom_not_known(self):
"""
This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found.
The output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be
repeatable on second run.
"""
description = "Test1NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(
self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t,
new_match_path_detector)
self.assertTrue(
new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# repeating should produce the same result
self.assertTrue(
new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# other MatchElement
self.assertTrue(
new_match_path_detector.receive_atom(
log_atom_decimal_integer_value_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector.__class__.__name__, description, 1,
self.match_path_d1, self.uid))
def test2_log_atom_known(self):
"""
This test case checks the functionality of the auto_include_flag.
If the same MatchElement is processed a second time and the auto_include_flag was True, no event must be triggered.
"""
description = "Test2NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(
self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t,
new_match_path_detector)
self.assertTrue(
new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# repeating should NOT produce the same result
self.assertTrue(
new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
# other MatchElement
self.assertTrue(
new_match_path_detector.receive_atom(
log_atom_decimal_integer_value_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector.__class__.__name__, description, 1,
self.match_path_d1, self.uid))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
self.assertTrue(
new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
new_match_path_detector.do_persist()
self.reset_output_stream()
otherNewMatchPathDetector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
otherLogAtomFixedDME = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme), t,
otherNewMatchPathDetector)
self.assertTrue(
otherNewMatchPathDetector.receive_atom(otherLogAtomFixedDME))
self.assertEqual(self.output_stream.getvalue(), '')
def test4_get_time_trigger_class(self):
"""
The known paths are to be periodically stored after a certain time. This requires a synchronization class.
The return of the correct class is to be checked in this test case.
"""
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.assertEqual(new_match_path_detector.get_time_trigger_class(), 1)
# skipcq: PYL-W0105
"""The following test cases should check if the doTimer() method is working properly.This includes the updating of nextPersistTime.
As it is not updated directly in the method this test cases are not correct. Due to that they are commented."""
# def test5_do_timer_next_persist_time_none(self):
# """During initialization, the next time is not determined (the value is initialized with None). In this case, the persistence is
# expected to occur after 600 milliseconds."""
# self.new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
# output_log_line=False)
# self.assertEqual(self.new_match_path_detector.do_timer(200), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(400), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(10000), 600)
#
# def test6_do_timer_delta_smaller_or_equal_zero(self):
# """If the NextPersistTime is less than or equal to zero, the data must be saved."""
# self.new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
# output_log_line=False)
# self.new_match_path_detector.nextPersistTime = 400
# self.assertEqual(self.new_match_path_detector.do_timer(400), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(1000), 600)
#
# def test7_do_timer_delta_greater_zero(self):
# """If the delta does not fall below the limit value, only the delta value should be returned."""
# # this test fails due to the missing update of the nextPersistTime variable in the doTimer method
# self.new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
# output_log_line=False)
# self.new_match_path_detector.nextPersistTime = 400
# self.assertEqual(self.new_match_path_detector.do_timer(200), 200)
# self.assertEqual(self.new_match_path_detector.do_timer(200), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(100), 500)
def test8_allowlist_event_type_exception(self):
"""This test case checks whether an exception is thrown when entering an event of another class."""
description = "Test8NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [], [self.stream_printer_event_handler],
'Default', True, True)
self.assertRaises(
Exception, new_match_path_detector.allowlist_event, self.analysis %
new_match_path_value_combo_detector.__class__.__name__,
self.output_stream.getvalue(), None)
def test9_allowlist_event_allowlisting_data_exception(self):
"""The NewMatchPathDetector can not handle allowlisting data and therefore an exception is expected."""
description = "Test9NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
self.assertRaises(
Exception, new_match_path_detector.allowlist_event,
self.analysis % new_match_path_detector.__class__.__name__,
self.output_stream.getvalue(), ['random', 'Data'])
def test10_allowlist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test10NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
self.assertEqual(
new_match_path_detector.allowlist_event(
self.analysis % new_match_path_detector.__class__.__name__,
self.match_element_fixed_dme.get_path(),
None), 'Allowlisted path(es) %s in %s.' %
(self.match_element_fixed_dme.get_path(),
self.analysis % new_match_path_detector.__class__.__name__))
new_match_path_detector.auto_include_flag = False
self.assertEqual(
new_match_path_detector.allowlist_event(
self.analysis % new_match_path_detector.__class__.__name__,
self.match_element_decimal_integer_value_me.get_path(),
None), 'Allowlisted path(es) %s in %s.' %
(self.match_element_decimal_integer_value_me.path,
self.analysis % new_match_path_detector.__class__.__name__))
class NewMatchPathValueDetectorTest(TestBase):
__expected_string = '%s New value(s) detected\n%s: "%s" (%d lines)\n %s\n\n'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
string = b'25537 uid=2'
first_f1_s1 = 'first/f1/s1'
string2 = "{'first/f1/s1': '25537 uid=2'}\nb'25537 uid=2'"
fixed_dme = FixedDataModelElement('s1', string)
decimalIntegerValueME = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_first_match_me = MatchContext(string)
first_match_me = FirstMatchModelElement('f1',
[fixed_dme, decimalIntegerValueME])
match_element_first_match_me = first_match_me.get_match_element(
'first', match_context_first_match_me)
match_context_first_match_me2 = MatchContext(string)
first_match_me2 = FirstMatchModelElement(
'f2', [decimalIntegerValueME, fixed_dme])
match_element_first_match_me2 = first_match_me2.get_match_element(
'second', match_context_first_match_me2)
def test1_log_atom_not_known(self):
"""This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found. The
output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be repeatable
on second run."""
description = "Test1NewMatchPathValueDetector"
new_match_path_value_detector = NewMatchPathValueDetector(
self.aminer_config, [self.first_f1_s1],
[self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_value_detector,
description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_first_match_me), t,
new_match_path_value_detector)
new_match_path_value_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_detector.__class__.__name__, description, 1,
self.string2))
self.reset_output_stream()
# repeating should produce the same result
new_match_path_value_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_detector.__class__.__name__, description, 1,
self.string2))
self.reset_output_stream()
new_match_path_value_detector2 = NewMatchPathValueDetector(
self.aminer_config, ['second/f2/d1'],
[self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
b'25537', ParserMatch(self.match_element_first_match_me2), t,
new_match_path_value_detector2)
# other MatchElement
new_match_path_value_detector2.receive_atom(log_atom_sequence_me2)
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_detector.__class__.__name__,
description + "2", 1, "{'second/f2/d1': 25537}\nb'25537'"))
def test2_log_atom_known(self):
"""This test case checks the functionality of the auto_include_flag. If the same MatchElement is processed a second time and the
auto_include_flag was True, no event must be triggered."""
description = "Test2NewMatchPathValueDetector"
new_match_path_value_detector = NewMatchPathValueDetector(
self.aminer_config, [self.first_f1_s1],
[self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_value_detector,
description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_first_match_me), t,
new_match_path_value_detector)
new_match_path_value_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_detector.__class__.__name__, description, 1,
self.string2))
self.reset_output_stream()
# repeating should NOT produce the same result
new_match_path_value_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
new_match_path_value_detector2 = NewMatchPathValueDetector(
self.aminer_config, ['second/f2/d1'],
[self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
b'25537', ParserMatch(self.match_element_first_match_me2), t,
new_match_path_value_detector2)
# other MatchElement
new_match_path_value_detector2.receive_atom(log_atom_sequence_me2)
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_detector.__class__.__name__,
description + "2", 1, "{'second/f2/d1': 25537}\nb'25537'"))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3NewMatchPathValueDetector"
new_match_path_value_detector = NewMatchPathValueDetector(
self.aminer_config, [self.first_f1_s1],
[self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_value_detector,
description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_first_match_me), t,
new_match_path_value_detector)
new_match_path_value_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_detector.__class__.__name__, description, 1,
self.string2))
new_match_path_value_detector.do_persist()
self.reset_output_stream()
other_new_match_path_value_detector = NewMatchPathValueDetector(
self.aminer_config, [self.first_f1_s1],
[self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_value_detector,
description + "2")
other_log_atom_fixed_dme = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_first_match_me), t,
other_new_match_path_value_detector)
other_new_match_path_value_detector.receive_atom(
other_log_atom_fixed_dme)
self.assertEqual(self.output_stream.getvalue(), '')
def test9_receive_atom_list_missing_value(self):
"""This test case checks if missing values are reported correctly."""
description = "Test90MissingMatchPathValueDetector"
match_context_fixed_dme = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s1', self.pid)
match_element_fixed_dme = fixed_dme.get_match_element(
"match1", match_context_fixed_dme)
match_context_decimal_integer_value_me = MatchContext(self.string)
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element_decimal_integer_value_me = decimal_integer_value_me.get_match_element(
"match2", match_context_decimal_integer_value_me)
missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, [
match_element_fixed_dme.get_path(),
match_element_decimal_integer_value_me.get_path()
], [self.stream_printer_event_handler], 'Default', True,
self.__default_interval, self.__realert_interval)
self.analysis_context.register_component(
missing_match_path_list_value_detector, description)
log_atom_fixed_dme = LogAtom(fixed_dme.fixed_data,
ParserMatch(match_element_fixed_dme),
round(time.time()),
missing_match_path_list_value_detector)
self.assertTrue(
missing_match_path_list_value_detector.receive_atom(
log_atom_fixed_dme))
past_time = 4000
missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, [
match_element_fixed_dme.get_path(),
match_element_decimal_integer_value_me.get_path()
], [self.stream_printer_event_handler], 'Default', True,
missing_match_path_list_value_detector.default_interval -
past_time, self.__realert_interval)
self.analysis_context.register_component(
missing_match_path_list_value_detector, description + "2")
log_atom_fixed_dme = LogAtom(fixed_dme.fixed_data,
ParserMatch(match_element_fixed_dme),
round(time.time()) + past_time,
missing_match_path_list_value_detector)
self.assertTrue(
missing_match_path_list_value_detector.receive_atom(
log_atom_fixed_dme))
self.assertTrue(
(self.output_stream.getvalue() == self.__expected_string %
(datetime.fromtimestamp(time.time() + past_time).strftime(
self.datetime_format_string),
missing_match_path_list_value_detector.__class__.__name__,
description + "2", 1,
"match1/s1, match2/d1: b' pid=' overdue 400s (interval -400)"))
or
(self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(time.time() + past_time + 1).strftime(
self.datetime_format_string),
missing_match_path_list_value_detector.__class__.__name__,
description + "2", 1,
"match1/s1, match2/d1: b' pid=' overdue 400s (interval -400)")))
def test1receive_atoms_with_default_values(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is empty and all paths are learned dynamically in variable_key_list.
"""
event_type_detector = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(line, ParserMatch(self.parsing_model.get_match_element('parser', MatchContext(line))), t, self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.total_records, i + 1)
def test4has_idle_source(self):
"""In this test case a source becomes idle and expires."""
description = "Test4SimpleMultisourceAtomSync"
sync_wait_time = 3
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector1,
description)
new_match_path_detector2 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector2,
description + "2")
simple_multisource_atom_sync = SimpleMultisourceAtomSync(
[new_match_path_detector1], sync_wait_time)
t = time()
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element(
'match', match_context)
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector1)
log_atom2 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector2)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
sleep(sync_wait_time + 1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
# log_atom1 is handled now, so new_match_path_detector1 should be deleted after waiting the sync_wait_time.
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
sleep(sync_wait_time + 1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom2))
self.assertEqual(
simple_multisource_atom_sync.sources_dict, {
new_match_path_detector1: [log_atom1.get_timestamp(), None],
new_match_path_detector2:
[log_atom2.get_timestamp(), log_atom2]
})
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
sleep(sync_wait_time + 1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
simple_multisource_atom_sync.sources_dict, {
new_match_path_detector1: [log_atom1.get_timestamp(), None],
new_match_path_detector2:
[log_atom2.get_timestamp(), log_atom2]
})
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t + 1,
new_match_path_detector1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
simple_multisource_atom_sync.sources_dict, {
new_match_path_detector1:
[log_atom1.get_timestamp() - 1, log_atom1],
new_match_path_detector2:
[log_atom2.get_timestamp(), log_atom2]
})
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t - 1,
new_match_path_detector1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
def check_anomaly_detection(self, ecd, t, diff):
"""Check if anomalies were detected as expected."""
for char in self.alphabet:
self.reset_output_stream()
char = bytes([char])
parser_match = ParserMatch(self.alphabet_model.get_match_element('parser', MatchContext(char)))
t += 5 * 3
ecd.receive_atom(LogAtom(char, parser_match, t, self.__class__.__name__))
# another LogAtom must be received to check the follow anomalies.
t += 5 * 3
ecd.receive_atom(LogAtom(char, parser_match, t, self.__class__.__name__))
# print(self.output_stream.getvalue())
# precede anomaly
for i in range(1, int(5 / diff) + 1, 1):
# print("in")
# print(bytes([self.alphabet[(self.alphabet.index(char) - i) % len(self.alphabet)]]))
self.assertIn('Event %s is missing, but should precede event %s' % (
bytes([self.alphabet[(self.alphabet.index(char) - i) % len(self.alphabet)]]), char), self.output_stream.getvalue())
for i in range(int(5 / diff) + 1, len(self.alphabet), 1):
# print("not in")
# print(bytes([self.alphabet[(self.alphabet.index(char) - i) % len(self.alphabet)]]))
self.assertNotIn('Event %s is missing, but should precede event %s' % (
bytes([self.alphabet[(self.alphabet.index(char) - i) % len(self.alphabet)]]), char), self.output_stream.getvalue())
# follow anomaly
for i in range(1, int(5 / diff) + 1, 1):
# print("in")
# print(bytes([self.alphabet[(self.alphabet.index(char) + i) % len(self.alphabet)]]))
self.assertIn('Event %s is missing, but should follow event %s' % (
bytes([self.alphabet[(self.alphabet.index(char) + i) % len(self.alphabet)]]), char), self.output_stream.getvalue())
for i in range(int(5 / diff) + 1, len(self.alphabet), 1):
# print("not in")
# print(bytes([self.alphabet[(self.alphabet.index(char) + i) % len(self.alphabet)]]))
self.assertNotIn('Event %s is missing, but should follow event %s' % (
bytes([self.alphabet[(self.alphabet.index(char) + i) % len(self.alphabet)]]), char), self.output_stream.getvalue())
def generate_perfect_data(self, iterations, diff):
"""Generate data without any error."""
log_atoms = []
t = time()
for i in range(1, iterations+1):
char = bytes([self.alphabet[i % len(self.alphabet)]])
parser_match = ParserMatch(self.alphabet_model.get_match_element('parser', MatchContext(char)))
t += diff
log_atoms.append(LogAtom(char, parser_match, t, self.__class__.__name__))
return log_atoms
def test3unsorted_log_atom(self):
"""In this test case multiple, UNSORTED LogAtoms of different sources are received by the class."""
description = "Test3SimpleMultisourceAtomSync"
sync_wait_time = 3
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector1,
description)
new_match_path_detector2 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector2,
description + "2")
simple_multisource_atom_sync = SimpleMultisourceAtomSync(
[new_match_path_detector1, new_match_path_detector2],
sync_wait_time)
t = time()
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element(
'match', match_context)
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector1)
log_atom2 = LogAtom(match_element.match_object,
ParserMatch(match_element), t - 1,
new_match_path_detector1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
sleep(sync_wait_time)
# unsorted, should be accepted
self.reset_output_stream()
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom2))
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t - 1).strftime(
self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description, 1,
self.match_path, self.calculation.decode()) +
self.__expected_string %
(datetime.fromtimestamp(t - 1).strftime(
self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description + "2", 1,
self.match_path, self.calculation.decode()) +
self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description, 1,
self.match_path, self.calculation.decode()) +
self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description + "2", 1,
self.match_path, self.calculation.decode()))
class TimeCorrelationDetectorTest(TestBase):
"""Unittests for the TimeCorrlelationDetectorTest."""
__expected_string = '%s Correlation report\nTimeCorrelationDetector: "%s" (%d lines)\n '
string = b'25537 uid=2'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
fixed_dme = FixedDataModelElement('s1', string)
decimal_integer_value_me = DecimalIntegerValueModelElement('d1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_first_match_me = MatchContext(string)
first_match_me = FirstMatchModelElement('f1', [fixed_dme, decimal_integer_value_me])
match_element_first_match_me = first_match_me.get_match_element('first', match_context_first_match_me)
match_context_first_match_me2 = MatchContext(string)
first_match_me2 = FirstMatchModelElement('f2', [decimal_integer_value_me, fixed_dme])
match_element_first_match_me2 = first_match_me2.get_match_element('second', match_context_first_match_me2)
def test1_normal_report(self):
"""
This test case unit the creation of a report.
As the rules are chosen randomly this test can not be very specific in checking he actual values of the report.
"""
description = "Test1TimeCorrelationDetector"
parallel_check_count = 2
record_count_before_event = 10
output_logline = True
use_path_match = True
use_value_match = True
min_rule_attributes = 1
max_rule_attributes = 5
time_correlation_detector = TimeCorrelationDetector(self.aminer_config, [self.stream_printer_event_handler], parallel_check_count,
'Default', record_count_before_event, output_logline, use_path_match,
use_value_match, min_rule_attributes, max_rule_attributes)
self.analysis_context.register_component(time_correlation_detector, component_name=description)
t = time.time()
for i in range(0, 10):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t).strftime(self.datetime_format_string), description, 10)))
self.reset_output_stream()
for i in range(0, 10):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t + 9).strftime(self.datetime_format_string), description, 20)))
self.reset_output_stream()
for i in range(10, 15):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
logAtomSequenceME2 = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me2), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME2)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t + 14).strftime(self.datetime_format_string), description, 30)))
def test4receive_match_after_max_allowed_time_diff_without_auto_include_flag(
self):
"""This test case checks if log_atoms are deleted after the maximal allowed time difference with the auto_include_flag=False."""
description = 'test4newMatchIdValueComboDetectorTest'
output_stream_empty_results = [
True, False, True, False, True, False, True, True, False, True,
False, True, True, True, False, False, False, True, False, True,
False
]
id_dict_current_results = [{
1: {
'parser/type/syscall/syscall': 1
}
}, {}, {
2: {
'parser/type/syscall/syscall': 2
}
}, {}, {
3: {
'parser/type/syscall/syscall': 3
}
}, {}, {
100: {
'parser/type/syscall/syscall': 1
}
}, {
100: {
'parser/type/syscall/syscall': 1
},
4: {
'parser/type/syscall/syscall': 1
}
}, {
100: {
'parser/type/syscall/syscall': 1
}
}, {
5: {
'parser/type/path/name': 'two'
},
100: {
'parser/type/syscall/syscall': 1
}
}, {}, {
6: {
'parser/type/syscall/syscall': 4
}
}, {
6: {
'parser/type/syscall/syscall': 4
},
7: {
'parser/type/path/name': 'five'
}
}, {
6: {
'parser/type/syscall/syscall': 4
},
7: {
'parser/type/path/name': 'five'
},
8: {
'parser/type/syscall/syscall': 6
}
}, {
7: {
'parser/type/path/name': 'five'
},
8: {
'parser/type/syscall/syscall': 6
}
}, {}, {}, {
9: {
'parser/type/syscall/syscall': 2
}
}, {}, {
10: {
'parser/type/path/name': 'one'
}
}, {}]
id_dict_old_results = [{}] * 10 + [{
100: {
'parser/type/syscall/syscall': 1
}
}] * 5 + [{
8: {
'parser/type/syscall/syscall': 6
}
}] + [{}] * 5
min_allowed_time_diff = 5
log_atoms = []
t = time.time()
for line in self.log_lines:
log_atoms.append(
LogAtom(
line,
ParserMatch(
self.parsing_model.get_match_element(
'parser', MatchContext(line))), t,
self.__class__.__name__))
t = t + min_allowed_time_diff * 0.25
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
self.aminer_config,
['parser/type/path/name', 'parser/type/syscall/syscall'],
[self.stream_printer_event_handler],
id_path_list=['parser/type/path/id', 'parser/type/syscall/id'],
min_allowed_time_diff=min_allowed_time_diff,
auto_include_flag=False,
allow_missing_values_flag=True,
persistence_id='audit_type_path',
output_log_line=False)
self.analysis_context.register_component(
new_match_id_value_combo_detector, description)
for i, log_atom in enumerate(log_atoms):
self.assertTrue(
new_match_id_value_combo_detector.receive_atom(log_atom))
self.assertEqual(self.output_stream.getvalue() == "",
output_stream_empty_results[i], log_atom.raw_data)
self.assertEqual(new_match_id_value_combo_detector.id_dict_current,
id_dict_current_results[i], log_atom.raw_data)
self.assertEqual(new_match_id_value_combo_detector.id_dict_old,
id_dict_old_results[i])
self.assertEqual(new_match_id_value_combo_detector.known_values,
[])
self.reset_output_stream()
def test20_suppress_output(self):
"""
Check if the suppress property and SuppressNewMatchPathDetector are working as expected.
This test only includes the StreamPrinterEventHandler.
"""
__expected_string1 = '%s New path(es) detected\n%s: "%s" (%d lines)\n %s\n%s\n\n'
t = time()
fixed_dme = FixedDataModelElement('s1', b' pid=')
match_context_fixed_dme = MatchContext(b' pid=')
match_element_fixed_dme = fixed_dme.get_match_element(
"", match_context_fixed_dme)
log_atom_fixed_dme = LogAtom(fixed_dme.fixed_data,
ParserMatch(match_element_fixed_dme), t,
'DefaultNewMatchPathDetector')
datetime_format_string = '%Y-%m-%d %H:%M:%S'
match_path_s1 = "['/s1']"
pid = "b' pid='"
__expected_string2 = '%s New value combination(s) detected\n%s: "%s" (%d lines)\n%s\n\n'
fixed_dme2 = FixedDataModelElement('s1', b'25537 uid=')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_sequence_me = MatchContext(b'25537 uid=2')
seq = SequenceModelElement('seq',
[fixed_dme2, decimal_integer_value_me])
match_element_sequence_me = seq.get_match_element(
'first', match_context_sequence_me)
string2 = " (b'25537 uid=', 2)\nb'25537 uid=2'"
spec = importlib.util.spec_from_file_location(
'aminer_config',
'/usr/lib/logdata-anomaly-miner/aminer/YamlConfig.py')
aminer_config = importlib.util.module_from_spec(spec)
spec.loader.exec_module(aminer_config)
aminer_config.load_yaml('unit/data/configfiles/suppress_config.yml')
context = AnalysisContext(aminer_config)
context.build_analysis_pipeline()
context.aminer_config.yaml_data['SuppressNewMatchPathDetector'] = False
self.stream_printer_event_handler = context.atomizer_factory.event_handler_list[
0]
self.stream_printer_event_handler.stream = self.output_stream
default_nmpd = context.registered_components[1][0]
default_nmpd.output_log_line = False
self.assertTrue(default_nmpd.receive_atom(log_atom_fixed_dme))
self.assertEqual(
self.output_stream.getvalue(), __expected_string1 %
(datetime.fromtimestamp(t).strftime(datetime_format_string),
default_nmpd.__class__.__name__, 'DefaultNewMatchPathDetector', 1,
match_path_s1, pid))
self.reset_output_stream()
context.aminer_config.yaml_data['SuppressNewMatchPathDetector'] = True
context = AnalysisContext(aminer_config)
context.build_analysis_pipeline()
self.stream_printer_event_handler = context.atomizer_factory.event_handler_list[
0]
self.stream_printer_event_handler.stream = self.output_stream
default_nmpd = context.registered_components[1][0]
default_nmpd.output_log_line = False
self.assertTrue(default_nmpd.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), "")
self.reset_output_stream()
value_combo_det = context.registered_components[2][0]
log_atom_sequence_me = LogAtom(
match_element_sequence_me.get_match_string(),
ParserMatch(match_element_sequence_me), t, value_combo_det)
self.stream_printer_event_handler = context.atomizer_factory.event_handler_list[
0]
self.stream_printer_event_handler.stream = self.output_stream
self.assertTrue(value_combo_det.receive_atom(log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), __expected_string2 %
(datetime.fromtimestamp(t).strftime(datetime_format_string),
value_combo_det.__class__.__name__, 'ValueComboDetector', 1,
string2))
self.reset_output_stream()
context.aminer_config.yaml_data['Analysis'][0]['suppress'] = True
context = AnalysisContext(aminer_config)
context.build_analysis_pipeline()
value_combo_det = context.registered_components[1][0]
self.stream_printer_event_handler = context.atomizer_factory.event_handler_list[
0]
self.stream_printer_event_handler.stream = self.output_stream
self.assertTrue(value_combo_det.receive_atom(log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), "")
self.reset_output_stream()
def test2start_year_value(self):
"""This test checks if they class is parsing dates without year values correctly."""
match_context = MatchContext(b'07.02 11:40:00: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m %H:%M:%S',
datetime.timezone.utc,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02 11:40:00 UTC+0000: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m %H:%M:%S %z',
datetime.timezone.utc,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02 11:40:00 UTC+0001: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486471200)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02 11:40:00 UTC+0000: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m %H:%M:%S %z',
datetime.timezone.utc,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07.02 11:40:00 UTC-0001: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486464000)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(b'07.02 11:40:00 CET+1: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m %H:%M:%S %z',
datetime.timezone.utc,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(b'07.02 11:40:00 CET+2: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486471200)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'07/Feb:11:40:00 +0000] "GET /login.php HTTP/1.1" 200 2532 "-" "Mozilla/5.0 (X11; Ubuntu; '
b'Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0')
date_time_model_element = DateTimeModelElement('path',
b'%d/%b:%H:%M:%S %z',
datetime.timezone.utc,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(
match_context.match_data,
b'] "GET /login.php HTTP/1.1" 200 2532 "-" "Mozilla/5.0 (X11; Ubuntu; '
b'Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0')
match_context = MatchContext(b'07.02 11:40:00 UTC: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m %H:%M:%S %z',
datetime.timezone.utc,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
class TimeCorrelationViolationDetectorTest(TestBase):
__expected_string = '%s Correlation rule "%s" violated\nTimeCorrelationViolationDetector: "%s" (%d lines)\n FAIL: '
__expected_string_not_found = __expected_string + 'B-Event for "%s" (%s) not found!\n\n'
__expected_string_too_early = __expected_string + 'B-Event for "%s" (%s) was found too early!\n\n\n'
__expected_string_too_late = __expected_string + 'B-Event for "%s" (%s) was not found in time!\n\n\n'
__expected_string_different_attributes = __expected_string + '"%s" (%s) %d is not equal %d\n\n\n'
model = '/model'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
service_children1 = [
FixedDataModelElement('Value1Key', b'Value1: '), FixedDataModelElement('Value1Value', b'fixed Value1'),
FixedDataModelElement('Value2Key', b', Value2: '), DecimalIntegerValueModelElement('Value2Value'),
FixedDataModelElement('Value3Key', b', Value3: '), FixedDataModelElement('Value3Value', b'fixed Value3'),
FixedDataModelElement('Value4Key', b', Value4: '), FixedDataModelElement('Value4Value', b'fixed Value4')]
service_children2 = [
FixedDataModelElement('Value1Key', b'Value1: '), FixedDataModelElement('Value1Value', b'fixed Value1'),
FixedDataModelElement('Value2Key', b', Value2: '), FixedDataModelElement('Value2Value', b'fixed Value2'),
FixedDataModelElement('Value3Key', b', Value3: '), DecimalIntegerValueModelElement('Value3Value'),
FixedDataModelElement('Value4Key', b', Value4: '), FixedDataModelElement('Value4Value', b'fixed Value4')]
match_context1 = MatchContext(b'Value1: fixed Value1, Value2: 22500, Value3: fixed Value3, Value4: fixed Value4')
match_context2 = MatchContext(b'Value1: fixed Value1, Value2: fixed Value2, Value3: 22500, Value4: fixed Value4')
match_context2_different = MatchContext(b'Value1: fixed Value1, Value2: fixed Value2, Value3: 22501, Value4: fixed Value4')
seq1 = SequenceModelElement('sequence1', service_children1)
seq2 = SequenceModelElement('sequence2', service_children2)
match_element1 = seq1.get_match_element(model, match_context1)
match_element2 = seq2.get_match_element(model, match_context2)
match_element2_different = seq2.get_match_element(model, match_context2_different)
def setUp(self):
TestBase.setUp(self)
self.correlation_rule = CorrelationRule('Correlation', 1, 1.2, max_artefacts_a_for_single_b=1, artefact_match_parameters=[
('/model/sequence1/Value2Value', '/model/sequence2/Value3Value')])
self.a_class_selector = EventClassSelector('Selector1', [self.correlation_rule], None)
self.b_class_selector = EventClassSelector('Selector2', None, [self.correlation_rule])
self.rules = []
self.rules.append(Rules.PathExistsMatchRule('/model/sequence1/Value2Key', self.a_class_selector))
self.rules.append(Rules.PathExistsMatchRule('/model/sequence2/Value3Key', self.b_class_selector))
def test1_check_status_ok(self):
"""In this test case the status is OK after receiving the expected data and no error message is returned. The output of the
do_timer-method is also tested in this test case. """
description = "Test1TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(time_correlation_violation_detector, component_name=description)
log_atom1 = LogAtom(self.match_context1.match_data, ParserMatch(self.match_element1), time.time(), self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data, ParserMatch(self.match_element2), time.time() + 1, self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(self.output_stream.getvalue(), "")
def test2_check_status_not_found_error(self):
"""In this test case the second log line is not found and an appropriate error message is expected from the check_status-method.
The output of the do_timer-method is also tested in this test case. """
description = "Test2TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data, ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
r = self.correlation_rule.check_status(t + 2)
self.assertEqual(r[0], 'FAIL: B-Event for "%s" (%s) was not found in time!\n' % (
self.match_element1.get_match_string().decode(), self.a_class_selector.action_id))
def test3_check_status_before_expected_timespan(self):
"""In this test case the second log line is found too early and an appropriate error message is expected from the
check_status-method. The output of the do_timer-method is also tested in this test case."""
description = "Test3TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data, ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data, ParserMatch(self.match_element2), time.time(), self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(self.output_stream.getvalue(), self.__expected_string_too_early % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), self.correlation_rule.rule_id, description, 1,
self.match_element1.get_match_string().decode(), self.a_class_selector.action_id))
def test4_check_status_after_expected_timespan(self):
"""In this test case the second log line is found too late and an appropriate error message is expected from the
check_status-method. The output of the do_timer-method is also tested in this test case."""
description = "Test4TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data, ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data, ParserMatch(self.match_element2), t + 5, self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(self.output_stream.getvalue(), self.__expected_string_too_late % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), self.correlation_rule.rule_id, description, 1,
self.match_element1.get_match_string().decode(), self.a_class_selector.action_id))
def test5_check_status_attributes_not_matching(self):
"""In this test case the second log line has different attributes than expected and an appropriate error message is expected from
the check_status-method. The output of the do_timer-method is also tested in this test case."""
description = "Test5TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data, ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data, ParserMatch(self.match_element2_different), t + 1, self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(self.output_stream.getvalue(), self.__expected_string_different_attributes % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), self.correlation_rule.rule_id, description, 1,
self.match_element1.get_match_string().decode(), self.a_class_selector.action_id, 22500, 22501))
def test6_prepare_history_entry(self):
"""In this test case the prepare_history_entry-method is tested with multiple artefact_match_parameters. Also the case of not
finding a parameter is tested."""
t = time.time()
p1 = ParserMatch(self.match_element1)
p2 = ParserMatch(self.match_element2)
log_atom1 = LogAtom(self.match_context1.match_data, p1, t, self)
log_atom2 = LogAtom(self.match_context2.match_data, p2, t + 5, self)
result = self.correlation_rule.prepare_history_entry(self.a_class_selector, log_atom1)
self.assertEqual(result, [t, 0, self.a_class_selector, p1, 22500])
result = self.correlation_rule.prepare_history_entry(self.b_class_selector, log_atom2)
self.assertEqual(result, [t + 5, 0, self.b_class_selector, p2, 22500])
def generate_errored_data(self, iterations, diff, error_rate):
"""Generate data with errors according to the error_rate."""
log_atoms = []
t = time()
divisor = 1
while error_rate * divisor < 1:
divisor = divisor * 10
err = divisor * error_rate
divisor //= err
for i in range(1, iterations+1):
if i % divisor == 0 and i != 0:
char = bytes([self.alphabet[int(i + random.uniform(diff+1, len(self.alphabet))) % len(self.alphabet)]])
else:
char = bytes([self.alphabet[i % len(self.alphabet)]])
parser_match = ParserMatch(self.alphabet_model.get_match_element('parser', MatchContext(char)))
t += diff
log_atoms.append(LogAtom(char, parser_match, t, self.__class__.__name__))
return log_atoms
def test2receive_atoms_with_defined_path_list(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is set to a static list of paths and variable_key_list should not be used.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler], path_list=['parser/type/path/nametype'])
results = [True, False, True, False, True, False, True, True, False, False, True, True, False, True, False, True, False, True,
False, False, True]
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(line, ParserMatch(self.parsing_model.get_match_element('parser', MatchContext(line))), t, self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
old_vals = (event_type_detector.num_events, event_type_detector.num_eventlines,
event_type_detector.total_records, event_type_detector.longest_path)
self.assertEqual(event_type_detector.receive_atom(log_atom), not results[i], i)
if results[i]:
self.assertEqual(old_vals, (
event_type_detector.num_events, event_type_detector.num_eventlines,
event_type_detector.total_records, event_type_detector.longest_path))
class RuleTest(TestBase):
"""
NOTE: DebugMatchRule and DebugHistoryMatchRule are intentionally not tested, as there is not much to be tested.
ParallelMatchRule is also not tested as it is very similar to the OrMatchRule.
"""
__expected_string = '%s This message was generated, when the unit were successful.\n%s: "%s" (%d lines)\n %s\n\n'
match_s1 = 'match/s1'
fixed_string = b'fixed String'
match_any = 'match/any'
alphabet = 'There are 26 letters in the english alphabet'
model_syslog_time = '/model/syslog/time'
model_syslog = '/model/syslog'
match_ipv4 = 'match/IPv4'
match_context_fixed_dme = MatchContext(b'25000')
fixed_dme = FixedDataModelElement('s1', b'25000')
match_element_fixed_dme = fixed_dme.get_match_element(
"fixed", match_context_fixed_dme)
def test1event_generation_match_action(self):
"""This test case checks if events are generated and pushed to all event handlers."""
description = "Test1Rules"
output_stream2 = StringIO()
message = 'This message was generated, when the unit were successful.'
match_context = MatchContext(b'25537')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
stream_printer_event_handler2 = StreamPrinterEventHandler(
self.analysis_context, output_stream2)
t = time()
event_generation_match_action = EventGenerationMatchAction(
'Test.%s' % self.__class__.__name__, message,
[self.stream_printer_event_handler, stream_printer_event_handler2])
self.analysis_context.register_component(event_generation_match_action,
description)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), t,
event_generation_match_action)
event_generation_match_action.match_action(log_atom)
self.assertEqual(self.output_stream.getvalue(),
output_stream2.getvalue())
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime("%Y-%m-%d %H:%M:%S"),
event_generation_match_action.__class__.__name__, description, 1,
log_atom.parser_match.match_element.annotate_match('')))
def test2atom_filter_match_action(self):
"""This test case proves the functionality of the AtomFilters."""
description = "Test2Rules"
newMatchPathDetector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
logAtomFixedDME = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
time(), newMatchPathDetector)
subhandlerFilter = SubhandlerFilter([newMatchPathDetector])
self.analysis_context.register_component(subhandlerFilter, description)
self.analysis_context.register_component(newMatchPathDetector,
description + "2")
self.assertTrue(subhandlerFilter.receive_atom(logAtomFixedDME))
def test3path_exists_match_rule(self):
"""This case unit the PathExistsMatchRule."""
description = "Test3Rules"
path_exists_match_rule = PathExistsMatchRule(self.match_s1, None)
self.analysis_context.register_component(path_exists_match_rule,
description)
self.fixed_dme = FixedDataModelElement('s1', self.fixed_string)
t = time()
match_context = MatchContext(self.fixed_string)
match_element = self.fixed_dme.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), t,
path_exists_match_rule)
self.assertTrue(path_exists_match_rule.match(log_atom))
self.fixed_dme = FixedDataModelElement('s2', self.fixed_string)
match_context = MatchContext(self.fixed_string)
match_element = self.fixed_dme.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), t,
path_exists_match_rule)
self.assertTrue(not path_exists_match_rule.match(log_atom))
def test4value_match_rule(self):
"""This case unit the ValueMatchRule."""
description = "Test4Rules"
value_match_rule = ValueMatchRule(self.match_s1, self.fixed_string,
None)
self.analysis_context.register_component(value_match_rule, description)
self.fixed_dme = FixedDataModelElement('s1', self.fixed_string)
match_context = MatchContext(self.fixed_string)
match_element = self.fixed_dme.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1, value_match_rule)
self.assertTrue(value_match_rule.match(log_atom))
self.fixed_dme = FixedDataModelElement('s1', b'another fixed String')
match_context = MatchContext(b'another fixed String')
match_element = self.fixed_dme.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1, value_match_rule)
self.assertTrue(not value_match_rule.match(log_atom))
def test5value_list_match_rule(self):
"""This case unit the ValueListMatchRule."""
description = "Test5Rules"
value_list_match_rule = ValueListMatchRule(
'match/d1', [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], None)
self.analysis_context.register_component(value_list_match_rule,
description)
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(b'64')
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_list_match_rule)
self.assertTrue(value_list_match_rule.match(log_atom))
match_context = MatchContext(b'4711')
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_list_match_rule)
self.assertTrue(not value_list_match_rule.match(log_atom))
def test6value_range_match_rule(self):
"""This case unit the ValueRangeMatchRule."""
description = "Test6Rules"
value_range_match_rule = ValueRangeMatchRule('match/d1', 1, 1000, None)
self.analysis_context.register_component(value_range_match_rule,
description)
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(b'1')
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_range_match_rule)
self.assertTrue(value_range_match_rule.match(log_atom))
match_context = MatchContext(b'1000')
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_range_match_rule)
self.assertTrue(value_range_match_rule.match(log_atom))
match_context = MatchContext(b'0')
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_range_match_rule)
self.assertTrue(not value_range_match_rule.match(log_atom))
match_context = MatchContext(b'1001')
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_range_match_rule)
self.assertTrue(not value_range_match_rule.match(log_atom))
def test7string_regex_match_rule(self):
"""This case unit the StringRegexMatchRule."""
description = "Test7Rules"
string_regex_match_rule = StringRegexMatchRule(self.match_any,
re.compile(r'\w'), None)
self.analysis_context.register_component(string_regex_match_rule,
description)
any_byte_date_me = AnyByteDataModelElement('any')
match_context = MatchContext(self.alphabet)
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
string_regex_match_rule)
self.assertTrue(string_regex_match_rule.match(log_atom))
match_context = MatchContext(
'--> There are 26 letters in the english alphabet')
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
string_regex_match_rule)
self.assertTrue(not string_regex_match_rule.match(log_atom))
def test8modulo_time_match_rule(self):
"""This case unit the ModuloTimeMatchRule."""
description = "Test8Rules"
modulo_time_match_rule = ModuloTimeMatchRule(self.model_syslog_time,
86400, 43200, 86400, None)
self.analysis_context.register_component(modulo_time_match_rule,
description)
date_time_model_element = DateTimeModelElement('time',
b'%d.%m.%Y %H:%M:%S',
timezone.utc)
match_context = MatchContext(b'14.02.2019 13:00:00')
match_element = date_time_model_element.get_match_element(
self.model_syslog, match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
date_time_model_element)
self.assertTrue(modulo_time_match_rule.match(log_atom))
match_context = MatchContext(b'15.02.2019 00:00:00')
match_element = date_time_model_element.get_match_element(
self.model_syslog, match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
date_time_model_element)
self.assertTrue(not modulo_time_match_rule.match(log_atom))
match_context = MatchContext(b'14.02.2019 12:00:00')
match_element = date_time_model_element.get_match_element(
self.model_syslog, match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
date_time_model_element)
self.assertTrue(modulo_time_match_rule.match(log_atom))
match_context = MatchContext(b'15.02.2019 01:00:00')
match_element = date_time_model_element.get_match_element(
self.model_syslog, match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
date_time_model_element)
self.assertTrue(not modulo_time_match_rule.match(log_atom))
def test9value_dependent_modulo_time_match_rule(self):
"""This case unit the ValueDependentModuloTimeMatchRule. Limit look up not working with tuples."""
description = "Test9Rules"
value_dependent_modulo_time_match_rule = ValueDependentModuloTimeMatchRule(
self.model_syslog_time, 86400, [self.model_syslog_time],
{1550145600: [43200, 86400]})
self.analysis_context.register_component(
value_dependent_modulo_time_match_rule, description)
date_time_model_element = DateTimeModelElement('time',
b'%d.%m.%Y %H:%M:%S',
timezone.utc)
match_context = MatchContext(b'14.02.2019 12:00:00')
match_element = date_time_model_element.get_match_element(
self.model_syslog, match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1550138400,
date_time_model_element)
self.assertTrue(value_dependent_modulo_time_match_rule.match(log_atom))
def test10ipv4_in_rfc1918_match_rule(self):
"""This case unit the ValueDependentModuloTimeMatchRule."""
description = "Test10Rules"
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule,
description)
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
# private addresses
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.16.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.31.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'10.0.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'10.255.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
# public addresses
match_context = MatchContext(b'192.167.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'192.169.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.15.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(not match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.32.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'9.255.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'11.0.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
def test11and_match_rule(self):
"""This case unit the AndMatchRule."""
description = "Test11Rules"
path_exists_match_rule = PathExistsMatchRule(self.match_ipv4, None)
self.analysis_context.register_component(path_exists_match_rule,
description)
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule,
description + "2")
and_match_rule = AndMatchRule(
[path_exists_match_rule, i_pv4_in_rfc1918_match_rule])
self.analysis_context.register_component(and_match_rule,
description + "3")
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(), and_match_rule)
self.assertTrue(and_match_rule.match(log_atom))
# changing to IPv6
path_exists_match_rule = PathExistsMatchRule('match/IPv6', None)
and_match_rule = AndMatchRule(
[path_exists_match_rule, i_pv4_in_rfc1918_match_rule])
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(), and_match_rule)
self.assertTrue(not and_match_rule.match(log_atom))
def test12or_match_rule(self):
"""This case unit the OrMatchRule."""
description = "Test12Rules"
path_exists_match_rule = PathExistsMatchRule(self.match_ipv4, None)
self.analysis_context.register_component(path_exists_match_rule,
description)
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule,
description + "2")
or_match_rule = OrMatchRule(
[path_exists_match_rule, i_pv4_in_rfc1918_match_rule])
self.analysis_context.register_component(or_match_rule,
description + "3")
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(), or_match_rule)
self.assertTrue(or_match_rule.match(log_atom))
# changing to IPv6
path_exists_match_rule = PathExistsMatchRule('match/IPv6', None)
or_match_rule = OrMatchRule(
[path_exists_match_rule, i_pv4_in_rfc1918_match_rule])
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(), or_match_rule)
self.assertTrue(or_match_rule.match(log_atom))
def test13value_dependent_delegated_match_rule(self):
"""This case unit the ValueDependentDelegatedMatchRule."""
description = "Test13Rules"
string_regex_match_rule = StringRegexMatchRule(self.match_any,
re.compile(r'\w'), None)
self.analysis_context.register_component(string_regex_match_rule,
description)
any_byte_date_me = AnyByteDataModelElement('any')
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule,
description + "2")
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
value_dependent_delegated_match_rule = ValueDependentDelegatedMatchRule(
[self.match_any, self.match_ipv4], {
(self.alphabet, None): string_regex_match_rule,
(None, 3232235520): i_pv4_in_rfc1918_match_rule
})
self.analysis_context.register_component(
value_dependent_delegated_match_rule, description + "3")
match_context = MatchContext(self.alphabet)
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(value_dependent_delegated_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(value_dependent_delegated_match_rule.match(log_atom))
# not matching values
match_context = MatchContext(
'.There are 26 letters in the english alphabet')
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(
not value_dependent_delegated_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.0.1')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(
not value_dependent_delegated_match_rule.match(log_atom))
def test14negation_match_rule(self):
"""This case unit the NegationMatchRule."""
description = "Test14Rules"
path_exists_match_rule = PathExistsMatchRule(self.match_s1, None)
self.analysis_context.register_component(path_exists_match_rule,
description)
negation_match_rule = NegationMatchRule(path_exists_match_rule)
self.analysis_context.register_component(negation_match_rule,
description + "2")
self.fixed_dme = FixedDataModelElement('s1', self.fixed_string)
match_context = MatchContext(self.fixed_string)
match_element = self.fixed_dme.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
path_exists_match_rule)
self.assertTrue(path_exists_match_rule.match(log_atom))
self.assertTrue(not negation_match_rule.match(log_atom))
def consume_data(self, stream_data, end_of_stream_flag=False):
"""
Consume data from the underlying stream for atomizing.
@return the number of consumed bytes, 0 if the atomizer would need more data for a complete atom or -1 when no data was
consumed at the moment but data might be consumed later on.
"""
# Loop until as much streamData as possible was processed and then return a result. The correct processing of endOfStreamFlag
# is tricky: by default, even when all data was processed, do one more iteration to handle also the flag.
consumed_length = 0
while True:
if self.last_unconsumed_log_atom is not None:
# Keep length before dispatching: dispatch will reset the field.
data_length = len(self.last_unconsumed_log_atom.raw_data)
if self.dispatch_atom(self.last_unconsumed_log_atom):
consumed_length += data_length + len(self.eol_sep)
continue
# Nothing consumed, tell upstream to wait if appropriate.
if consumed_length == 0:
consumed_length = -1
break
line_end = None
global breakout # skipcq: PYL-W0603
breakout = False
global data # skipcq: PYL-W0603
data = None
valid_json = False
if self.json_format:
state = json_machine(found_json)
i = 0
for i, char in enumerate(stream_data[consumed_length:]):
state = state(char)
if breakout or state is None or i > self.max_line_length:
break
# check if the json is still valid, but the stream_data is at the end
if not breakout and state is not None and i + consumed_length == len(
stream_data) - 1 and not end_of_stream_flag:
return consumed_length
if 0 < i <= self.max_line_length and b'{' in stream_data[
consumed_length:consumed_length + i +
1] and data is not None:
line_end = consumed_length + i + 1
valid_json = True
elif i > self.max_line_length:
self.in_overlong_line_flag = True
if line_end is None:
line_end = stream_data.find(self.eol_sep, consumed_length)
if self.in_overlong_line_flag:
if line_end < 0:
consumed_length = len(stream_data)
if end_of_stream_flag:
self.dispatch_event(
'Overlong line terminated by end of stream',
stream_data)
self.in_overlong_line_flag = False
break
consumed_length = line_end + len(self.eol_sep)
self.in_overlong_line_flag = False
continue
# This is the valid start of a normal/incomplete/overlong line.
if line_end < 0:
tail_length = len(stream_data) - consumed_length
if tail_length > self.max_line_length:
self.dispatch_event('Start of overlong line detected',
stream_data[consumed_length:])
self.in_overlong_line_flag = True
consumed_length = len(stream_data)
# Stay in loop to handle also endOfStreamFlag!
continue
if end_of_stream_flag and (tail_length != 0):
self.dispatch_event('Incomplete last line',
stream_data[consumed_length:])
consumed_length = len(stream_data)
break
# This is at least a complete/overlong line.
line_length = line_end + len(self.eol_sep) - consumed_length
if line_length > self.max_line_length and not valid_json:
self.dispatch_event('Overlong line detected',
stream_data[consumed_length:line_end])
consumed_length = line_end + len(self.eol_sep)
continue
# This is a normal line.
line_data = stream_data[consumed_length:line_end]
log_atom = LogAtom(line_data, None, None, self)
if self.parsing_model is not None:
match_context = MatchContext(line_data)
match_element = self.parsing_model.get_match_element(
'', match_context)
if (match_element
is not None) and not match_context.match_data:
log_atom.parser_match = ParserMatch(match_element)
for default_timestamp_path in self.default_timestamp_paths:
ts_match = log_atom.parser_match.get_match_dictionary(
).get(default_timestamp_path, None)
if ts_match is not None:
log_atom.set_timestamp(ts_match.match_object)
break
if self.dispatch_atom(log_atom):
consumed_length = line_end + len(self.eol_sep) - (
valid_json
and stream_data[line_end:line_end + len(self.eol_sep)] !=
self.eol_sep)
continue
if consumed_length == 0:
# Downstream did not want the data, so tell upstream to block for a while.
consumed_length = -1
break
return consumed_length
def test10ipv4_in_rfc1918_match_rule(self):
"""This case unit the ValueDependentModuloTimeMatchRule."""
description = "Test10Rules"
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule,
description)
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
# private addresses
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.16.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.31.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'10.0.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'10.255.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom))
# public addresses
match_context = MatchContext(b'192.167.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'192.169.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.15.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(not match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'172.32.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'9.255.255.255')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
match_context = MatchContext(b'11.0.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), time(),
i_pv4_in_rfc1918_match_rule)
self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
class PersistenceUtilTest(TestBase):
"""Unittests for the PersistenceUtil class."""
string = b'25537 uid=2'
match_context_fixed_dme = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
match_element_fixed_dme = fixed_dme.get_match_element(
"", match_context_fixed_dme)
match_context_decimal_integer_value_me = MatchContext(string)
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element_decimal_integer_value_me = decimal_integer_value_me.get_match_element(
"", match_context_decimal_integer_value_me)
fixed_dme = FixedDataModelElement('s1', string)
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_first_match_me = MatchContext(string)
first_match_me = FirstMatchModelElement(
'f1', [fixed_dme, decimal_integer_value_me])
match_element_first_match_me = first_match_me.get_match_element(
'first', match_context_first_match_me)
match_context_first_match_me2 = MatchContext(string)
first_match_me2 = FirstMatchModelElement(
'f2', [decimal_integer_value_me, fixed_dme])
match_element_first_match_me2 = first_match_me2.get_match_element(
'second', match_context_first_match_me2)
def test1persist_multiple_objects_of_single_class(self):
"""In this test case multiple instances of one class are to be persisted and loaded."""
description = "Test1PersistenceUtil"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
True)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(
self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t,
new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me)
other_new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'otherDetector', True)
self.analysis_context.register_component(other_new_match_path_detector,
description + "2")
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, other_new_match_path_detector)
other_new_match_path_detector.receive_atom(log_atom_fixed_dme)
PersistenceUtil.persist_all()
persistence_data = PersistenceUtil.load_json(
new_match_path_detector.persistence_file_name)
self.assertTrue(persistence_data in ([
self.match_element_fixed_dme.get_path(),
self.match_element_decimal_integer_value_me.get_path()
], [
self.match_element_decimal_integer_value_me.get_path(),
self.match_element_fixed_dme.get_path()
]))
self.assertEqual(
PersistenceUtil.load_json(
other_new_match_path_detector.persistence_file_name),
[self.match_element_fixed_dme.get_path()])
def test2persist_multiple_objects_of_multiple_class(self):
"""In this test case multiple instances of multiple classes are to be persisted and loaded."""
description = "Test2PersistenceUtil"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default2', True)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(
self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t,
new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me)
other_new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'otherDetector2', True)
self.analysis_context.register_component(other_new_match_path_detector,
description + "2")
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, other_new_match_path_detector)
other_new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, ['first/f1/s1'],
[self.stream_printer_event_handler], 'Default', False, True)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description + "3")
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_first_match_me), t,
new_match_path_value_combo_detector)
new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me)
PersistenceUtil.persist_all()
persistence_data = PersistenceUtil.load_json(
new_match_path_detector.persistence_file_name)
self.assertTrue(persistence_data in ([
self.match_element_fixed_dme.get_path(),
self.match_element_decimal_integer_value_me.get_path()
], [
self.match_element_decimal_integer_value_me.get_path(),
self.match_element_fixed_dme.get_path()
]))
self.assertEqual(
PersistenceUtil.load_json(
other_new_match_path_detector.persistence_file_name),
[self.match_element_fixed_dme.get_path()])
self.assertEqual(
PersistenceUtil.load_json(
new_match_path_value_combo_detector.persistence_file_name),
([[log_atom_sequence_me.raw_data]]))
def test4max_items_overflow(self):
"""In this test case more events than the VolatileLogarithmicBackoffEventHistory can handle are received."""
deviation = 0.05
size = 100000
msg = "%s=%f is not between %f and %f"
match_context = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s1', self.pid)
match_element = fixed_dme.get_match_element("match", match_context)
t = time()
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self)
message = self.new_val % (self.match_s1, self.match_s2, repr(match_element.match_object))
first = 0
second = 0
third = 0
fourth = 0
for _ in range(size):
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(2)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
history = volatile_logarithmic_backoff_event_history.get_history()
if history == [(0, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
first += 1
elif history == [(1, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
second += 1
elif history == [(2, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
third += 1
elif history == [(3, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
fourth += 1
val = 0.5 * 0.5 * 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= first <= maximum, msg % ("first", first, minimum, maximum))
val = 0.5 * 0.5 * 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= second <= maximum, msg % ("second", second, minimum, maximum))
val = 2 * 0.5 * 0.5 * 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= third <= maximum, msg % ("third", third, minimum, maximum))
val = 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= fourth <= maximum, msg % ("fourth", fourth, minimum, maximum))
class NewMatchPathValueComboDetectorTest(TestBase):
"""Unittests for the NewMatchPathValueComboDetector."""
__expected_string = '%s New value combination(s) detected\n%s: "%s" (%d lines)\n%s\n\n'
fixed_dme = FixedDataModelElement('s1', b'25537 uid=')
fixed_dme2 = FixedDataModelElement('s2', b' uid=2')
datetime_format_string = '%Y-%m-%d %H:%M:%S'
first_seq_s1 = 'first/seq/s1'
first_seq_d1 = 'first/seq/d1'
string = " first/seq: b'25537 uid=2'\n " + first_seq_s1 + ": 25537 uid=\n " + first_seq_d1 + ": 2\n(b'25537 uid=', 2)"
string2 = " (b'25537 uid=', 2)\n25537 uid=2"
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_sequence_me = MatchContext(b'25537 uid=2')
seq = SequenceModelElement('seq', [fixed_dme, decimal_integer_value_me])
match_element_sequence_me = seq.get_match_element(
'first', match_context_sequence_me)
match_context_sequence_me2 = MatchContext(b'25537 uid=2')
seq2 = SequenceModelElement('seq2', [decimal_integer_value_me, fixed_dme2])
match_element_sequence_me2 = seq2.get_match_element(
'second', match_context_sequence_me2)
def test1_log_atom_not_known(self):
"""
This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found.
The output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be
repeatable on second run.
"""
description = "Test1NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(
new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
self.reset_output_stream()
# repeating should produce the same result
self.assertTrue(
new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
self.reset_output_stream()
new_match_path_value_combo_detector2 = NewMatchPathValueComboDetector(
self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(
new_match_path_value_combo_detector2.receive_atom(
log_atom_sequence_me2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1, " (25537, b' uid=2')\n25537 uid=2"))
def test2_log_atom_known(self):
"""
This test case checks the functionality of the auto_include_flag.
If the same MatchElement is processed a second time and the auto_include_flag was True, no event must be triggered.
"""
description = "Test2NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(
new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
self.reset_output_stream()
# repeating should NOT produce the same result
self.assertTrue(
new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
new_match_path_value_combo_detector2 = NewMatchPathValueComboDetector(
self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(
new_match_path_value_combo_detector2.receive_atom(
log_atom_sequence_me2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1, " (25537, b' uid=2')\n25537 uid=2"))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(
new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
new_match_path_value_combo_detector.do_persist()
self.reset_output_stream()
other_new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
other_new_match_path_value_combo_detector, description + "2")
other_log_atom_fixed_dme = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
other_new_match_path_value_combo_detector)
self.assertTrue(
other_new_match_path_value_combo_detector.receive_atom(
other_log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), '')
def test4_allowlist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test4NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(
new_match_path_value_combo_detector.allowlist_event(
'Analysis.%s' %
new_match_path_value_combo_detector.__class__.__name__,
self.match_element_sequence_me.get_path(), None),
'Allowlisted path(es) %s with %s.' %
(", ".join(new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me.get_path()))
new_match_path_value_combo_detector.auto_include_flag = False
self.assertEqual(
new_match_path_value_combo_detector.allowlist_event(
'Analysis.%s' %
new_match_path_value_combo_detector.__class__.__name__,
self.match_element_sequence_me2.get_path(), None),
'Allowlisted path(es) %s with %s.' %
(", ".join(new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me2.path))
class JsonConverterHandlerTest(TestBase):
"""Unittests for the JsonConverterHandler."""
output_log_line = True
match_context = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
match_element = fixed_dme.get_match_element("match", match_context)
t = time.time()
test_detector = 'Analysis.TestDetector'
event_message = 'An event happened!'
sorted_log_lines = ['Event happend at /path/ 5 times.', '', '', '', '']
persistence_id = 'Default'
description = 'jsonConverterHandlerDescription'
expected_string = '%s %s\n%s: "%s" (5 lines)\n {\n "AnalysisComponent": {\n "AnalysisComponentIdentifier": 0,\n' \
' "AnalysisComponentType": "%s",\n "AnalysisComponentName": "%s",\n "Message": "%s",\n' \
' "PersistenceFileName": "%s",\n "AffectedParserPaths": [\n "test/path/1",\n' \
' "test/path/2"\n ]\n },\n "LogData": {\n "RawLogData": [\n " pid="\n ],\n ' \
'"Timestamps": [\n %s\n ],\n "DetectionTimestamp": %s,\n "LogLinesCount": 5,\n' \
' "AnnotatedMatchElement": "match/s1: b\' pid=\'"\n }%s\n}\n\n'
def test1receive_expected_event(self):
"""In this test case a normal Event happens and the json output should be sent to a StreamPrinterEventHandler."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
detection_timestamp = None
for line in self.output_stream.getvalue().split('\n'):
if "DetectionTimestamp" in line:
detection_timestamp = line.split(':')[1].strip(' ,')
self.assertEqual(
self.output_stream.getvalue(), self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), detection_timestamp, ""))
def test2receive_event_with_same_event_data_attributes(self):
"""In this test case an attribute of AnalysisComponent is overwritten and an JsonError attribute is expected."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2'],
'Message': 'An other event happened too!'
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
detection_timestamp = None
for line in self.output_stream.getvalue().split('\n'):
if "DetectionTimestamp" in line:
detection_timestamp = line.split(':')[1].strip(' ,')
self.assertEqual(
self.output_stream.getvalue(), self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), detection_timestamp,
',\n "JsonError": "AnalysisComponent attribute \'Message\' is already in use and can not be overwritten!\\n"'
))
def test10_missing_value_on_persisted(self):
"""Persisting lists is tested in this test case."""
description = "Test91MissingMatchPathValueDetector"
match_context_fixed_dme = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s2', self.pid)
match_element_fixed_dme = fixed_dme.get_match_element(
"match3", match_context_fixed_dme)
match_context_decimal_integer_value_me = MatchContext(self.string)
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd2', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element_decimal_integer_value_me = decimal_integer_value_me.get_match_element(
"match4", match_context_decimal_integer_value_me)
missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, [
match_element_fixed_dme.get_path(),
match_element_decimal_integer_value_me.get_path()
], [self.stream_printer_event_handler], 'Default', True,
self.__default_interval, self.__realert_interval)
self.analysis_context.register_component(
missing_match_path_list_value_detector, description)
log_atom_fixed_dme = LogAtom(fixed_dme.fixed_data,
ParserMatch(match_element_fixed_dme),
round(time.time()),
missing_match_path_list_value_detector)
self.assertTrue(
missing_match_path_list_value_detector.receive_atom(
log_atom_fixed_dme))
missing_match_path_list_value_detector.do_persist()
past_time = 4000
t = time.time()
other_missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, [
match_element_fixed_dme.get_path(),
match_element_decimal_integer_value_me.get_path()
], [self.stream_printer_event_handler], 'Default', True,
self.__default_interval, self.__realert_interval)
self.analysis_context.register_component(
other_missing_match_path_list_value_detector, description + "2")
other_missing_match_path_list_value_detector.set_check_value(
other_missing_match_path_list_value_detector.get_channel_key(
log_atom_fixed_dme), self.__default_interval - past_time)
log_atom_fixed_dme = LogAtom(
fixed_dme.fixed_data, ParserMatch(match_element_fixed_dme),
round(time.time()) + past_time,
other_missing_match_path_list_value_detector)
self.assertTrue(
other_missing_match_path_list_value_detector.receive_atom(
log_atom_fixed_dme))
self.assertTrue(
(self.output_stream.getvalue() == self.__expected_string %
(datetime.fromtimestamp(t + past_time).strftime(
self.datetime_format_string),
other_missing_match_path_list_value_detector.__class__.__name__,
description + "2", 1,
"match3/s2, match4/d2: b' pid=' overdue 400s (interval -400)"))
or
(self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + past_time + 1).strftime(
self.datetime_format_string),
other_missing_match_path_list_value_detector.__class__.__name__,
description + "2", 1,
"match3/s2, match4/d2: b' pid=' overdue 400s (interval -400)")))
class AtomFiltersTest(TestBase):
match_context_fixed_dme = MatchContext(b'25000')
fixed_dme = FixedDataModelElement('s1', b'25000')
match_element_fixed_dme = fixed_dme.get_match_element(
"fixed", match_context_fixed_dme)
match_context_decimal_integer_value_me = MatchContext(b'25000')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element_decimal_integer_value_me = decimal_integer_value_me.get_match_element(
"integer", match_context_decimal_integer_value_me)
def test1_no_list_or_no_atom_handler_list(self):
"""This test case verifies, that exceptions are raised when using wrong parameters."""
self.assertRaises(
Exception, SubhandlerFilter,
NewMatchPathDetector(self.aminer_config, [], 'Default', True),
False)
self.assertRaises(Exception, SubhandlerFilter,
FixedDataModelElement('fixed', b'gesuchter String'),
False)
def test2receive_atom_unhandled(self):
"""In this test case no handler can handle the log atom."""
description = "Test2AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
subhandler_filter = SubhandlerFilter([], True)
self.assertTrue(not subhandler_filter.receive_atom(log_atom_fixed_dme))
def test3receive_atom_handled_by_more_handlers(self):
"""In this test case more than one handler can handle the log atom. The impact of the stop_when_handled flag is tested."""
description = "Test3AtomFilters"
other_description = "Test3OtherAtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
other_new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(other_new_match_path_detector,
other_description)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
subhandler_filter = SubhandlerFilter(
[new_match_path_detector, other_new_match_path_detector], False)
self.assertTrue(subhandler_filter.receive_atom(log_atom_fixed_dme))
result = self.output_stream.getvalue()
self.reset_output_stream()
new_match_path_detector.receive_atom(log_atom_fixed_dme)
resultFixedDME = self.output_stream.getvalue()
self.reset_output_stream()
other_new_match_path_detector.receive_atom(log_atom_fixed_dme)
result_decimal_integer_value_me = self.output_stream.getvalue()
self.assertEqual(result,
resultFixedDME + result_decimal_integer_value_me)
def test4match_path_filter_receive_atom_path_in_dictionary(self):
"""There is a path in the dictionary and the handler is not None. The default_parsed_atom_handler is None."""
description = "Test4AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
match_path_filter = MatchPathFilter([
(self.match_element_fixed_dme.get_path(), new_match_path_detector)
], None)
self.assertTrue(match_path_filter.receive_atom(log_atom_fixed_dme))
def test5match_path_filter_receive_atom_path_not_in_dictionary(self):
"""The searched path is not in the dictionary. The default_parsed_atom_handler is None."""
description = "Test5AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
match_path_filter = MatchPathFilter(
[(self.match_element_decimal_integer_value_me.get_path(),
new_match_path_detector)], None)
self.assertTrue(not match_path_filter.receive_atom(log_atom_fixed_dme))
def test6match_path_filter_receive_atom_path_not_in_dictionary_default_set(
self):
"""The searched path is not in the dictionary. The default_parsed_atom_handler is set."""
description = "Test6AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
match_path_filter = MatchPathFilter(
[(self.match_element_decimal_integer_value_me.get_path(),
new_match_path_detector)], new_match_path_detector)
self.assertTrue(match_path_filter.receive_atom(log_atom_fixed_dme))
def test7match_value_filter_receive_atom_target_value_and_handler_found(
self):
"""A target_value and a handler, which can handle the matchObject is found."""
description = "Test7AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
match_value_filter = MatchValueFilter(
self.match_element_fixed_dme.get_path(),
{self.fixed_dme.fixed_data: new_match_path_detector}, None)
self.assertTrue(match_value_filter.receive_atom(log_atom_fixed_dme))
def test8match_value_filter_receive_atom_target_value_found_handler_not_found(
self):
"""A target_value was found, but no handler can handle it. DefaultParsedAtomHandler = None"""
description = "Test8AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
match_value_filter = MatchValueFilter(
self.match_element_fixed_dme.get_path(),
{self.fixed_dme.fixed_data: None}, None)
self.assertTrue(
not match_value_filter.receive_atom(log_atom_fixed_dme))
def test9match_value_filter_receive_atom_target_value_not_found(self):
"""No target_value was found in the dictionary."""
description = "Test9AtomFilters"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler], 'Default',
False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(b'24999',
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
match_value_filter = MatchValueFilter(
self.match_element_fixed_dme.get_path(),
{self.fixed_dme.fixed_data: None}, None)
self.assertTrue(
not match_value_filter.receive_atom(log_atom_fixed_dme))
def test1log_multiple_lines_event(self):
"""
In this test case multiple lines should be received, before sending an email to root@localhost.
Make sure no mail notifications are in /var/spool/mail/root, before running this test. This test case must wait some time to
ensure, that the mail can be read.
"""
description = "Test1DefaultMailNotificationEventHandler"
match_context = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s1', self.pid)
match_element = fixed_dme.get_match_element("match", match_context)
match_context = MatchContext(self.pid)
fixed_dme2 = FixedDataModelElement('s2', self.pid)
match_element2 = fixed_dme2.get_match_element("match", match_context)
default_mail_notification_event_handler = DefaultMailNotificationEventHandler(
self.analysis_context)
self.analysis_context.register_component(self, description)
t = time()
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t,
self)
default_mail_notification_event_handler.receive_event(
self.test % self.__class__.__name__,
'New value for pathes %s, %s: %s' %
('match/s1', 'match/s2', repr(match_element.match_object)),
[log_atom.raw_data, log_atom.raw_data], None, log_atom, self)
t += 600
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t,
self)
# set the next_alert_time instead of sleeping 10 seconds
default_mail_notification_event_handler.next_alert_time = time()
default_mail_notification_event_handler.receive_event(
self.test % self.__class__.__name__,
'New value for pathes %s, %s: %s' %
('match/s1', 'match/s2', repr(match_element.match_object)),
[log_atom.raw_data, log_atom.raw_data], None, log_atom, self)
sleep(2)
# skipcq: PYL-W1510, BAN-B602
result = subprocess.run(self.mail_call,
shell=True,
stdout=subprocess.PIPE)
# skipcq: PYL-W1510, BAN-B602
subprocess.run(self.mail_delete_call,
shell=True,
stdout=subprocess.PIPE)
if datetime.fromtimestamp(t - 600).strftime(
self.datetime_format_string) not in str(
result.stdout, 'utf-8'):
print("ERROR: %s t-600 not found in mail!" % description,
file=sys.stderr)
if datetime.fromtimestamp(t).strftime(
self.datetime_format_string) not in str(
result.stdout, 'utf-8'):
print("ERROR: %s t not found in mail!" % description,
file=sys.stderr)
self.assertTrue(
self.__expected_string %
("" + match_element.get_path() + ", " + match_element2.get_path(),
match_element.get_match_object(), self.__class__.__name__,
description, 2, match_element.get_match_string().decode() +
"\n " + match_element2.get_match_string().decode())
in str(result.stdout, 'utf-8'),
msg="%s vs \n %s" %
(self.__expected_string %
(match_element.get_path(), match_element.get_match_object(),
self.__class__.__name__, description, 1,
match_element.get_match_string().decode() + "\n\n"),
str(result.stdout, 'utf-8')))
self.assertTrue(
self.__expected_string %
("" + match_element.get_path() + ", " + match_element2.get_path(),
match_element.get_match_object(), self.__class__.__name__,
description, 2, match_element.get_match_string().decode() +
"\n " + match_element2.get_match_string().decode() +
"\n\n") in str(result.stdout, 'utf-8'))
class KafkaEventHandlerTest(TestBase):
"""Unittests for the KafkaEventHandler."""
output_log_line = True
kafka_topic = 'test_topic'
kafka_group = 'test_group'
consumer = None
match_context = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
other_data = 4
match_element = fixed_dme.get_match_element("match", match_context)
description = 'jsonConverterHandlerDescription'
t = time.time()
persistence_id = 'Default'
test_detector = 'Analysis.TestDetector'
event_message = 'An event happened!'
sorted_log_lines = ['Event happend at /path/ 5 times.', '', '', '', '']
expected_string = '{\n "AnalysisComponent": {\n "AnalysisComponentIdentifier": 0,\n' \
' "AnalysisComponentType": "%s",\n "AnalysisComponentName": "%s",\n "Message": "%s",\n' \
' "PersistenceFileName": "%s",\n "AffectedParserPaths": [\n "test/path/1",\n' \
' "test/path/2"\n ]\n },\n "LogData": {\n "RawLogData": [\n " pid="\n ],\n ' \
'"Timestamps": [\n %s\n ],\n "DetectionTimestamp": %s,\n "LogLinesCount": 5,\n' \
' "AnnotatedMatchElement": {\n "match/s1": " pid="\n }\n }%s\n}\n'
@classmethod
def setUpClass(cls):
"""Start a KafkaConsumer."""
cls.consumer = KafkaConsumer(cls.kafka_topic,
bootstrap_servers=['localhost:9092'],
enable_auto_commit=True,
consumer_timeout_ms=10000,
group_id=cls.kafka_group,
value_deserializer=lambda x: x.decode(),
api_version=(2, 0, 1),
auto_offset_reset='earliest')
@classmethod
def tearDownClass(cls):
"""Shutdown the KafkaConsumer."""
cls.consumer.close()
def test1receive_serialized_data(self):
"""This unittest tests the receive_event method with serialized data from the JsonConverterHandler."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
output = self.output_stream.getvalue()
kafka_event_handler = KafkaEventHandler(
self.analysis_context, self.kafka_topic, {
'bootstrap_servers': ['localhost:9092'],
'api_version': (2, 0, 1)
})
self.assertTrue(
kafka_event_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, output,
log_atom, self))
val = self.consumer.__next__().value
detection_timestamp = None
for line in val.split('\n'):
if "DetectionTimestamp" in line:
detection_timestamp = line.split(':')[1].strip(' ,')
self.assertEqual(
val, self.expected_string %
(self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), detection_timestamp, ""))
def test2receive_non_serialized_data(self):
"""This unittest tests the receive_event method with not serialized data."""
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
kafka_event_handler = KafkaEventHandler(
self.analysis_context, self.kafka_topic, {
'bootstrap_servers': ['localhost:9092'],
'api_version': (2, 0, 1)
})
self.assertFalse(
kafka_event_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines,
event_data, log_atom, self))
self.assertRaises(StopIteration, self.consumer.__next__)
def test1update_successful(self):
"""Update the MatchContext and DebugMatchContext with allowed values."""
data = b"this is an example of a log line."
match_context = MatchContext(data)
match_context.update(b"this is an example")
self.assertEqual(match_context.match_data, b" of a log line.")
match_context = MatchContext(data)
match_context.update([b"t", b"h", b"i", b"s"])
self.assertEqual(match_context.match_data,
b" is an example of a log line.")
match_context = MatchContext(data)
match_context.update(b"some other text")
self.assertEqual(match_context.match_data, b"ple of a log line.")
match_context = DebugMatchContext(data)
match_context.update(b"this is an example ")
self.assertEqual(match_context.match_data, b"of a log line.")
self.assertEqual(
match_context.get_debug_info(),
'Starting match update on "this is an example of a log line."\n Removed: "this is an example ", remaining 14'
' bytes\n Shortest unmatched data: "of a log line."\n')
self.assertEqual(match_context.get_debug_info(),
' Shortest unmatched data: "of a log line."\n')
match_context.update(b"of")
self.assertEqual(
match_context.get_debug_info(),
' Removed: "of", remaining 12 bytes\n Shortest unmatched data: " a log line."\n'
)
match_context.update(b" a log line.")
self.assertEqual(
match_context.get_debug_info(),
' Removed: " a log line.", remaining 0 bytes\n Shortest unmatched data: ""\n'
)
self.assertRaises(ValueError, match_context.update, b" a log line.")
self.assertEqual(
match_context.get_debug_info(),
' Current data does not start with " a log line."\n Shortest unmatched data: ""\n'
)
match_context.update(b"")
def parse_json_object(self, json_dict, json_match_data, key, split_key,
current_path, match_context): # skipcq: PYL-R0201
"""Parse a literal from the json object."""
data = json_match_data[split_key]
enc = "utf-8"
if isinstance(data, str):
if self.is_escaped_unicode(data) and self.dec_escapes:
enc = "unicode-escape"
data = data.encode(enc)
elif isinstance(data, bool):
data = str(data).replace("T", "t").replace("F", "f").encode()
elif data is None:
data = b"null"
if self.is_nullable_key(key) or json_dict[key] == "NULL_OBJECT":
start = 0
if "null" in key:
start = match_context.match_data.find(data) + 4
index = match_context.match_data.find(data, start)
if match_context.match_data[index + 4] == 34:
index += 1
return MatchElement(current_path, data, data,
None), index, data
return None, -1, data
elif not isinstance(data, bytes):
data = str(data).encode()
if json_dict[key] == "ALLOW_ALL":
logging.getLogger(DEBUG_LOG_NAME).debug(debug_log_prefix +
"ALLOW_ALL (DICT)\n" +
data.decode())
match_element = MatchElement(current_path, data, data, None)
last_bracket = match_context.match_data.find(b"}", len(data))
while match_context.match_data.count(
b"{", 0, last_bracket) - match_context.match_data.count(
b"}", 0, last_bracket) > 0:
last_bracket = match_context.match_data.find(
b"}", last_bracket) + 1
index = last_bracket - len(data)
elif json_dict[key] == "EMPTY_STRING":
if data == b"":
match_element = MatchElement(current_path, data, data, None)
index = match_context.match_data.find(
split_key.encode()) + len(split_key)
index += match_context.match_data[index:].find(b'""') + len(
b'""')
else:
match_element = None
index = -1
else:
match_element = json_dict[key].get_match_element(
current_path, MatchContext(data))
if match_element is not None and len(
match_element.match_string) != len(data) and (
not isinstance(match_element.match_object, bytes)
or len(match_element.match_object) != len(data)):
logging.getLogger(DEBUG_LOG_NAME).debug(
debug_log_prefix +
"Data length not matching! match_string: %d, data: %d, data: %s"
% (len(match_element.match_string), len(data),
data.decode()))
match_element = None
index = max([
match_context.match_data.replace(b"\\",
b"").find(split_key.encode()),
match_context.match_data.find(split_key.encode()),
match_context.match_data.decode().find(split_key)
])
index += match_context.match_data[index:].find(
split_key.encode() + b'":') + len(split_key.encode() + b'":')
try:
index += max([
match_context.match_data.replace(b"\\",
b"")[index:].find(data),
match_context.match_data[index:].find(data),
match_context.match_data.decode(enc)[index:].find(
data.decode(enc))
])
except UnicodeDecodeError:
index += max([
match_context.match_data.replace(b"\\",
b"")[index:].find(data),
match_context.match_data[index:].find(data),
match_context.match_data.decode()[index:].find(
data.decode())
])
index += len(match_context.match_data[index:]) - len(
match_context.match_data[index:].lstrip(b" \r\t\n"))
if match_context.match_data[index:].find(b'"') == 0:
index += len(b'"')
# for example float scientific representation is converted to normal float..
if index == -1 and match_element is not None and isinstance(
json_match_data[split_key], float):
indices = [
match_context.match_data.find(
b",",
len(match_element.match_string) // 3),
match_context.match_data.find(b"]"),
match_context.match_data.find(b"}")
]
indices = [x for x in indices if x >= 0]
index = min(indices)
if match_element is None:
index = -1
return match_element, index, data
