def test19date_before_unix_timestamps(self):
"""Check if timestamps before the unix timestamp are processed properly."""
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S", None, None)])
match_context = DummyMatchContext(b"01.01.1900 11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"01.01.1900 11:40:00")
self.assertEqual(match_element.match_object, -2208946800)
def test21date_formats_input_validation(self):
"""Check if date_format is validated and only valid values can be entered."""
allowed_format_specifiers = b"bdfHMmSsYz%"
# check if allowed values do not raise any exception.
format_specifiers = b""
for c in allowed_format_specifiers:
format_specifiers += b"%" + str(chr(c)).encode()
MultiLocaleDateTimeModelElement("s0", [(b"%" + str(chr(c)).encode(), None, None)])
# check if all allowed values can not be used together. An exception should be raised, because of multiple month representations
# and %s with non-second formats.
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", format_specifiers)
MultiLocaleDateTimeModelElement("s0", [(format_specifiers.replace(b"%m", b"").replace(b"%s", b""), None, None)])
MultiLocaleDateTimeModelElement("s0", [(format_specifiers.replace(b"%b", b"").replace(b"%s", b""), None, None)])
MultiLocaleDateTimeModelElement("s0", [(b"%s%z%f", None, None)])
for c in allowed_format_specifiers.replace(b"s", b"").replace(b"z", b"").replace(b"f", b"").replace(b"%", b""):
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", [(b"%s%" + str(chr(c)).encode(), None, None)])
# test non-existent specifiers
for c in b"aceghijklnopqrtuvwxyABCDEFGIJKLNOPQRTUVWXZ":
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", [(b"%" + str(chr(c)).encode(), None, None)])
# test multiple specifiers. % and z specifiers are allowed multiple times.
MultiLocaleDateTimeModelElement("s0", [(b"%%%z%z", None, None)])
for c in allowed_format_specifiers.replace(b"%", b"").replace(b"z", b""):
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", [(
b"%" + str(chr(c)).encode() + b"%" + str(chr(c)).encode(), None, None)])
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", [])
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", [(b"%s%z%f", None)])
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement, "s0", [(b"", None, None)])
def test25max_time_jump_seconds_input_validation(self):
"""Check if max_time_jump_seconds is validated."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", timezone.utc, None)], None)
self.assertEqual(multi_locale_dtme.max_time_jump_seconds, 86400)
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
100000)
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None, 1)
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement,
self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
-1)
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement,
self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
0)
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
"1000")
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
True)
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
1.25)
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
{"key": 2020})
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], None,
[1000])
def test5get_match_element_with_unclean_format_string(self):
"""This test case checks if unclean format_strings can be used."""
# example "Date: 09.03.2021, Time: 10:02"
match_context = DummyMatchContext(b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"Date %%d: %d.%m.%Y %H:%M:%S%z", None, None)])
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_string, b"Date %d: 07.02.2018 11:40:00 UTC+0000")
def test15max_time_jump_seconds_in_time(self):
"""
Test if the max_time_jump_seconds parameter works if the next date is in time.
Warnings with unqualified timestamp year wraparound.
"""
log_stream = StringIO()
logging.basicConfig(stream=log_stream, level=logging.INFO)
max_time_jump_seconds = 86400
start_year = 2020
match_context = DummyMatchContext(b"31.12 23:59:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year,
max_time_jump_seconds=max_time_jump_seconds)
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609459140)
self.assertEqual(match_context.match_string, b"31.12 23:59:00")
self.assertEqual(multi_locale_dtme.start_year, 2020)
match_context = DummyMatchContext(b"01.01 23:59:00: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609545540)
self.assertEqual(match_context.match_string, b"01.01 23:59:00")
self.assertEqual(multi_locale_dtme.start_year, 2021)
self.assertIn("WARNING:DEBUG:DateTimeModelElement unqualified timestamp year wraparound detected from 2021-01-01T23:59:00+00:00 to "
"2021-01-01T23:59:00+00:00", log_stream.getvalue())
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
initialize_loggers(self.aminer_config, getpwnam("aminer").pw_uid, getgrnam("aminer").gr_gid)
def test23text_locale_input_validation(self):
"""
Check if text_locale is validated and only valid values can be entered.
An exception has to be raised if the locale is not installed on the system.
"""
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", timezone.utc, "en_US.UTF-8")])
MultiLocaleDateTimeModelElement(self.id_,
[(b"%d.%m %H:%M:%S", timezone.utc,
("en_US", "UTF-8"))])
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement,
self.id_, [(b"%d.%m.%Y %H:%M:%S", None, "")])
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement,
self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, tuple("en_US.UTF-8"))])
self.assertRaises(ValueError, MultiLocaleDateTimeModelElement,
self.id_, [(b"%d.%m.%Y %H:%M:%S", None,
("en_US", "UTF-8", "t"))])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, b"")])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, 1)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, 1.2)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, True)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, ["en_US", "UTF-8"])])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", None, {
"en_US": "UTF-8"
})])
def test7get_match_element_with_different_text_locales(self):
"""Test if data with different text locales can be handled with different text_locale parameters."""
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", timezone.utc, "en_US.UTF-8")])
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", timezone.utc, "de_AT.UTF-8")])
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", timezone.utc, "de_AT.ISO-8859-1")])
def test12get_match_element_without_leap_start_year(self):
"""Check if normal start_years can not parse the 29th February."""
data = b"29.02 11:40:00: it still works"
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2019)
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test17time_change_cest_cet(self):
"""Check if the time change from CET to CEST and vice versa work as expected."""
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S%z", None, None)])
match_context = DummyMatchContext(b"24.03.2018 11:40:00 CET: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1521888000)
self.assertEqual(match_context.match_string, b"24.03.2018 11:40:00 CET")
match_context = DummyMatchContext(b"25.03.2018 11:40:00 CEST: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1521970800)
self.assertEqual(match_context.match_string, b"25.03.2018 11:40:00 CEST")
match_context = DummyMatchContext(b"27.10.2018 11:40:00 CEST: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540633200)
self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 CEST")
match_context = DummyMatchContext(b"28.10.2018 11:40:00 CET: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540723200)
self.assertEqual(match_context.match_string, b"28.10.2018 11:40:00 CET")
match_context = DummyMatchContext(b"27.10.2018 11:40:00 EST: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540658400)
self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 EST")
match_context = DummyMatchContext(b"27.10.2018 11:40:00 PDT: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540665600)
self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 PDT")
match_context = DummyMatchContext(b"27.10.2018 11:40:00 GMT: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540640400)
self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 GMT")
def test6get_match_element_with_different_time_zones(self):
"""Test if different time_zones work with the MultiLocaleDateTimeModelElement."""
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S%z", None, None)])
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-1200: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518046800)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-1200")
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-12: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518046800)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-12")
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-5: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518021600)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-5")
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-0500: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518021600)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-0500")
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC+0000: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+0000")
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC+0100: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518000000)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+0100")
match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC+1400: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1517953200)
self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+1400")
def test24start_year_input_validation(self):
"""Check if start_year is validated."""
multi_locale_dtme = MultiLocaleDateTimeModelElement("s0", [(b"%d.%m %H:%M:%S", timezone.utc, None)], None)
self.assertEqual(multi_locale_dtme.start_year, datetime.now().year)
MultiLocaleDateTimeModelElement("s0", [(b"%d.%m %H:%M:%S", timezone.utc, None)], 2020)
MultiLocaleDateTimeModelElement("s0", [(b"%d.%m %H:%M:%S", timezone.utc, None)], -630)
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, "s0", [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], "2020")
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, "s0", [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], True)
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, "s0", [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], 1.25)
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, "s0", [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], [2020])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, "s0", [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)], {"key": 2020})
def test7get_match_element_with_different_text_locales(self):
"""Test if data with different text locales can be handled with different text_locale parameters."""
installed = False
try:
self.assertEqual("de_AT.UTF-8", locale.setlocale(locale.LC_TIME, "de_AT.UTF-8"))
installed = True
except locale.Error:
pass
MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", timezone.utc, "en_US.UTF-8")])
if installed:
MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", timezone.utc, "de_AT.UTF-8")])
def test5get_match_element_with_unclean_format_string(self):
"""This test case checks if unclean format_strings can be used."""
data = b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works"
date = b"Date %d: 07.02.2018 11:40:00 UTC+0000"
match_context = DummyMatchContext(data)
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"Date %%d: %d.%m.%Y %H:%M:%S%z", None, None)])
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518003600, None)
def test19date_before_unix_timestamps(self):
"""Check if timestamps before the unix timestamp are processed properly."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", None, None)])
data = b"01.01.1900 11:40:00: it still works"
date = b"01.01.1900 11:40:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
-2208946800, None)
def test11get_match_element_with_leap_start_year(self):
"""Check if leap start_years can parse the 29th February."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2020)
data = b"29.02 11:40:00: it still works"
date = b"29.02 11:40:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1582976400, None)
def test14learn_new_start_year_without_start_year_set(self):
"""Test if a new year is learned successfully with the start year being None."""
match_context = DummyMatchContext(b"31.12 23:59:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)])
self.assertIsNotNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_string, b"31.12 23:59:00")
start_year = multi_locale_dtme.start_year
match_context = DummyMatchContext(b"01.01 11:20:00: it still works")
self.assertIsNotNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_string, b"01.01 11:20:00")
self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test13learn_new_start_year_with_start_year_set(self):
"""Test if a new year is learned successfully with the start year being set."""
start_year = 2020
match_context = DummyMatchContext(b"31.12 23:59:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year)
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609459140)
self.assertEqual(match_context.match_string, b"31.12 23:59:00")
self.assertEqual(multi_locale_dtme.start_year, start_year)
match_context = DummyMatchContext(b"01.01 11:20:00: it still works")
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609500000)
self.assertEqual(match_context.match_string, b"01.01 11:20:00")
self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test18same_timestamp_multiple_times(self):
"""Test if the MultiLocaleDateTimeModelElement can handle multiple same timestamps."""
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S", None, None)])
match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
def test10get_match_element_without_start_year_defined(self):
"""Test if dates without year can still be parsed, even without defining the start_year."""
data = b"07.02 11:40:00: it still works"
date = b"07.02 11:40:00"
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)])
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
dtm = datetime(datetime.now().year, 2, 7, 11, 40, tzinfo=timezone.utc)
total_seconds = (
dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
total_seconds, None)
def test18same_timestamp_multiple_times(self):
"""Test if the MultiLocaleDateTimeModelElement can handle multiple same timestamps."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", None, None)])
data = b"07.02.2019 11:40:00: it still works"
date = b"07.02.2019 11:40:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1549539600, None)
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1549539600, None)
def test22time_zone_input_validation(self):
"""Check if time_zone is validated and only valid values can be entered."""
en_gb_utf8 = "en_GB.utf8"
en_us_utf8 = "en_US.utf8"
de_at_utf8 = "de_AT.utf8"
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S.%f", None, None),
(b"%d.%m.%Y %H:%M:%S%z", None, None),
(b"%d.%m.%Y %H:%M:%S", None, None),
(b"%d.%m.%YT%H:%M:%S", None, None),
(b"%d.%m.%Y", None, None),
(b"%H:%M:%S:%f", None, de_at_utf8),
(b"%H:%M:%S", None, None),
(b"%d %b %Y", None, en_gb_utf8),
(b"%dth %b %Y", None, en_gb_utf8),
(b"%d/%m/%Y", None, en_gb_utf8),
(b"%m-%d-%Y", None, en_us_utf8),
(b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)])
for dtme in multi_locale_dtme.date_time_model_elements:
self.assertEqual(dtme.time_zone, timezone.utc)
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", timezone.utc, None)])
for tz in pytz.all_timezones:
MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", pytz.timezone(tz), None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", b"", None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", "UTC", None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", 1, None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", 1.25, None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", True, None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", [timezone.utc], None)])
self.assertRaises(TypeError, MultiLocaleDateTimeModelElement, self.id_,
[(b"%d.%m.%Y %H:%M:%S", {
"time_zone": timezone.utc
}, None)])
def test4wrong_date(self):
"""Test if wrong input data does not return a match."""
tz_gmt10 = pytz.timezone("Etc/GMT+10")
en_gb_utf8 = "en_GB.utf8"
en_us_utf8 = "en_US.utf8"
de_at_utf8 = "de_AT.utf8"
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S.%f", None, None),
(b"%d.%m.%Y %H:%M:%S%z", None, None),
(b"%d.%m.%Y %H:%M:%S", None, None),
(b"%d.%m.%YT%H:%M:%S", None, None),
(b"%d.%m.%Y", None, None),
(b"%H:%M:%S:%f", None, de_at_utf8),
(b"%H:%M:%S", None, None),
(b"%b %d", tz_gmt10, de_at_utf8),
(b"%d %b %Y", None, en_gb_utf8),
(b"%dth %b %Y", None, en_gb_utf8),
(b"%d/%m/%Y", None, en_gb_utf8),
(b"%m-%d-%Y", None, en_us_utf8),
(b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)])
# wrong day
data = b"32.03.2019 11:40:00: it still works"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# wrong month
data = b"01.13.2019 11:40:00: it still works"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# wrong year
data = b"01.01.00 11:40:00: it still works"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# wrong date leap year
data = b"29.02.2019 11:40:00: it still works"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# British date
data = b"13 Dezember 2019"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test16max_time_jump_seconds_exceeded(self):
"""
Test if the start_year is not updated, when the next date exceeds the max_time_jump_seconds.
A time inconsistency warning must occur.
"""
log_stream = StringIO()
logging.basicConfig(stream=log_stream, level=logging.INFO)
max_time_jump_seconds = 86400
start_year = 2020
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)],
start_year=start_year,
max_time_jump_seconds=max_time_jump_seconds)
data = b"31.12 23:59:00: it still works"
date = b"31.12 23:59:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1609459140, None)
self.assertEqual(multi_locale_dtme.start_year, start_year)
data = b"01.01 23:59:01: it still works"
date = b"01.01 23:59:01"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1577923141, None)
self.assertEqual(multi_locale_dtme.start_year, start_year)
self.assertIn(
"WARNING:DEBUG:DateTimeModelElement time inconsistencies parsing b'01.01 23:59:01', expecting value around "
"1609459140. Check your settings!", log_stream.getvalue())
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
initialize_loggers(self.aminer_config,
getpwnam("aminer").pw_uid,
getgrnam("aminer").gr_gid)
def test26get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S", None, None)])
data = b"07.02.2019 11:40:00: it still works"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test14learn_new_start_year_without_start_year_set(self):
"""Test if a new year is learned successfully with the start year being None."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)])
data = b"31.12 23:59:00: it still works"
date = b"31.12 23:59:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
dtm = datetime(datetime.now().year,
12,
31,
23,
59,
tzinfo=timezone.utc)
total_seconds = (
dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
total_seconds, None)
start_year = multi_locale_dtme.start_year
data = b"01.01 11:20:00: it still works"
date = b"01.01 11:20:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
dtm = datetime(datetime.now().year + 1,
1,
1,
11,
20,
tzinfo=timezone.utc)
total_seconds = (
dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
total_seconds, None)
self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test13learn_new_start_year_with_start_year_set(self):
"""Test if a new year is learned successfully with the start year being set."""
start_year = 2020
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year)
data = b"31.12 23:59:00: it still works"
date = b"31.12 23:59:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1609459140, None)
self.assertEqual(multi_locale_dtme.start_year, start_year)
data = b"01.01 11:20:00: it still works"
date = b"01.01 11:20:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1609500000, None)
self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test9get_match_element_with_start_year(self):
"""Test if dates without year can be parsed, when the start_year is defined."""
match_context = DummyMatchContext(b"07.02 11:40:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2017)
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1486467600)
self.assertEqual(match_context.match_string, b"07.02 11:40:00")
match_context = DummyMatchContext(b"07.02 11:40:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2019)
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1549539600)
self.assertEqual(match_context.match_string, b"07.02 11:40:00")
def test4wrong_date(self):
"""Test if wrong input data does not return a match."""
tz_gmt10 = pytz.timezone("Etc/GMT+10")
en_gb_utf8 = "en_GB.utf8"
en_us_utf8 = "en_US.utf8"
de_at_utf8 = "de_AT.utf8"
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [
(b"%d.%m.%Y %H:%M:%S.%f", None, None), (b"%d.%m.%Y %H:%M:%S%z", None, None), (b"%d.%m.%Y %H:%M:%S", None, None),
(b"%d.%m.%YT%H:%M:%S", None, None), (b"%d.%m.%Y", None, None), (b"%H:%M:%S:%f", None, de_at_utf8),
(b"%H:%M:%S", None, None), (b"%b %d", tz_gmt10, de_at_utf8), (b"%d %b %Y", None, en_gb_utf8),
(b"%dth %b %Y", None, en_gb_utf8), (b"%d/%m/%Y", None, en_gb_utf8), (b"%m-%d-%Y", None, en_us_utf8),
(b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)])
# wrong day
match_context = DummyMatchContext(b"32.03.2019 11:40:00: it still works")
self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_data, b"32.03.2019 11:40:00: it still works")
self.assertEqual(match_context.match_string, b"")
# wrong month
match_context = DummyMatchContext(b"01.13.2019 11:40:00: it still works")
self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_data, b"01.13.2019 11:40:00: it still works")
self.assertEqual(match_context.match_string, b"")
# wrong year
match_context = DummyMatchContext(b"01.01.00 11:40:00: it still works")
self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_data, b"01.01.00 11:40:00: it still works")
self.assertEqual(match_context.match_string, b"")
# wrong date leap year
match_context = DummyMatchContext(b"29.02.2019 11:40:00: it still works")
self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_data, b"29.02.2019 11:40:00: it still works")
self.assertEqual(match_context.match_string, b"")
# British date
string = b"13 Dezember 2019"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
self.assertEqual(match_element, None)
def test9get_match_element_with_start_year(self):
"""Test if dates without year can be parsed, when the start_year is defined."""
data = b"07.02 11:40:00: it still works"
date = b"07.02 11:40:00"
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2017)
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1486467600, None)
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2019)
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1549539600, None)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement('Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%'),
DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')]
service_children_login_details = [
FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']),
OptionalMatchModelElement('PastTime', SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')]))]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')]
service_children_random_time = [FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random')]
service_children_sensors = [SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')]), FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)]
service_children_user_ip_address = [
FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')]
service_children_audit = [SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO),
FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')])]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'),
SequenceModelElement('se', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';')])]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement("wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement("decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement('ElementValueBranchModelElement', FirstMatchModelElement("first", [
SequenceModelElement("seq1", [fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])]), "wordlist",
{0: decimal_integer_value_model_element, 1: fixed_data_me2}))
service_children_parsing_model_element.append(HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(SequenceModelElement('se2', [
FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement')]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1))
service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(SequenceModelElement('se', [
WhiteSpaceLimitedDataModelElement('WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ')]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement('/', FirstMatchModelElement('FirstMatchModelElement//optional', [
FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [
FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])])))
alphabet = b'ghijkl'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement),
SequenceModelElement('CronExecution', service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element)])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust([atom_filter])
analysis_context.register_component(simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
from aminer.events.JsonConverterHandler import JsonConverterHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
json_converter_handler = JsonConverterHandler([stream_printer_event_handler], analysis_context)
anomaly_event_handlers = [json_converter_handler]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True)
analysis_context.register_component(verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.AndMatchRule([
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails'),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10)
analysis_context.register_component(parser_count, component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers)
analysis_context.register_component(etd, component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False,
output_log_line=False, ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5,
ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True,
hypothesis_max_delta_time=1.0)
analysis_context.register_component(ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector
efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1)
analysis_context.register_component(efd, component_name="EventFrequencyDetector")
atom_filter.add_handler(efd)
from aminer.analysis.EventSequenceDetector import EventSequenceDetector
esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random',
'/model/RandomTime', '/model/DailyCron'])
analysis_context.register_component(esd, component_name="EventSequenceDetector")
atom_filter.add_handler(esd)
from aminer.analysis.MatchFilter import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[
1, 10, 100], output_log_line=True)
analysis_context.register_component(match_filter, component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(analysis_context.aminer_config, [
'/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function, output_log_line=True)
analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
import re
ip_match_action = Rules.EventGenerationMatchAction(
"Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers)
vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3])
mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None)
time_allowlist_rules = [
Rules.AndMatchRule([
Rules.ParallelMatchRule([
Rules.ValueDependentDelegatedMatchRule([
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], {
(b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt),
Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action),
Rules.DebugHistoryMatchRule(debug_match_result=True)
]),
# IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed
Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])),
Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)),
Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$")))
])
]
time_allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist")
atom_filter.add_handler(time_allowlist_violation_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(analysis_context.aminer_config, [
('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(analysis_context.aminer_config, anomaly_event_handlers, None, [
'/model/Random'], 100, 10, output_log_line=True)
analysis_context.register_component(match_value_average_change_detector, component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(
sys.stdout, ['/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM'], b';', b'')
analysis_context.register_component(match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(analysis_context.aminer_config, [
'/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=[
'/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(analysis_context.aminer_config, [
'/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True)
analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2,
realert_interval=5, output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5,
record_count_before_event=10000, output_log_line=True)
analysis_context.register_component(time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule('CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[
('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement])
rules = [Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)]
time_correlation_violation_detector = TimeCorrelationViolationDetector(analysis_context.aminer_config, rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
