def test25max_time_jump_seconds_input_validation(self):
"""Check if max_time_jump_seconds is validated."""
dtme = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
None, None)
self.assertEqual(dtme.max_time_jump_seconds, 86400)
DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc,
None, None, 100000)
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, -1)
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, 0)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None,
"100000")
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, True)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, 1.25)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None,
[2020])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, [])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None,
{"key": 2020})
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, ())
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None,
set())
def test23text_locale_input_validation(self):
"""
Check if text_locale is validated and only valid values can be entered.
An exception has to be raised if the locale is not installed on the system.
"""
DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
"en_US.UTF-8")
DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
("en_US", "UTF-8"))
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, 1)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, 1.2)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, True)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, ["en_US", "UTF-8"])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, {"en_US": "UTF-8"})
self.assertRaises(ValueError, DateTimeModelElement,
self.id_, b"%d.%m %H:%M:%S", timezone.utc,
tuple("en_US.UTF-8"))
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, set())
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc, ())
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"%d.%m %H:%M:%S", timezone.utc,
("en_US", "UTF-8", "de_AT", "UTF-8"))
def test16max_time_jump_seconds_exceeded(self):
"""
Test if the start_year is not updated, when the next date exceeds the max_time_jump_seconds.
A time inconsistency warning must occur.
"""
log_stream = StringIO()
logging.basicConfig(stream=log_stream, level=logging.INFO)
max_time_jump_seconds = 86400
start_year = 2020
match_context = DummyMatchContext(b"31.12 23:59:00: it still works")
date_time_model_element = DateTimeModelElement(
"path",
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=start_year,
max_time_jump_seconds=max_time_jump_seconds)
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1609459140)
self.assertEqual(match_context.match_string, b"31.12 23:59:00")
self.assertEqual(date_time_model_element.start_year, 2020)
match_context = DummyMatchContext(b"01.01 23:59:01: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1577923141)
self.assertEqual(match_context.match_string, b"01.01 23:59:01")
self.assertEqual(date_time_model_element.start_year, 2020)
self.assertIn(
"WARNING:DEBUG:DateTimeModelElement time inconsistencies parsing b'01.01 23:59:01', expecting value around "
"1609459140. Check your settings!", log_stream.getvalue())
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
initialize_loggers(self.aminer_config,
getpwnam("aminer").pw_uid,
getgrnam("aminer").gr_gid)
def test24start_year_input_validation(self):
"""Check if start_year is validated."""
dtme = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
None, None)
self.assertEqual(dtme.start_year, datetime.now().year)
DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc,
None, 2020)
DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc,
None, -630)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, "2020")
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, True)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, 1.25)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, [2020])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, [])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None,
{"key": 2020})
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, set())
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", timezone.utc, None, ())
def test22time_zone_input_validation(self):
"""Check if time_zone is validated and only valid values can be entered."""
dtme = DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S")
self.assertEqual(dtme.time_zone, timezone.utc)
DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc)
for tz in pytz.all_timezones:
DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S",
pytz.timezone(tz))
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", b"UTC")
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", "UTC")
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", 1)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", 1.25)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", True)
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", {"time_zone": timezone.utc})
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", [timezone.utc])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", [])
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", set())
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
b"%d.%m.%Y %H:%M:%S", ())
def test21date_format_input_validation(self):
"""Check if date_format is validated and only valid values can be entered."""
allowed_format_specifiers = b"bdfHMmSsYz%"
# check if allowed values do not raise any exception.
format_specifiers = b""
for c in allowed_format_specifiers:
format_specifiers += b"%" + str(chr(c)).encode()
DateTimeModelElement(self.id_, b"%" + str(chr(c)).encode())
# check if all allowed values can not be used together. An exception should be raised, because of multiple month representations
# and %s with non-second formats.
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
format_specifiers)
DateTimeModelElement(
self.id_,
format_specifiers.replace(b"%m", b"").replace(b"%s", b""))
DateTimeModelElement(
self.id_,
format_specifiers.replace(b"%b", b"").replace(b"%s", b""))
DateTimeModelElement(self.id_, b"%s%z%f")
for c in allowed_format_specifiers.replace(b"s", b"").replace(
b"z", b"").replace(b"f", b"").replace(b"%", b""):
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"%s%" + str(chr(c)).encode())
# test non-existent specifiers
for c in b"aceghijklnopqrtuvwxyABCDEFGIJKLNOPQRTUVWXZ":
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"%" + str(chr(c)).encode())
# test multiple specifiers. % and z specifiers are allowed multiple times.
DateTimeModelElement(self.id_, b"%%%z%z")
for c in allowed_format_specifiers.replace(b"%",
b"").replace(b"z", b""):
self.assertRaises(
ValueError, DateTimeModelElement, self.id_,
b"%" + str(chr(c)).encode() + b"%" + str(chr(c)).encode())
self.assertRaises(ValueError, DateTimeModelElement, self.id_,
b"") # empty date_format
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
None) # None date_format
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
"") # string date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
123) # integer date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
123.22) # float date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
True) # boolean date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
{"id": "path"}) # dict date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
["path"]) # list date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
[]) # empty list date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
()) # empty tuple date_format is not allowed
self.assertRaises(TypeError, DateTimeModelElement, self.id_,
set()) # empty set date_format is not allowed
def test7get_match_element_with_different_text_locales(self):
"""Test if data with different text locales can be handled with different text_locale parameters."""
DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
"en_US.UTF-8")
DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
"de_AT.UTF-8")
DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc,
"de_AT.ISO-8859-1")
def test12get_match_element_without_leap_start_year(self):
"""Check if normal start_years can not parse the 29th February."""
match_context = DummyMatchContext(b"29.02 11:40:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=2019)
self.assertIsNone(
date_time_model_element.get_match_element("match1", match_context))
def test10get_match_element_without_start_year_defined(self):
"""Test if dates without year can still be parsed, even without defining the start_year."""
match_context = DummyMatchContext(b"07.02 11:40:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m %H:%M:%S",
timezone.utc)
self.assertIsNotNone(
date_time_model_element.get_match_element("match1", match_context))
self.assertEqual(match_context.match_string, b"07.02 11:40:00")
def test2start_year_value(self):
"""This test checks if they class is parsing dates without year values correctly."""
match_context = MatchContext(b'07.02 11:40:00: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m %H:%M:%S', None,
None, 2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1486467600)
def test6get_match_element_with_different_time_zones(self):
"""Test if different time_zones work with the DateTimeModelElement."""
date_time_model_element = DateTimeModelElement("path",
b"%d.%m.%Y %H:%M:%S%z",
timezone.utc)
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC-1200: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518046800)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC-1200")
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC-12: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518046800)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC-12")
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC-5: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518021600)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC-5")
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC-0500: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518021600)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC-0500")
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC+0000: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC+0000")
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC+0100: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518000000)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC+0100")
match_context = DummyMatchContext(
b"07.02.2018 11:40:00 UTC+1400: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1517953200)
self.assertEqual(match_context.match_string,
b"07.02.2018 11:40:00 UTC+1400")
def test12get_match_element_without_leap_start_year(self):
"""Check if normal start_years can not parse the 29th February."""
data = b"29.02 11:40:00: it still works"
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=2019)
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test5get_match_element_with_unclean_format_string(self):
"""This test case checks if unclean format_strings can be used."""
data = b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works"
date = b"Date %d: 07.02.2018 11:40:00 UTC+0000"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(
self.id_, b"Date %%d: %d.%m.%Y %H:%M:%S%z", timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518003600, None)
def test11get_match_element_with_leap_start_year(self):
"""Check if leap start_years can parse the 29th February."""
match_context = DummyMatchContext(b"29.02 11:40:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=2020)
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1582976400)
self.assertEqual(match_context.match_string, b"29.02 11:40:00")
def test5get_match_element_with_unclean_format_string(self):
"""This test case checks if unclean format_strings can be used."""
match_context = DummyMatchContext(
b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works")
date_time_model_element = DateTimeModelElement(
"path", b"Date %%d: %d.%m.%Y %H:%M:%S%z", timezone.utc)
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_string,
b"Date %d: 07.02.2018 11:40:00 UTC+0000")
def test19date_before_unix_timestamps(self):
"""Check if timestamps before the unix timestamp are processed properly."""
match_context = DummyMatchContext(
b"01.01.1900 11:40:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m.%Y %H:%M:%S",
timezone.utc)
match_element = date_time_model_element.get_match_element(
"match1", match_context)
self.assertEqual(match_element.match_string, b"01.01.1900 11:40:00")
self.assertEqual(match_element.match_object, -2208946800)
def test9value_dependent_modulo_time_match_rule(self):
"""This case unit the ValueDependentModuloTimeMatchRule. Limit look up not working with tuples."""
description = "Test9Rules"
value_dependent_modulo_time_match_rule = ValueDependentModuloTimeMatchRule(self.model_syslog_time, 86400, [self.model_syslog_time],
{1550145600: [43200, 86400]})
self.analysis_context.register_component(value_dependent_modulo_time_match_rule, description)
date_time_model_element = DateTimeModelElement('time', b'%d.%m.%Y %H:%M:%S')
match_context = MatchContext(b'14.02.2019 12:00:00')
match_element = date_time_model_element.get_match_element(self.model_syslog, match_context)
log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), 1550138400, date_time_model_element)
self.assertTrue(value_dependent_modulo_time_match_rule.match(log_atom))
def test1date_formats_exceptions(self):
"""This test case verifies, if all date_format qualifiers are valid and exceptions are raised, if they are invalid."""
match_context = MatchContext(b'07.02.2019 11:40:00: it still works')
date_time_model_element = DateTimeModelElement('path',
b'%d.%m.%Y %H:%M:%S')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_string(),
b'07.02.2019 11:40:00')
self.assertRaises(Exception, DateTimeModelElement, 'path', b'%h %b')
self.assertRaises(Exception, DateTimeModelElement, 'path', b'%H%H')
self.assertRaises(Exception, DateTimeModelElement, 'path', b'%H%s')
def test11get_match_element_with_leap_start_year(self):
"""Check if leap start_years can parse the 29th February."""
data = b"29.02 11:40:00: it still works"
date = b"29.02 11:40:00"
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=2020)
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1582976400, None)
def test19date_before_unix_timestamps(self):
"""Check if timestamps before the unix timestamp are processed properly."""
data = b"01.01.1900 11:40:00: it still works"
date = b"01.01.1900 11:40:00"
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m.%Y %H:%M:%S",
timezone.utc)
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, -2208946800,
None)
def test7get_match_element_with_different_text_locales(self):
"""Test if data with different text locales can be handled with different text_locale parameters."""
installed = False
try:
self.assertEqual("de_AT.UTF-8",
locale.setlocale(locale.LC_TIME, "de_AT.UTF-8"))
installed = True
except locale.Error:
pass
DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc,
"en_US.UTF-8")
if installed:
DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc,
"de_AT.UTF-8")
def test10get_match_element_without_start_year_defined(self):
"""Test if dates without year can still be parsed, even without defining the start_year."""
data = b"07.02 11:40:00: it still works"
date = b"07.02 11:40:00"
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m %H:%M:%S",
timezone.utc)
match_context = DummyMatchContext(data)
dtm = datetime(datetime.now().year, 2, 7, 11, 40, tzinfo=timezone.utc)
total_seconds = (
dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, total_seconds,
None)
def get_model():
"""Return a model to parse Suricata Fast logs from the AIT-LDS."""
model = SequenceModelElement('model', [
DateTimeModelElement('time', b'%m/%d/%Y-%H:%M:%S.%f'),
FixedDataModelElement('brack_str1', b' [**] ['),
DecimalIntegerValueModelElement('id1'),
FixedDataModelElement('sep1', b':'),
DecimalIntegerValueModelElement('id2'),
FixedDataModelElement('sep2', b':'),
DecimalIntegerValueModelElement('id3'),
FixedDataModelElement('sep3', b'] '),
DelimitedDataModelElement('message', b' [**] '),
FixedDataModelElement('classification_str',
b' [**] [Classification: '),
DelimitedDataModelElement('classification', b']'),
FixedDataModelElement('priority_str', b'] [Priority: '),
DecimalIntegerValueModelElement('priority'),
FixedDataModelElement('brack_str1', b'] {'),
DelimitedDataModelElement('conn', b'}'),
FixedDataModelElement('brack_str2', b'} '),
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('arrow_str', b' -> '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('dst_port'),
])
return model
def get_model():
"""Return a model to parse Suricata Fast logs from the AIT-LDS."""
model = SequenceModelElement("model", [
DateTimeModelElement("time", b"%m/%d/%Y-%H:%M:%S.%f"),
FixedDataModelElement("brack_str1", b" [**] ["),
DecimalIntegerValueModelElement("id1"),
FixedDataModelElement("sep1", b":"),
DecimalIntegerValueModelElement("id2"),
FixedDataModelElement("sep2", b":"),
DecimalIntegerValueModelElement("id3"),
FixedDataModelElement("sep3", b"] "),
DelimitedDataModelElement("message", b" [**] "),
FixedDataModelElement("classification_str",
b" [**] [Classification: "),
DelimitedDataModelElement("classification", b"]"),
FixedDataModelElement("priority_str", b"] [Priority: "),
DecimalIntegerValueModelElement("priority"),
FixedDataModelElement("brack_str1", b"] {"),
DelimitedDataModelElement("conn", b"}"),
FixedDataModelElement("brack_str2", b"} "),
IpAddressDataModelElement("src_ip"),
FixedDataModelElement("colon", b":"),
DecimalIntegerValueModelElement("src_port"),
FixedDataModelElement("arrow_str", b" -> "),
IpAddressDataModelElement("dst_ip"),
FixedDataModelElement("colon", b":"),
DecimalIntegerValueModelElement("dst_port"),
])
return model
def test14learn_new_start_year_without_start_year_set(self):
"""Test if a new year is learned successfully with the start year being None."""
match_context = DummyMatchContext(b"31.12 23:59:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m %H:%M:%S",
timezone.utc)
self.assertIsNotNone(
date_time_model_element.get_match_element("match1", match_context))
self.assertEqual(match_context.match_string, b"31.12 23:59:00")
start_year = date_time_model_element.start_year
match_context = DummyMatchContext(b"01.01 11:20:00: it still works")
self.assertIsNotNone(
date_time_model_element.get_match_element("match1", match_context))
self.assertEqual(match_context.match_string, b"01.01 11:20:00")
self.assertEqual(date_time_model_element.start_year, start_year + 1)
def test5_unclean_format_string(self):
"""This test case checks if unclean format_strings can be used."""
match_context = MatchContext(
b'Test 07.02.2018 11:40:00 UTC+0000: it still works')
date_time_model_element = DateTimeModelElement(
'path', b'Test %d.%m.%Y %H:%M:%S %z', datetime.timezone.utc, None,
2017)
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518003600)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
match_context = MatchContext(
b'Test 07.02.2018 11:40:00 UTC-0001: it still works')
self.assertEqual(
date_time_model_element.get_match_element(
'match1', match_context).get_match_object(), 1518000000)
self.assertEqual(match_context.match_data,
self.__expected_match_context)
def get_model():
"""Return a parser for apache2 access.log."""
new_time_model = SequenceModelElement('time_model', [
DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S '),
FixedWordlistDataModelElement('sign', [b'+', b'-']),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('bracket', b']')
])
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identity_model = VariableByteDataModelElement(
'ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
request_method_model = FixedWordlistDataModelElement(
'method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT',
b'OPTIONS', b'TRACE', b'PATCH'
])
request_model = VariableByteDataModelElement(
'request',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+'
)
version_model = VariableByteDataModelElement('version', b'0123456789.')
status_code_model = DecimalIntegerValueModelElement('status')
size_model = DecimalIntegerValueModelElement('size')
user_agent_model = VariableByteDataModelElement(
'useragent',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+;:_ '
)
whitespace_str = b' '
model = SequenceModelElement('accesslog', [
host_name_model,
FixedDataModelElement('sp0', whitespace_str),
identity_model,
FixedDataModelElement('sp1', whitespace_str),
user_name_model,
FixedDataModelElement('sp2', whitespace_str),
new_time_model,
FixedDataModelElement('sp3', b' "'),
request_method_model,
FixedDataModelElement('sp4', whitespace_str),
request_model,
FixedDataModelElement('sp5', b' HTTP/'),
version_model,
FixedDataModelElement('sp6', b'" '),
status_code_model,
FixedDataModelElement('sp7', whitespace_str),
size_model,
FixedDataModelElement('sp8', b' "-" "'),
user_agent_model,
FixedDataModelElement('sp9', b'"'),
])
return model
def test18same_timestamp_multiple_times(self):
"""Test if the DateTimeModelElement can handle multiple same timestamps."""
match_context = DummyMatchContext(
b"07.02.2019 11:40:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m.%Y %H:%M:%S",
timezone.utc)
match_element = date_time_model_element.get_match_element(
"match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
match_context = DummyMatchContext(
b"07.02.2019 11:40:00: it still works")
match_element = date_time_model_element.get_match_element(
"match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
def get_model():
"""Return a model to parse Apache Error logs from the AIT-LDS."""
model = SequenceModelElement('model', [
FixedDataModelElement('sp1', b'['),
FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']),
FixedDataModelElement('sp2', b' '),
DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'),
FixedDataModelElement('error_str', b'] [:error] [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('sp3', b'] [client '),
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('client_port'),
FixedDataModelElement('php', b'] PHP '),
FirstMatchModelElement('fphp', [
SequenceModelElement('warning', [
FixedDataModelElement('warning_str', b'Warning: '),
FirstMatchModelElement('warning', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function', b')'),
FixedDataModelElement('compatible_str', b') should be compatible with '),
DelimitedDataModelElement('function2', b')'),
FixedDataModelElement('compatible_str', b') in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer')]),
SequenceModelElement('system', [
FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')])])]),
SequenceModelElement('notice', [
FixedDataModelElement('notice_str', b'Notice: Undefined index: '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('sp', b' in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')]),
SequenceModelElement('deprecated', [
FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class '
b'will not be constructors in a future version of PHP; '),
DelimitedDataModelElement('class', b' '),
FixedDataModelElement('constructor_str', b' has a deprecated constructor in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer'),
])])])
return model
def test13learn_new_start_year_with_start_year_set(self):
"""Test if a new year is learned successfully with the start year being set."""
start_year = 2020
match_context = DummyMatchContext(b"31.12 23:59:00: it still works")
date_time_model_element = DateTimeModelElement("path",
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=start_year)
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1609459140)
self.assertEqual(match_context.match_string, b"31.12 23:59:00")
self.assertEqual(date_time_model_element.start_year, start_year)
match_context = DummyMatchContext(b"01.01 11:20:00: it still works")
self.assertEqual(
date_time_model_element.get_match_element(
"match1", match_context).get_match_object(), 1609500000)
self.assertEqual(match_context.match_string, b"01.01 11:20:00")
self.assertEqual(date_time_model_element.start_year, start_year + 1)
