def test25max_time_jump_seconds_input_validation(self): """Check if max_time_jump_seconds is validated.""" dtme = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, None, None) self.assertEqual(dtme.max_time_jump_seconds, 86400) DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, 100000) self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, -1) self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, 0) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, "100000") self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, True) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, 1.25) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, [2020]) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, []) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, {"key": 2020}) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, ()) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, None, set())
def test23text_locale_input_validation(self): """ Check if text_locale is validated and only valid values can be entered. An exception has to be raised if the locale is not installed on the system. """ DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, "en_US.UTF-8") DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, ("en_US", "UTF-8")) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, 1) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, 1.2) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, True) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, ["en_US", "UTF-8"]) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, {"en_US": "UTF-8"}) self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, tuple("en_US.UTF-8")) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, set()) self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, ()) self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%d.%m %H:%M:%S", timezone.utc, ("en_US", "UTF-8", "de_AT", "UTF-8"))
def test16max_time_jump_seconds_exceeded(self): """ Test if the start_year is not updated, when the next date exceeds the max_time_jump_seconds. A time inconsistency warning must occur. """ log_stream = StringIO() logging.basicConfig(stream=log_stream, level=logging.INFO) max_time_jump_seconds = 86400 start_year = 2020 match_context = DummyMatchContext(b"31.12 23:59:00: it still works") date_time_model_element = DateTimeModelElement( "path", b"%d.%m %H:%M:%S", timezone.utc, start_year=start_year, max_time_jump_seconds=max_time_jump_seconds) self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1609459140) self.assertEqual(match_context.match_string, b"31.12 23:59:00") self.assertEqual(date_time_model_element.start_year, 2020) match_context = DummyMatchContext(b"01.01 23:59:01: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1577923141) self.assertEqual(match_context.match_string, b"01.01 23:59:01") self.assertEqual(date_time_model_element.start_year, 2020) self.assertIn( "WARNING:DEBUG:DateTimeModelElement time inconsistencies parsing b'01.01 23:59:01', expecting value around " "1609459140. Check your settings!", log_stream.getvalue()) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) initialize_loggers(self.aminer_config, getpwnam("aminer").pw_uid, getgrnam("aminer").gr_gid)
def test24start_year_input_validation(self): """Check if start_year is validated.""" dtme = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, None, None) self.assertEqual(dtme.start_year, datetime.now().year) DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, 2020) DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, -630) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, "2020") self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, True) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, 1.25) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, [2020]) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, []) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, {"key": 2020}) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, set()) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc, None, ())
def test22time_zone_input_validation(self): """Check if time_zone is validated and only valid values can be entered.""" dtme = DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S") self.assertEqual(dtme.time_zone, timezone.utc) DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc) for tz in pytz.all_timezones: DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", pytz.timezone(tz)) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", b"UTC") self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", "UTC") self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", 1) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", 1.25) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", True) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", {"time_zone": timezone.utc}) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", [timezone.utc]) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", []) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", set()) self.assertRaises(TypeError, DateTimeModelElement, self.id_, b"%d.%m.%Y %H:%M:%S", ())
def test21date_format_input_validation(self): """Check if date_format is validated and only valid values can be entered.""" allowed_format_specifiers = b"bdfHMmSsYz%" # check if allowed values do not raise any exception. format_specifiers = b"" for c in allowed_format_specifiers: format_specifiers += b"%" + str(chr(c)).encode() DateTimeModelElement(self.id_, b"%" + str(chr(c)).encode()) # check if all allowed values can not be used together. An exception should be raised, because of multiple month representations # and %s with non-second formats. self.assertRaises(ValueError, DateTimeModelElement, self.id_, format_specifiers) DateTimeModelElement( self.id_, format_specifiers.replace(b"%m", b"").replace(b"%s", b"")) DateTimeModelElement( self.id_, format_specifiers.replace(b"%b", b"").replace(b"%s", b"")) DateTimeModelElement(self.id_, b"%s%z%f") for c in allowed_format_specifiers.replace(b"s", b"").replace( b"z", b"").replace(b"f", b"").replace(b"%", b""): self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%s%" + str(chr(c)).encode()) # test non-existent specifiers for c in b"aceghijklnopqrtuvwxyABCDEFGIJKLNOPQRTUVWXZ": self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"%" + str(chr(c)).encode()) # test multiple specifiers. % and z specifiers are allowed multiple times. DateTimeModelElement(self.id_, b"%%%z%z") for c in allowed_format_specifiers.replace(b"%", b"").replace(b"z", b""): self.assertRaises( ValueError, DateTimeModelElement, self.id_, b"%" + str(chr(c)).encode() + b"%" + str(chr(c)).encode()) self.assertRaises(ValueError, DateTimeModelElement, self.id_, b"") # empty date_format self.assertRaises(TypeError, DateTimeModelElement, self.id_, None) # None date_format self.assertRaises(TypeError, DateTimeModelElement, self.id_, "") # string date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, 123) # integer date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, 123.22) # float date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, True) # boolean date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, {"id": "path"}) # dict date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, ["path"]) # list date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, []) # empty list date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, ()) # empty tuple date_format is not allowed self.assertRaises(TypeError, DateTimeModelElement, self.id_, set()) # empty set date_format is not allowed
def test7get_match_element_with_different_text_locales(self): """Test if data with different text locales can be handled with different text_locale parameters.""" DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, "en_US.UTF-8") DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, "de_AT.UTF-8") DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, "de_AT.ISO-8859-1")
def test12get_match_element_without_leap_start_year(self): """Check if normal start_years can not parse the 29th February.""" match_context = DummyMatchContext(b"29.02 11:40:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc, start_year=2019) self.assertIsNone( date_time_model_element.get_match_element("match1", match_context))
def test10get_match_element_without_start_year_defined(self): """Test if dates without year can still be parsed, even without defining the start_year.""" match_context = DummyMatchContext(b"07.02 11:40:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc) self.assertIsNotNone( date_time_model_element.get_match_element("match1", match_context)) self.assertEqual(match_context.match_string, b"07.02 11:40:00")
def test2start_year_value(self): """This test checks if they class is parsing dates without year values correctly.""" match_context = MatchContext(b'07.02 11:40:00: it still works') date_time_model_element = DateTimeModelElement('path', b'%d.%m %H:%M:%S', None, None, 2017) self.assertEqual( date_time_model_element.get_match_element( 'match1', match_context).get_match_object(), 1486467600)
def test6get_match_element_with_different_time_zones(self): """Test if different time_zones work with the DateTimeModelElement.""" date_time_model_element = DateTimeModelElement("path", b"%d.%m.%Y %H:%M:%S%z", timezone.utc) match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC-1200: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518046800) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-1200") match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC-12: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518046800) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-12") match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC-5: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518021600) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-5") match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC-0500: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518021600) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-0500") match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC+0000: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518003600) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+0000") match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC+0100: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518000000) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+0100") match_context = DummyMatchContext( b"07.02.2018 11:40:00 UTC+1400: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1517953200) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+1400")
def test12get_match_element_without_leap_start_year(self): """Check if normal start_years can not parse the 29th February.""" data = b"29.02 11:40:00: it still works" date_time_model_element = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, start_year=2019) match_context = DummyMatchContext(data) match_element = date_time_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def test5get_match_element_with_unclean_format_string(self): """This test case checks if unclean format_strings can be used.""" data = b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works" date = b"Date %d: 07.02.2018 11:40:00 UTC+0000" match_context = DummyMatchContext(data) date_time_model_element = DateTimeModelElement( self.id_, b"Date %%d: %d.%m.%Y %H:%M:%S%z", timezone.utc) match_element = date_time_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, date, 1518003600, None)
def test11get_match_element_with_leap_start_year(self): """Check if leap start_years can parse the 29th February.""" match_context = DummyMatchContext(b"29.02 11:40:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc, start_year=2020) self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1582976400) self.assertEqual(match_context.match_string, b"29.02 11:40:00")
def test5get_match_element_with_unclean_format_string(self): """This test case checks if unclean format_strings can be used.""" match_context = DummyMatchContext( b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works") date_time_model_element = DateTimeModelElement( "path", b"Date %%d: %d.%m.%Y %H:%M:%S%z", timezone.utc) self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1518003600) self.assertEqual(match_context.match_string, b"Date %d: 07.02.2018 11:40:00 UTC+0000")
def test19date_before_unix_timestamps(self): """Check if timestamps before the unix timestamp are processed properly.""" match_context = DummyMatchContext( b"01.01.1900 11:40:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m.%Y %H:%M:%S", timezone.utc) match_element = date_time_model_element.get_match_element( "match1", match_context) self.assertEqual(match_element.match_string, b"01.01.1900 11:40:00") self.assertEqual(match_element.match_object, -2208946800)
def test9value_dependent_modulo_time_match_rule(self): """This case unit the ValueDependentModuloTimeMatchRule. Limit look up not working with tuples.""" description = "Test9Rules" value_dependent_modulo_time_match_rule = ValueDependentModuloTimeMatchRule(self.model_syslog_time, 86400, [self.model_syslog_time], {1550145600: [43200, 86400]}) self.analysis_context.register_component(value_dependent_modulo_time_match_rule, description) date_time_model_element = DateTimeModelElement('time', b'%d.%m.%Y %H:%M:%S') match_context = MatchContext(b'14.02.2019 12:00:00') match_element = date_time_model_element.get_match_element(self.model_syslog, match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), 1550138400, date_time_model_element) self.assertTrue(value_dependent_modulo_time_match_rule.match(log_atom))
def test1date_formats_exceptions(self): """This test case verifies, if all date_format qualifiers are valid and exceptions are raised, if they are invalid.""" match_context = MatchContext(b'07.02.2019 11:40:00: it still works') date_time_model_element = DateTimeModelElement('path', b'%d.%m.%Y %H:%M:%S') self.assertEqual( date_time_model_element.get_match_element( 'match1', match_context).get_match_string(), b'07.02.2019 11:40:00') self.assertRaises(Exception, DateTimeModelElement, 'path', b'%h %b') self.assertRaises(Exception, DateTimeModelElement, 'path', b'%H%H') self.assertRaises(Exception, DateTimeModelElement, 'path', b'%H%s')
def test11get_match_element_with_leap_start_year(self): """Check if leap start_years can parse the 29th February.""" data = b"29.02 11:40:00: it still works" date = b"29.02 11:40:00" date_time_model_element = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc, start_year=2020) match_context = DummyMatchContext(data) match_element = date_time_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, date, 1582976400, None)
def test19date_before_unix_timestamps(self): """Check if timestamps before the unix timestamp are processed properly.""" data = b"01.01.1900 11:40:00: it still works" date = b"01.01.1900 11:40:00" date_time_model_element = DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S", timezone.utc) match_context = DummyMatchContext(data) match_element = date_time_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, date, -2208946800, None)
def test7get_match_element_with_different_text_locales(self): """Test if data with different text locales can be handled with different text_locale parameters.""" installed = False try: self.assertEqual("de_AT.UTF-8", locale.setlocale(locale.LC_TIME, "de_AT.UTF-8")) installed = True except locale.Error: pass DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc, "en_US.UTF-8") if installed: DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc, "de_AT.UTF-8")
def test10get_match_element_without_start_year_defined(self): """Test if dates without year can still be parsed, even without defining the start_year.""" data = b"07.02 11:40:00: it still works" date = b"07.02 11:40:00" date_time_model_element = DateTimeModelElement(self.id_, b"%d.%m %H:%M:%S", timezone.utc) match_context = DummyMatchContext(data) dtm = datetime(datetime.now().year, 2, 7, 11, 40, tzinfo=timezone.utc) total_seconds = ( dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() match_element = date_time_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, date, total_seconds, None)
def get_model(): """Return a model to parse Suricata Fast logs from the AIT-LDS.""" model = SequenceModelElement('model', [ DateTimeModelElement('time', b'%m/%d/%Y-%H:%M:%S.%f'), FixedDataModelElement('brack_str1', b' [**] ['), DecimalIntegerValueModelElement('id1'), FixedDataModelElement('sep1', b':'), DecimalIntegerValueModelElement('id2'), FixedDataModelElement('sep2', b':'), DecimalIntegerValueModelElement('id3'), FixedDataModelElement('sep3', b'] '), DelimitedDataModelElement('message', b' [**] '), FixedDataModelElement('classification_str', b' [**] [Classification: '), DelimitedDataModelElement('classification', b']'), FixedDataModelElement('priority_str', b'] [Priority: '), DecimalIntegerValueModelElement('priority'), FixedDataModelElement('brack_str1', b'] {'), DelimitedDataModelElement('conn', b'}'), FixedDataModelElement('brack_str2', b'} '), IpAddressDataModelElement('src_ip'), FixedDataModelElement('colon', b':'), DecimalIntegerValueModelElement('src_port'), FixedDataModelElement('arrow_str', b' -> '), IpAddressDataModelElement('dst_ip'), FixedDataModelElement('colon', b':'), DecimalIntegerValueModelElement('dst_port'), ]) return model
def get_model(): """Return a model to parse Suricata Fast logs from the AIT-LDS.""" model = SequenceModelElement("model", [ DateTimeModelElement("time", b"%m/%d/%Y-%H:%M:%S.%f"), FixedDataModelElement("brack_str1", b" [**] ["), DecimalIntegerValueModelElement("id1"), FixedDataModelElement("sep1", b":"), DecimalIntegerValueModelElement("id2"), FixedDataModelElement("sep2", b":"), DecimalIntegerValueModelElement("id3"), FixedDataModelElement("sep3", b"] "), DelimitedDataModelElement("message", b" [**] "), FixedDataModelElement("classification_str", b" [**] [Classification: "), DelimitedDataModelElement("classification", b"]"), FixedDataModelElement("priority_str", b"] [Priority: "), DecimalIntegerValueModelElement("priority"), FixedDataModelElement("brack_str1", b"] {"), DelimitedDataModelElement("conn", b"}"), FixedDataModelElement("brack_str2", b"} "), IpAddressDataModelElement("src_ip"), FixedDataModelElement("colon", b":"), DecimalIntegerValueModelElement("src_port"), FixedDataModelElement("arrow_str", b" -> "), IpAddressDataModelElement("dst_ip"), FixedDataModelElement("colon", b":"), DecimalIntegerValueModelElement("dst_port"), ]) return model
def test14learn_new_start_year_without_start_year_set(self): """Test if a new year is learned successfully with the start year being None.""" match_context = DummyMatchContext(b"31.12 23:59:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc) self.assertIsNotNone( date_time_model_element.get_match_element("match1", match_context)) self.assertEqual(match_context.match_string, b"31.12 23:59:00") start_year = date_time_model_element.start_year match_context = DummyMatchContext(b"01.01 11:20:00: it still works") self.assertIsNotNone( date_time_model_element.get_match_element("match1", match_context)) self.assertEqual(match_context.match_string, b"01.01 11:20:00") self.assertEqual(date_time_model_element.start_year, start_year + 1)
def test5_unclean_format_string(self): """This test case checks if unclean format_strings can be used.""" match_context = MatchContext( b'Test 07.02.2018 11:40:00 UTC+0000: it still works') date_time_model_element = DateTimeModelElement( 'path', b'Test %d.%m.%Y %H:%M:%S %z', datetime.timezone.utc, None, 2017) self.assertEqual( date_time_model_element.get_match_element( 'match1', match_context).get_match_object(), 1518003600) self.assertEqual(match_context.match_data, self.__expected_match_context) match_context = MatchContext( b'Test 07.02.2018 11:40:00 UTC-0001: it still works') self.assertEqual( date_time_model_element.get_match_element( 'match1', match_context).get_match_object(), 1518000000) self.assertEqual(match_context.match_data, self.__expected_match_context)
def get_model(): """Return a parser for apache2 access.log.""" new_time_model = SequenceModelElement('time_model', [ DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S '), FixedWordlistDataModelElement('sign', [b'+', b'-']), DecimalIntegerValueModelElement('tz'), FixedDataModelElement('bracket', b']') ]) host_name_model = VariableByteDataModelElement( 'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:') identity_model = VariableByteDataModelElement( 'ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:') user_name_model = VariableByteDataModelElement( 'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-') request_method_model = FixedWordlistDataModelElement( 'method', [ b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH' ]) request_model = VariableByteDataModelElement( 'request', b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+' ) version_model = VariableByteDataModelElement('version', b'0123456789.') status_code_model = DecimalIntegerValueModelElement('status') size_model = DecimalIntegerValueModelElement('size') user_agent_model = VariableByteDataModelElement( 'useragent', b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+;:_ ' ) whitespace_str = b' ' model = SequenceModelElement('accesslog', [ host_name_model, FixedDataModelElement('sp0', whitespace_str), identity_model, FixedDataModelElement('sp1', whitespace_str), user_name_model, FixedDataModelElement('sp2', whitespace_str), new_time_model, FixedDataModelElement('sp3', b' "'), request_method_model, FixedDataModelElement('sp4', whitespace_str), request_model, FixedDataModelElement('sp5', b' HTTP/'), version_model, FixedDataModelElement('sp6', b'" '), status_code_model, FixedDataModelElement('sp7', whitespace_str), size_model, FixedDataModelElement('sp8', b' "-" "'), user_agent_model, FixedDataModelElement('sp9', b'"'), ]) return model
def test18same_timestamp_multiple_times(self): """Test if the DateTimeModelElement can handle multiple same timestamps.""" match_context = DummyMatchContext( b"07.02.2019 11:40:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m.%Y %H:%M:%S", timezone.utc) match_element = date_time_model_element.get_match_element( "match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00") match_context = DummyMatchContext( b"07.02.2019 11:40:00: it still works") match_element = date_time_model_element.get_match_element( "match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
def get_model(): """Return a model to parse Apache Error logs from the AIT-LDS.""" model = SequenceModelElement('model', [ FixedDataModelElement('sp1', b'['), FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']), FixedDataModelElement('sp2', b' '), DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'), FixedDataModelElement('error_str', b'] [:error] [pid '), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('sp3', b'] [client '), IpAddressDataModelElement('client_ip'), FixedDataModelElement('colon', b':'), DecimalIntegerValueModelElement('client_port'), FixedDataModelElement('php', b'] PHP '), FirstMatchModelElement('fphp', [ SequenceModelElement('warning', [ FixedDataModelElement('warning_str', b'Warning: '), FirstMatchModelElement('warning', [ SequenceModelElement('declaration', [ FixedDataModelElement('declaration_str', b'Declaration of '), DelimitedDataModelElement('function', b')'), FixedDataModelElement('compatible_str', b') should be compatible with '), DelimitedDataModelElement('function2', b')'), FixedDataModelElement('compatible_str', b') in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line'), FixedDataModelElement('referer_str', b', referer: '), AnyByteDataModelElement('referer')]), SequenceModelElement('system', [ FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line')])])]), SequenceModelElement('notice', [ FixedDataModelElement('notice_str', b'Notice: Undefined index: '), DelimitedDataModelElement('command', b' '), FixedDataModelElement('sp', b' in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line')]), SequenceModelElement('deprecated', [ FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class ' b'will not be constructors in a future version of PHP; '), DelimitedDataModelElement('class', b' '), FixedDataModelElement('constructor_str', b' has a deprecated constructor in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line'), FixedDataModelElement('referer_str', b', referer: '), AnyByteDataModelElement('referer'), ])])]) return model
def test13learn_new_start_year_with_start_year_set(self): """Test if a new year is learned successfully with the start year being set.""" start_year = 2020 match_context = DummyMatchContext(b"31.12 23:59:00: it still works") date_time_model_element = DateTimeModelElement("path", b"%d.%m %H:%M:%S", timezone.utc, start_year=start_year) self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1609459140) self.assertEqual(match_context.match_string, b"31.12 23:59:00") self.assertEqual(date_time_model_element.start_year, start_year) match_context = DummyMatchContext(b"01.01 11:20:00: it still works") self.assertEqual( date_time_model_element.get_match_element( "match1", match_context).get_match_object(), 1609500000) self.assertEqual(match_context.match_string, b"01.01 11:20:00") self.assertEqual(date_time_model_element.start_year, start_year + 1)