def test15max_time_jump_seconds_in_time(self): """ Test if the max_time_jump_seconds parameter works if the next date is in time. Warnings with unqualified timestamp year wraparound. """ log_stream = StringIO() logging.basicConfig(stream=log_stream, level=logging.INFO) max_time_jump_seconds = 86400 start_year = 2020 match_context = DummyMatchContext(b"31.12 23:59:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year, max_time_jump_seconds=max_time_jump_seconds) self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609459140) self.assertEqual(match_context.match_string, b"31.12 23:59:00") self.assertEqual(multi_locale_dtme.start_year, 2020) match_context = DummyMatchContext(b"01.01 23:59:00: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609545540) self.assertEqual(match_context.match_string, b"01.01 23:59:00") self.assertEqual(multi_locale_dtme.start_year, 2021) self.assertIn("WARNING:DEBUG:DateTimeModelElement unqualified timestamp year wraparound detected from 2021-01-01T23:59:00+00:00 to " "2021-01-01T23:59:00+00:00", log_stream.getvalue()) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) initialize_loggers(self.aminer_config, getpwnam("aminer").pw_uid, getgrnam("aminer").gr_gid)
def test26get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S", None, None)]) data = b"07.02.2019 11:40:00: it still works" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(self.path, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def test4wrong_date(self): """Test if wrong input data does not return a match.""" tz_gmt10 = pytz.timezone("Etc/GMT+10") en_gb_utf8 = "en_GB.utf8" en_us_utf8 = "en_US.utf8" de_at_utf8 = "de_AT.utf8" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S.%f", None, None), (b"%d.%m.%Y %H:%M:%S%z", None, None), (b"%d.%m.%Y %H:%M:%S", None, None), (b"%d.%m.%YT%H:%M:%S", None, None), (b"%d.%m.%Y", None, None), (b"%H:%M:%S:%f", None, de_at_utf8), (b"%H:%M:%S", None, None), (b"%b %d", tz_gmt10, de_at_utf8), (b"%d %b %Y", None, en_gb_utf8), (b"%dth %b %Y", None, en_gb_utf8), (b"%d/%m/%Y", None, en_gb_utf8), (b"%m-%d-%Y", None, en_us_utf8), (b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)]) # wrong day data = b"32.03.2019 11:40:00: it still works" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) # wrong month data = b"01.13.2019 11:40:00: it still works" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) # wrong year data = b"01.01.00 11:40:00: it still works" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) # wrong date leap year data = b"29.02.2019 11:40:00: it still works" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) # British date data = b"13 Dezember 2019" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def test9get_match_element_with_start_year(self): """Test if dates without year can be parsed, when the start_year is defined.""" match_context = DummyMatchContext(b"07.02 11:40:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2017) self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1486467600) self.assertEqual(match_context.match_string, b"07.02 11:40:00") match_context = DummyMatchContext(b"07.02 11:40:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2019) self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1549539600) self.assertEqual(match_context.match_string, b"07.02 11:40:00")
def test14learn_new_start_year_without_start_year_set(self): """Test if a new year is learned successfully with the start year being None.""" match_context = DummyMatchContext(b"31.12 23:59:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)]) self.assertIsNotNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_string, b"31.12 23:59:00") start_year = multi_locale_dtme.start_year match_context = DummyMatchContext(b"01.01 11:20:00: it still works") self.assertIsNotNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_string, b"01.01 11:20:00") self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test13learn_new_start_year_with_start_year_set(self): """Test if a new year is learned successfully with the start year being set.""" start_year = 2020 match_context = DummyMatchContext(b"31.12 23:59:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year) self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609459140) self.assertEqual(match_context.match_string, b"31.12 23:59:00") self.assertEqual(multi_locale_dtme.start_year, start_year) match_context = DummyMatchContext(b"01.01 11:20:00: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1609500000) self.assertEqual(match_context.match_string, b"01.01 11:20:00") self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test18same_timestamp_multiple_times(self): """Test if the MultiLocaleDateTimeModelElement can handle multiple same timestamps.""" multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S", None, None)]) match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00") match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
def test5get_match_element_with_unclean_format_string(self): """This test case checks if unclean format_strings can be used.""" # example "Date: 09.03.2021, Time: 10:02" match_context = DummyMatchContext(b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"Date %%d: %d.%m.%Y %H:%M:%S%z", None, None)]) self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518003600) self.assertEqual(match_context.match_string, b"Date %d: 07.02.2018 11:40:00 UTC+0000")
def test19date_before_unix_timestamps(self): """Check if timestamps before the unix timestamp are processed properly.""" multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S", None, None)]) match_context = DummyMatchContext(b"01.01.1900 11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"01.01.1900 11:40:00") self.assertEqual(match_element.match_object, -2208946800)
def test17time_change_cest_cet(self): """Check if the time change from CET to CEST and vice versa work as expected.""" multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S%z", None, None)]) match_context = DummyMatchContext(b"24.03.2018 11:40:00 CET: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1521888000) self.assertEqual(match_context.match_string, b"24.03.2018 11:40:00 CET") match_context = DummyMatchContext(b"25.03.2018 11:40:00 CEST: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1521970800) self.assertEqual(match_context.match_string, b"25.03.2018 11:40:00 CEST") match_context = DummyMatchContext(b"27.10.2018 11:40:00 CEST: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540633200) self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 CEST") match_context = DummyMatchContext(b"28.10.2018 11:40:00 CET: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540723200) self.assertEqual(match_context.match_string, b"28.10.2018 11:40:00 CET") match_context = DummyMatchContext(b"27.10.2018 11:40:00 EST: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540658400) self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 EST") match_context = DummyMatchContext(b"27.10.2018 11:40:00 PDT: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540665600) self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 PDT") match_context = DummyMatchContext(b"27.10.2018 11:40:00 GMT: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1540640400) self.assertEqual(match_context.match_string, b"27.10.2018 11:40:00 GMT")
def test6get_match_element_with_different_time_zones(self): """Test if different time_zones work with the MultiLocaleDateTimeModelElement.""" multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m.%Y %H:%M:%S%z", None, None)]) match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-1200: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518046800) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-1200") match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-12: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518046800) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-12") match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-5: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518021600) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-5") match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC-0500: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518021600) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC-0500") match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC+0000: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518003600) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+0000") match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC+0100: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1518000000) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+0100") match_context = DummyMatchContext(b"07.02.2018 11:40:00 UTC+1400: it still works") self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1517953200) self.assertEqual(match_context.match_string, b"07.02.2018 11:40:00 UTC+1400")
def test12get_match_element_without_leap_start_year(self): """Check if normal start_years can not parse the 29th February.""" data = b"29.02 11:40:00: it still works" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2019) match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def test18same_timestamp_multiple_times(self): """Test if the MultiLocaleDateTimeModelElement can handle multiple same timestamps.""" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S", None, None)]) data = b"07.02.2019 11:40:00: it still works" date = b"07.02.2019 11:40:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1549539600, None) match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1549539600, None)
def test4wrong_date(self): """Test if wrong input data does not return a match.""" tz_gmt10 = pytz.timezone("Etc/GMT+10") en_gb_utf8 = "en_GB.utf8" en_us_utf8 = "en_US.utf8" de_at_utf8 = "de_AT.utf8" multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [ (b"%d.%m.%Y %H:%M:%S.%f", None, None), (b"%d.%m.%Y %H:%M:%S%z", None, None), (b"%d.%m.%Y %H:%M:%S", None, None), (b"%d.%m.%YT%H:%M:%S", None, None), (b"%d.%m.%Y", None, None), (b"%H:%M:%S:%f", None, de_at_utf8), (b"%H:%M:%S", None, None), (b"%b %d", tz_gmt10, de_at_utf8), (b"%d %b %Y", None, en_gb_utf8), (b"%dth %b %Y", None, en_gb_utf8), (b"%d/%m/%Y", None, en_gb_utf8), (b"%m-%d-%Y", None, en_us_utf8), (b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)]) # wrong day match_context = DummyMatchContext(b"32.03.2019 11:40:00: it still works") self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_data, b"32.03.2019 11:40:00: it still works") self.assertEqual(match_context.match_string, b"") # wrong month match_context = DummyMatchContext(b"01.13.2019 11:40:00: it still works") self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_data, b"01.13.2019 11:40:00: it still works") self.assertEqual(match_context.match_string, b"") # wrong year match_context = DummyMatchContext(b"01.01.00 11:40:00: it still works") self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_data, b"01.01.00 11:40:00: it still works") self.assertEqual(match_context.match_string, b"") # wrong date leap year match_context = DummyMatchContext(b"29.02.2019 11:40:00: it still works") self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_data, b"29.02.2019 11:40:00: it still works") self.assertEqual(match_context.match_string, b"") # British date string = b"13 Dezember 2019" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) self.assertEqual(match_element, None)
def test9get_match_element_with_start_year(self): """Test if dates without year can be parsed, when the start_year is defined.""" data = b"07.02 11:40:00: it still works" date = b"07.02 11:40:00" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2017) match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1486467600, None) multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2019) match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1549539600, None)
def test5get_match_element_with_unclean_format_string(self): """This test case checks if unclean format_strings can be used.""" data = b"Date %d: 07.02.2018 11:40:00 UTC+0000: it still works" date = b"Date %d: 07.02.2018 11:40:00 UTC+0000" match_context = DummyMatchContext(data) multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"Date %%d: %d.%m.%Y %H:%M:%S%z", None, None)]) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518003600, None)
def test11get_match_element_with_leap_start_year(self): """Check if leap start_years can parse the 29th February.""" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=2020) data = b"29.02 11:40:00: it still works" date = b"29.02 11:40:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1582976400, None)
def test19date_before_unix_timestamps(self): """Check if timestamps before the unix timestamp are processed properly.""" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S", None, None)]) data = b"01.01.1900 11:40:00: it still works" date = b"01.01.1900 11:40:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, -2208946800, None)
def test16max_time_jump_seconds_exceeded(self): """ Test if the start_year is not updated, when the next date exceeds the max_time_jump_seconds. A time inconsistency warning must occur. """ log_stream = StringIO() logging.basicConfig(stream=log_stream, level=logging.INFO) max_time_jump_seconds = 86400 start_year = 2020 multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year, max_time_jump_seconds=max_time_jump_seconds) data = b"31.12 23:59:00: it still works" date = b"31.12 23:59:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1609459140, None) self.assertEqual(multi_locale_dtme.start_year, start_year) data = b"01.01 23:59:01: it still works" date = b"01.01 23:59:01" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1577923141, None) self.assertEqual(multi_locale_dtme.start_year, start_year) self.assertIn( "WARNING:DEBUG:DateTimeModelElement time inconsistencies parsing b'01.01 23:59:01', expecting value around " "1609459140. Check your settings!", log_stream.getvalue()) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) initialize_loggers(self.aminer_config, getpwnam("aminer").pw_uid, getgrnam("aminer").gr_gid)
def test14learn_new_start_year_without_start_year_set(self): """Test if a new year is learned successfully with the start year being None.""" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)]) data = b"31.12 23:59:00: it still works" date = b"31.12 23:59:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) dtm = datetime(datetime.now().year, 12, 31, 23, 59, tzinfo=timezone.utc) total_seconds = ( dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, total_seconds, None) start_year = multi_locale_dtme.start_year data = b"01.01 11:20:00: it still works" date = b"01.01 11:20:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) dtm = datetime(datetime.now().year + 1, 1, 1, 11, 20, tzinfo=timezone.utc) total_seconds = ( dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, total_seconds, None) self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test13learn_new_start_year_with_start_year_set(self): """Test if a new year is learned successfully with the start year being set.""" start_year = 2020 multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)], start_year=start_year) data = b"31.12 23:59:00: it still works" date = b"31.12 23:59:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1609459140, None) self.assertEqual(multi_locale_dtme.start_year, start_year) data = b"01.01 11:20:00: it still works" date = b"01.01 11:20:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1609500000, None) self.assertEqual(multi_locale_dtme.start_year, start_year + 1)
def test10get_match_element_without_start_year_defined(self): """Test if dates without year can still be parsed, even without defining the start_year.""" data = b"07.02 11:40:00: it still works" date = b"07.02 11:40:00" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m %H:%M:%S", None, None)]) match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) dtm = datetime(datetime.now().year, 2, 7, 11, 40, tzinfo=timezone.utc) total_seconds = ( dtm - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, total_seconds, None)
def test17time_change_cest_cet(self): """Check if the time change from CET to CEST and vice versa work as expected.""" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S%z", None, None)]) data = b"24.03.2018 11:40:00 CET: it still works" date = b"24.03.2018 11:40:00 CET" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1521888000, None) data = b"25.03.2018 11:40:00 CEST: it still works" date = b"25.03.2018 11:40:00 CEST" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1521970800, None) data = b"27.10.2018 11:40:00 CEST: it still works" date = b"27.10.2018 11:40:00 CEST" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1540633200, None) data = b"28.10.2018 11:40:00 CET: it still works" date = b"28.10.2018 11:40:00 CET" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1540723200, None) data = b"27.10.2018 11:40:00 EST: it still works" date = b"27.10.2018 11:40:00 EST" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1540658400, None) data = b"27.10.2018 11:40:00 PDT: it still works" date = b"27.10.2018 11:40:00 PDT" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1540665600, None) data = b"27.10.2018 11:40:00 GMT: it still works" date = b"27.10.2018 11:40:00 GMT" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1540640400, None)
def test10get_match_element_without_start_year_defined(self): """Test if dates without year can still be parsed, even without defining the start_year.""" match_context = DummyMatchContext(b"07.02 11:40:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)]) self.assertIsNotNone(multi_locale_dtme.get_match_element("match1", match_context)) self.assertEqual(match_context.match_string, b"07.02 11:40:00")
def test3get_match_element_with_different_date_formats(self): """Test if different date_formats can be used to match data.""" tz_gmt10 = pytz.timezone("Etc/GMT+10") en_gb_utf8 = "en_GB.utf8" en_us_utf8 = "en_US.utf8" de_at_utf8 = "de_AT.utf8" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S.%f", None, None), (b"%d.%m.%Y %H:%M:%S%z", None, None), (b"%d.%m.%Y %H:%M:%S", None, None), (b"%d.%m.%YT%H:%M:%S", None, None), (b"%d.%m.%Y", None, None), (b"%H:%M:%S:%f", None, de_at_utf8), (b"%H:%M:%S", None, None), (b"%b %d", tz_gmt10, de_at_utf8), (b"%d %b %Y", None, en_gb_utf8), (b"%dth %b %Y", None, en_gb_utf8), (b"%d/%m/%Y", None, en_gb_utf8), (b"%m-%d-%Y", None, en_us_utf8), (b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)]) # test normal date data = b"07.02.2019 11:40:00: it still works" date = b"07.02.2019 11:40:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format1", self.path, date, 1549539600, None) # test leap year date data = b"29.02.2020 11:40:00: it still works" date = b"29.02.2020 11:40:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format1", self.path, date, 1582976400, None) # test normal date with T data = b"07.02.2019T11:40:00: it still works" date = b"07.02.2019T11:40:00" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format3", self.path, date, 1549539600, None) # test normal date with fractions data = b"07.02.2019 11:40:00.123456: it still works" date = b"07.02.2019 11:40:00.123456" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1549539600.123456, None) # test normal date with z data = b"07.02.2019 11:40:00+0000: it still works" date = b"07.02.2019 11:40:00+0000" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format1", self.path, date, 1549539600, None) # test with only date defined data = b"07.02.2019: it still works" date = b"07.02.2019" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format4", self.path, date, 1549497600, None) # test with only time defined. Here obviously the seconds can not be tested. data = b"11:40:23: it still works" date = b"11:40:23" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format6", self.path, date, match_element.match_object, None) data = b"Feb 25 something happened" date = b"Feb 25" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) dtm = datetime(datetime.now().year, 2, 25, tzinfo=tz_gmt10) # total_seconds should be in UTC, so the timezones are parsed out. total_seconds = (dtm - datetime(1970, 1, 1, tzinfo=tz_gmt10) ).days * 86400 - dtm.utcoffset().total_seconds() self.compare_match_results(data, match_element, match_context, self.id_ + "/format7", self.path, date, total_seconds, None) # British date data = b"13 Apr 2019 something happened" date = b"13 Apr 2019" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format8", self.path, date, 1555113600, None) # British date 2 data = b"13th Apr 2019 something happened" date = b"13th Apr 2019" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format9", self.path, date, 1555113600, None) # British date 3 data = b"13/04/2019 something happened" date = b"13/04/2019" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format10", self.path, date, 1555113600, None) # US date data = b"04-13-2019 something happened" date = b"04-13-2019" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format11", self.path, date, 1555113600, None) # Austrian date no year - year should already be learnt. # start year has to be 2021, because all other formats have defined years. data = b"13.04. 15:12:54:201 something happened" date = b"13.04. 15:12:54:201" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format12", self.path, date, 1618326774.201, None) multi_locale_dtme.latest_parsed_timestamp = None # Austrian time no date data = b"15:12:54:201 something happened" date = b"15:12:54:201" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) dtm = datetime(datetime.now().year, datetime.now().month, datetime.now().day, 15, 12, 54, 201, tzinfo=timezone.utc) # total_seconds should be in UTC, so the timezones are parsed out. delta = (dtm - datetime(1970, 1, 1, tzinfo=dtm.tzinfo)) total_seconds = delta.days * 86400 + delta.seconds + delta.microseconds / 1000 self.compare_match_results(data, match_element, match_context, self.id_ + "/format5", self.path, date, total_seconds, None)
def test11get_match_element_with_leap_start_year(self): """Check if leap start_years can parse the 29th February.""" match_context = DummyMatchContext(b"29.02 11:40:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2020) self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1582976400) self.assertEqual(match_context.match_string, b"29.02 11:40:00")
def test12get_match_element_without_leap_start_year(self): """Check if normal start_years can not parse the 29th February.""" match_context = DummyMatchContext(b"29.02 11:40:00: it still works") multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2019) self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context))
def test6get_match_element_with_different_time_zones(self): """Test if different time_zones work with the MultiLocaleDateTimeModelElement.""" multi_locale_dtme = MultiLocaleDateTimeModelElement( self.id_, [(b"%d.%m.%Y %H:%M:%S%z", None, None)]) data = b"07.02.2018 11:40:00 UTC-1200: it still works" date = b"07.02.2018 11:40:00 UTC-1200" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518046800, None) data = b"07.02.2018 11:40:00 UTC-12: it still works" date = b"07.02.2018 11:40:00 UTC-12" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518046800, None) data = b"07.02.2018 11:40:00 UTC-5: it still works" date = b"07.02.2018 11:40:00 UTC-5" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518021600, None) data = b"07.02.2018 11:40:00 UTC-0500: it still works" date = b"07.02.2018 11:40:00 UTC-0500" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518021600, None) data = b"07.02.2018 11:40:00 UTC+0000: it still works" date = b"07.02.2018 11:40:00 UTC+0000" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518003600, None) data = b"07.02.2018 11:40:00 UTC+0100: it still works" date = b"07.02.2018 11:40:00 UTC+0100" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1518000000, None) data = b"07.02.2018 11:40:00 UTC+1400: it still works" date = b"07.02.2018 11:40:00 UTC+1400" match_context = DummyMatchContext(data) match_element = multi_locale_dtme.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_ + "/format0", self.path, date, 1517953200, None)
def test3get_match_element_with_different_date_formats(self): """Test if different date_formats can be used to match data.""" tz_gmt10 = pytz.timezone("Etc/GMT+10") en_gb_utf8 = "en_GB.utf8" en_us_utf8 = "en_US.utf8" de_at_utf8 = "de_AT.utf8" multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [ (b"%d.%m.%Y %H:%M:%S.%f", None, None), (b"%d.%m.%Y %H:%M:%S%z", None, None), (b"%d.%m.%Y %H:%M:%S", None, None), (b"%d.%m.%YT%H:%M:%S", None, None), (b"%d.%m.%Y", None, None), (b"%H:%M:%S:%f", None, de_at_utf8), (b"%H:%M:%S", None, None), (b"%b %d", tz_gmt10, de_at_utf8), (b"%d %b %Y", None, en_gb_utf8), (b"%dth %b %Y", None, en_gb_utf8), (b"%d/%m/%Y", None, en_gb_utf8), (b"%m-%d-%Y", None, en_us_utf8), (b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)]) # test normal date match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00") # test leap year date match_context = DummyMatchContext(b"29.02.2020 11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"29.02.2020 11:40:00") self.assertEqual(match_element.match_object, 1582976400) self.assertEqual(match_context.match_string, b"29.02.2020 11:40:00") # test normal date with T match_context = DummyMatchContext(b"07.02.2019T11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019T11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019T11:40:00") # test normal date with fractions match_context = DummyMatchContext(b"07.02.2019 11:40:00.123456: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00.123456") self.assertEqual(match_element.match_object, 1549539600.123456) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00.123456") # test normal date with z match_context = DummyMatchContext(b"07.02.2019 11:40:00+0000: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00+0000") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00+0000") match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00") self.assertEqual(match_element.match_object, 1549539600) self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00") # test with only date defined match_context = DummyMatchContext(b"07.02.2019: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"07.02.2019") self.assertEqual(match_element.match_object, 1549497600) self.assertEqual(match_context.match_string, b"07.02.2019") # test with only time defined. Here obviously the seconds can not be tested. match_context = DummyMatchContext(b"11:40:23: it still works") match_element = multi_locale_dtme.get_match_element("match1", match_context) self.assertEqual(match_element.match_string, b"11:40:23") self.assertEqual(match_context.match_string, b"11:40:23") string = b"Feb 25" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) date = datetime(datetime.now().year, 2, 25, tzinfo=tz_gmt10) # total_seconds should be in UTC, so the timezones are parsed out. total_seconds = (date - datetime(1970, 1, 1, tzinfo=tz_gmt10)).days * 86400 - date.utcoffset().total_seconds() self.assertEqual(match_element.match_string, b"Feb 25") self.assertEqual(match_element.match_object, total_seconds) self.assertEqual(match_context.match_string, b"Feb 25") # British date string = b"13 Apr 2019" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) self.assertEqual(match_element.match_string, b"13 Apr 2019") self.assertEqual(match_element.match_object, 1555113600) self.assertEqual(match_context.match_string, b"13 Apr 2019") # British date 2 string = b"13th Apr 2019" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) self.assertEqual(match_element.match_string, b"13th Apr 2019") self.assertEqual(match_element.match_object, 1555113600) self.assertEqual(match_context.match_string, b"13th Apr 2019") # British date 3 string = b"13/04/2019" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) self.assertEqual(match_element.match_string, b"13/04/2019") self.assertEqual(match_element.match_object, 1555113600) self.assertEqual(match_context.match_string, b"13/04/2019") # US date string = b"04-13-2019" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) self.assertEqual(match_element.match_string, b"04-13-2019") self.assertEqual(match_element.match_object, 1555113600) self.assertEqual(match_context.match_string, b"04-13-2019") # Austrian date no year - year should already be learnt. # start year has to be 2021, because all other formats have defined years. string = b"13.04. 15:12:54:201" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) self.assertEqual(match_element.match_string, b"13.04. 15:12:54:201") self.assertEqual(match_element.match_object, 1618326774.201) self.assertEqual(match_context.match_string, b"13.04. 15:12:54:201") multi_locale_dtme.latest_parsed_timestamp = None # Austrian time no date string = b"15:12:54:201" match_context = DummyMatchContext(string) match_element = multi_locale_dtme.get_match_element("match", match_context) date = datetime(datetime.now().year, datetime.now().month, datetime.now().day, 15, 12, 54, 201, tzinfo=timezone.utc) # total_seconds should be in UTC, so the timezones are parsed out. delta = (date - datetime(1970, 1, 1, tzinfo=date.tzinfo)) total_seconds = delta.days * 86400 + delta.seconds + delta.microseconds / 1000 self.assertEqual(match_element.match_string, b"15:12:54:201") self.assertEqual(match_element.match_object, total_seconds) self.assertEqual(match_context.match_string, b"15:12:54:201")