def test3get_match_element_with_different_date_formats(self):
"""Test if different date_formats can be used to match data."""
tz_gmt10 = pytz.timezone("Etc/GMT+10")
en_gb_utf8 = "en_GB.utf8"
en_us_utf8 = "en_US.utf8"
de_at_utf8 = "de_AT.utf8"
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S.%f", None, None),
(b"%d.%m.%Y %H:%M:%S%z", None, None),
(b"%d.%m.%Y %H:%M:%S", None, None),
(b"%d.%m.%YT%H:%M:%S", None, None),
(b"%d.%m.%Y", None, None),
(b"%H:%M:%S:%f", None, de_at_utf8),
(b"%H:%M:%S", None, None),
(b"%b %d", tz_gmt10, de_at_utf8),
(b"%d %b %Y", None, en_gb_utf8),
(b"%dth %b %Y", None, en_gb_utf8),
(b"%d/%m/%Y", None, en_gb_utf8),
(b"%m-%d-%Y", None, en_us_utf8),
(b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)])
# test normal date
data = b"07.02.2019 11:40:00: it still works"
date = b"07.02.2019 11:40:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format1", self.path, date,
1549539600, None)
# test leap year date
data = b"29.02.2020 11:40:00: it still works"
date = b"29.02.2020 11:40:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format1", self.path, date,
1582976400, None)
# test normal date with T
data = b"07.02.2019T11:40:00: it still works"
date = b"07.02.2019T11:40:00"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format3", self.path, date,
1549539600, None)
# test normal date with fractions
data = b"07.02.2019 11:40:00.123456: it still works"
date = b"07.02.2019 11:40:00.123456"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1549539600.123456, None)
# test normal date with z
data = b"07.02.2019 11:40:00+0000: it still works"
date = b"07.02.2019 11:40:00+0000"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format1", self.path, date,
1549539600, None)
# test with only date defined
data = b"07.02.2019: it still works"
date = b"07.02.2019"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format4", self.path, date,
1549497600, None)
# test with only time defined. Here obviously the seconds can not be tested.
data = b"11:40:23: it still works"
date = b"11:40:23"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format6", self.path, date,
match_element.match_object, None)
data = b"Feb 25 something happened"
date = b"Feb 25"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
dtm = datetime(datetime.now().year, 2, 25, tzinfo=tz_gmt10)
# total_seconds should be in UTC, so the timezones are parsed out.
total_seconds = (dtm - datetime(1970, 1, 1, tzinfo=tz_gmt10)
).days * 86400 - dtm.utcoffset().total_seconds()
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format7", self.path, date,
total_seconds, None)
# British date
data = b"13 Apr 2019 something happened"
date = b"13 Apr 2019"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format8", self.path, date,
1555113600, None)
# British date 2
data = b"13th Apr 2019 something happened"
date = b"13th Apr 2019"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format9", self.path, date,
1555113600, None)
# British date 3
data = b"13/04/2019 something happened"
date = b"13/04/2019"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format10", self.path, date,
1555113600, None)
# US date
data = b"04-13-2019 something happened"
date = b"04-13-2019"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format11", self.path, date,
1555113600, None)
# Austrian date no year - year should already be learnt.
# start year has to be 2021, because all other formats have defined years.
data = b"13.04. 15:12:54:201 something happened"
date = b"13.04. 15:12:54:201"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format12", self.path, date,
1618326774.201, None)
multi_locale_dtme.latest_parsed_timestamp = None
# Austrian time no date
data = b"15:12:54:201 something happened"
date = b"15:12:54:201"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
dtm = datetime(datetime.now().year,
datetime.now().month,
datetime.now().day,
15,
12,
54,
201,
tzinfo=timezone.utc)
# total_seconds should be in UTC, so the timezones are parsed out.
delta = (dtm - datetime(1970, 1, 1, tzinfo=dtm.tzinfo))
total_seconds = delta.days * 86400 + delta.seconds + delta.microseconds / 1000
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format5", self.path, date,
total_seconds, None)
def test12get_match_element_without_leap_start_year(self):
"""Check if normal start_years can not parse the 29th February."""
match_context = DummyMatchContext(b"29.02 11:40:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2019)
self.assertIsNone(multi_locale_dtme.get_match_element("match1", match_context))
def test6get_match_element_with_different_time_zones(self):
"""Test if different time_zones work with the MultiLocaleDateTimeModelElement."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S%z", None, None)])
data = b"07.02.2018 11:40:00 UTC-1200: it still works"
date = b"07.02.2018 11:40:00 UTC-1200"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518046800, None)
data = b"07.02.2018 11:40:00 UTC-12: it still works"
date = b"07.02.2018 11:40:00 UTC-12"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518046800, None)
data = b"07.02.2018 11:40:00 UTC-5: it still works"
date = b"07.02.2018 11:40:00 UTC-5"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518021600, None)
data = b"07.02.2018 11:40:00 UTC-0500: it still works"
date = b"07.02.2018 11:40:00 UTC-0500"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518021600, None)
data = b"07.02.2018 11:40:00 UTC+0000: it still works"
date = b"07.02.2018 11:40:00 UTC+0000"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518003600, None)
data = b"07.02.2018 11:40:00 UTC+0100: it still works"
date = b"07.02.2018 11:40:00 UTC+0100"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1518000000, None)
data = b"07.02.2018 11:40:00 UTC+1400: it still works"
date = b"07.02.2018 11:40:00 UTC+1400"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1517953200, None)
def test10get_match_element_without_start_year_defined(self):
"""Test if dates without year can still be parsed, even without defining the start_year."""
match_context = DummyMatchContext(b"07.02 11:40:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)])
self.assertIsNotNone(multi_locale_dtme.get_match_element("match1", match_context))
self.assertEqual(match_context.match_string, b"07.02 11:40:00")
def test11get_match_element_with_leap_start_year(self):
"""Check if leap start_years can parse the 29th February."""
match_context = DummyMatchContext(b"29.02 11:40:00: it still works")
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [(b"%d.%m %H:%M:%S", None, None)], start_year=2020)
self.assertEqual(multi_locale_dtme.get_match_element("match1", match_context).get_match_object(), 1582976400)
self.assertEqual(match_context.match_string, b"29.02 11:40:00")
def test3get_match_element_valid_match(self):
"""Try all values and check if the desired results are produced."""
allowed_chars = [
b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7", b"8", b"9", b"a",
b"b", b"c", b"d", b"e", b"f"
]
char1 = b"\x00"
char2 = b"\x00"
hex_string_model_element = HexStringModelElement(self.id_)
while ord(char2) < ord(b"\x7F"):
data = char2 + char1
match_context = DummyMatchContext(data)
match_element = hex_string_model_element.get_match_element(
self.path, match_context)
if char2 in allowed_chars:
if char1 in allowed_chars:
match_context.match_string = bytes.fromhex(data.decode(
)) # match_context.match_string check has to be skipped.
match_context.match_data = data[len(
match_context.match_string
):] # match_context.match_data has to be rewritten.
self.compare_match_results(data, match_element,
match_context, self.id_,
self.path,
bytes.fromhex(data.decode()),
data, None)
self.assertEqual(match_element.get_match_object(), data)
else:
match_context.match_string = bytes.fromhex(
"0" + char2.decode()
) # match_context.match_string check has to be skipped.
self.compare_match_results(
data, match_element,
match_context, self.id_, self.path,
bytes.fromhex("0" + char2.decode()), char2, None)
self.assertEqual(match_element.get_match_object(), char2)
else:
self.compare_no_match_results(data, match_element,
match_context)
if ord(char1) == 0x7f:
char1 = b"\x00"
char2 = bytes(chr(ord(char2) + 1), "utf-8")
else:
char1 = bytes(chr(ord(char1) + 1), "utf-8")
allowed_chars = [
b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7", b"8", b"9", b"A",
b"B", b"C", b"D", b"E", b"F"
]
char1 = b"\x00"
char2 = b"\x00"
hex_string_model_element = HexStringModelElement(self.id_, True)
while ord(char2) < ord(b"\x7F"):
data = char2 + char1
match_context = DummyMatchContext(data)
match_element = hex_string_model_element.get_match_element(
self.path, match_context)
if char2 in allowed_chars:
if char1 in allowed_chars:
self.assertEqual(match_element.get_match_object(), data)
else:
self.assertEqual(match_element.get_match_object(), char2)
else:
self.compare_no_match_results(data, match_element,
match_context)
if ord(char1) == 0x7f:
char1 = b"\x00"
char2 = bytes(chr(ord(char2) + 1), "utf-8")
else:
char1 = bytes(chr(ord(char1) + 1), "utf-8")
def test6get_match_element_null_value(self):
"""Test if null values are parsed to "null"."""
key_parser_dict = {
"works": DummyFirstMatchModelElement("id", [
DummyFixedDataModelElement("abc", b"abc"), DummyFixedDataModelElement("123", b"123")]),
"null": DummyFirstMatchModelElement("wordlist", [
DummyFixedDataModelElement("allowed", b"allowed value"), DummyFixedDataModelElement("problem", b"null")])}
data1 = b"""{
"works": "abc",
"null": "allowed value"
}"""
data2 = b"""{
"works": "123",
"null": null
}"""
data3 = b"""{"a": {"b": "c"}}"""
data4 = b"""{"a": null}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
data = data1
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
data = data2
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
key_parser_dict = {"a": {"b": DummyFixedDataModelElement("c", b"c")}}
json_model_element = JsonModelElement(self.id_, key_parser_dict)
data = data3
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
data = data4
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test8get_match_element_same_value_as_key(self):
"""Test if object with the same key-value pairs are parsed correctly."""
key_parser_dict = {
"abc":
DummyFirstMatchModelElement("first", [
DummyFixedDataModelElement("abc", b"abc"),
DummyFixedDataModelElement("abc", b"ab"),
DummyFixedDataModelElement("abc", b"bc"),
DummyFixedDataModelElement("abc", b"ba"),
DummyFixedDataModelElement("abc", b"b"),
DummyFixedDataModelElement("abc", b"d")
])
}
data = b"""{"abc":"abc"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"ab"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"bc"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"b"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"d"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"ba"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test9get_match_element_empty_array_empty_object_null(self):
"""Test if the keywords EMPTY_ARRAY, EMPTY_OBJECT, EMPTY_STRING and None (null) work properly."""
key_parser_dict = {
"menu": {
"id": "EMPTY_OBJECT",
"value": "EMPTY_ARRAY",
"popup": {
"menuitem": [{
"value":
DummyFixedDataModelElement("null", b"null"),
"onclick":
DummyFirstMatchModelElement("buttonOnclick", [
DummyFixedDataModelElement("create_new_doc",
b"CreateNewDoc()"),
DummyFixedDataModelElement("open_doc",
b"OpenDoc()"),
DummyFixedDataModelElement("close_doc",
b"CloseDoc()")
]),
"optional_key_clickable":
DummyFirstMatchModelElement("clickable", [
DummyFixedDataModelElement("true", b"true"),
DummyFixedDataModelElement("false", b"false")
])
}]
}
},
"a": "EMPTY_ARRAY",
"b": "EMPTY_OBJECT",
"c": "EMPTY_STRING"
}
json_model_element = JsonModelElement(self.id_, key_parser_dict)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {}, "c": ""}'
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"menu": {"id": {\n}, "value": [\n], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": ' \
b'null, "onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {}, "c": ""}'
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
JsonModelElement(self.id_, {"a": "EMPTY_ARRAY"})
JsonModelElement(self.id_, {"a": "EMPTY_OBJECT"})
JsonModelElement(self.id_, {"a": "EMPTY_STRING"})
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": ["a"], "b": {}, "c": ""}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {"a": "a"}, "c": ""}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {}, "c": "ab"}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": []}}, "a": [], "b": {}, "c": ""}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated with all characters."""
json_model_element = JsonModelElement(self.id_, self.key_parser_dict)
data = self.single_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.multi_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.everything_new_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
# Test if keys differently ordered than in the key_parser_dict are parsed properly.
data = self.single_line_different_order_with_optional_key_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.single_line_empty_array
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_allow_all)
data = self.single_line_different_order_with_optional_key_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_array)
data = self.single_line_json_array
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_escaped)
data = self.single_line_escaped_json.decode("unicode-escape").encode()
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(
self.id_, self.key_parser_dict_array_of_arrays)
data = self.array_of_arrays
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test5get_match_element_with_allow_all(self):
"""Test a simplified key_parser_dict with ALLOW_ALL."""
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_allow_all)
data = self.single_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.multi_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.everything_new_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test17time_change_cest_cet(self):
"""Check if the time change from CET to CEST and vice versa work as expected."""
data = b"24.03.2018 11:40:00 CET: it still works"
date = b"24.03.2018 11:40:00 CET"
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m.%Y %H:%M:%S%z",
timezone.utc)
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1521888000, None)
# make sure format changes with longer format specifiers also work
data = b"25.03.2018 11:40:00 CEST: it still works"
date = b"25.03.2018 11:40:00 CEST"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1521970800, None)
data = b"27.10.2018 11:40:00 CEST: it still works"
date = b"27.10.2018 11:40:00 CEST"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1540633200, None)
data = b"28.10.2018 11:40:00 CET: it still works"
date = b"28.10.2018 11:40:00 CET"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1540723200, None)
data = b"27.10.2018 11:40:00 EST: it still works"
date = b"27.10.2018 11:40:00 EST"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1540658400, None)
data = b"27.10.2018 11:40:00 PDT: it still works"
date = b"27.10.2018 11:40:00 PDT"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1540665600, None)
data = b"27.10.2018 11:40:00 GMT: it still works"
date = b"27.10.2018 11:40:00 GMT"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1540640400, None)
def test16max_time_jump_seconds_exceeded(self):
"""
Test if the start_year is not updated, when the next date exceeds the max_time_jump_seconds.
A time inconsistency warning must occur.
"""
log_stream = StringIO()
logging.basicConfig(stream=log_stream, level=logging.INFO)
max_time_jump_seconds = 86400
start_year = 2020
date_time_model_element = DateTimeModelElement(
self.id_,
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=start_year,
max_time_jump_seconds=max_time_jump_seconds)
data = b"31.12 23:59:00: it still works"
date = b"31.12 23:59:00"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1609459140, None)
self.assertEqual(date_time_model_element.start_year, start_year)
data = b"01.01 23:59:01: it still works"
date = b"01.01 23:59:01"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1577923141, None)
self.assertEqual(date_time_model_element.start_year, start_year)
self.assertIn(
"WARNING:DEBUG:DateTimeModelElement time inconsistencies parsing b'01.01 23:59:01', expecting value around "
"1609459140. Check your settings!", log_stream.getvalue())
date_time_model_element = DateTimeModelElement(
self.id_,
b"%d.%m %H:%M:%S",
timezone.utc,
start_year=start_year,
max_time_jump_seconds=max_time_jump_seconds)
data = b"05.03 06:29:07: it still works"
date = b"05.03 06:29:07"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1583389747, None)
self.assertEqual(date_time_model_element.start_year, start_year)
self.assertIn(
"WARNING:DEBUG:DateTimeModelElement time inconsistencies parsing b'01.01 23:59:01', expecting value around "
"1609459140. Check your settings!", log_stream.getvalue())
data = b"29.02 07:24:02: it still works"
date = b"29.02 07:24:02"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1582961042, None)
self.assertEqual(date_time_model_element.start_year, start_year)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
initialize_loggers(self.aminer_config,
getpwnam("aminer").pw_uid,
getgrnam("aminer").gr_gid)
def test3get_match_element_with_different_date_formats(self):
"""Test if different date_formats can be used to match data."""
# test normal date
data = b"07.02.2019 11:40:00: it still works"
date = b"07.02.2019 11:40:00"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m.%Y %H:%M:%S",
timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549539600, None)
# test leap year date
data = b"29.02.2020 11:40:00: it still works"
date = b"29.02.2020 11:40:00"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1582976400, None)
# test normal date with T
data = b"07.02.2019T11:40:00: it still works"
date = b"07.02.2019T11:40:00"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m.%YT%H:%M:%S",
timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549539600, None)
# test normal date with fractions
data = b"07.02.2019 11:40:00.123456: it still works"
date = b"07.02.2019 11:40:00.123456"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(
self.id_, b"%d.%m.%Y %H:%M:%S.%f", timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date,
1549539600.123456, None)
# test normal date with z
data = b"07.02.2019 11:40:00+0000: it still works"
date = b"07.02.2019 11:40:00+0000"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m.%Y %H:%M:%S%z",
timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549539600, None)
# test normal date with z
data = b"07.02.2019 11:40:00 UTC: it still works"
date = b"07.02.2019 11:40:00 UTC"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549539600, None)
# test normal date with z
data = b"07.02.2019 11:40:00 GMT: it still works"
date = b"07.02.2019 11:40:00 GMT"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549539600, None)
# test normal date with z
data = b"07.02.2019 11:40:00 UTC+01: it still works"
date = b"07.02.2019 11:40:00 UTC+01"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549536000, None)
# wrong timezone identifiers for offsets
data = b"07.02.2019 11:40:00 CET+01: it still works"
date = b"07.02.2019 11:40:00 CET"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549536000, None)
data = b"07.02.2019 11:40:00: it still works"
date = b"07.02.2019 11:40:00"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549539600, None)
# test with only date defined
data = b"07.02.2019: it still works"
date = b"07.02.2019"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(self.id_, b"%d.%m.%Y",
timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1549497600, None)
# test with only time defined. Here obviously the seconds can not be tested.
data = b"11:40:23: it still works"
date = b"11:40:23"
match_context = DummyMatchContext(data)
date_time_model_element = DateTimeModelElement(self.id_, b"%H:%M:%S",
timezone.utc)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date,
match_element.match_object, None)
self.assertEqual(match_element.match_string, b"11:40:23")
self.assertEqual(match_context.match_string, b"11:40:23")
def test17time_change_cest_cet(self):
"""Check if the time change from CET to CEST and vice versa work as expected."""
multi_locale_dtme = MultiLocaleDateTimeModelElement(
self.id_, [(b"%d.%m.%Y %H:%M:%S%z", None, None)])
data = b"24.03.2018 11:40:00 CET: it still works"
date = b"24.03.2018 11:40:00 CET"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1521888000, None)
data = b"25.03.2018 11:40:00 CEST: it still works"
date = b"25.03.2018 11:40:00 CEST"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1521970800, None)
data = b"27.10.2018 11:40:00 CEST: it still works"
date = b"27.10.2018 11:40:00 CEST"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1540633200, None)
data = b"28.10.2018 11:40:00 CET: it still works"
date = b"28.10.2018 11:40:00 CET"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1540723200, None)
data = b"27.10.2018 11:40:00 EST: it still works"
date = b"27.10.2018 11:40:00 EST"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1540658400, None)
data = b"27.10.2018 11:40:00 PDT: it still works"
date = b"27.10.2018 11:40:00 PDT"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1540665600, None)
data = b"27.10.2018 11:40:00 GMT: it still works"
date = b"27.10.2018 11:40:00 GMT"
match_context = DummyMatchContext(data)
match_element = multi_locale_dtme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_ + "/format0", self.path, date,
1540640400, None)
def test10get_match_element_float_exponents(self):
"""
Parse float values with exponents.
The principle of only testing dummy classes can not be applied here, as the functionality between the JsonModelElement and
DecimalFloatValueModelElement must be tested directly.
"""
json_model_element = JsonModelElement(
self.id_, {
"a":
DecimalFloatValueModelElement(
self.id_,
exponent_type=DecimalFloatValueModelElement.
EXP_TYPE_OPTIONAL),
"b":
DecimalFloatValueModelElement(
self.id_,
exponent_type=DecimalFloatValueModelElement.
EXP_TYPE_OPTIONAL)
})
def format_float(val):
"""
This function formats the float-value and parses the sign and the exponent
"""
exp = None
if "e" in val:
exp = "e"
elif "E" in val:
exp = "E"
if "+" in val:
sign = "+"
else:
sign = "-"
if exp is not None:
pos_point = val.find(exp)
if "." in val:
pos_point = val.find(".")
if len(val) - val.find(sign) <= 2:
result = format(float(val),
"1.%dE" % (val.find(exp) - pos_point))[:-2]
result += format(float(val),
"1.%dE" % (val.find(exp) - pos_point))[-1]
return result
return format(float(val),
"1.%dE" % (val.find(exp) - pos_point))
return float(val)
data = b'{"a": 111.1, "b": 111.1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 1E-01, "b": 111.1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 111.1, "b": 1E-1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 1E-1, "b": 1E-1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test4get_match_element_min_max_repeats(self):
"""This test case verifies the functionality of setting the minimal and maximal repeats."""
fixed_dme = DummyFixedDataModelElement(self.fixed_id, self.fixed_data)
repeated_dme = RepeatedElementDataModelElement(self.id_,
fixed_dme,
min_repeat=2,
max_repeat=5)
same_min_max_repeat_dme = RepeatedElementDataModelElement(self.id_,
fixed_dme,
min_repeat=3,
max_repeat=3)
data = b"other data"
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_no_match_results(data, match_element, match_context)
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"fixed data "
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_no_match_results(data, match_element, match_context)
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"fixed data fixed data "
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
fixed_dme.get_match_element("%s/%s/0" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/1" % (self.path, self.id_),
DummyMatchContext(data))
])
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"fixed data fixed data fixed data "
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
fixed_dme.get_match_element("%s/%s/0" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/1" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/2" % (self.path, self.id_),
DummyMatchContext(data))
])
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
fixed_dme.get_match_element("%s/%s/0" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/1" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/2" % (self.path, self.id_),
DummyMatchContext(data))
])
data = b"fixed data fixed data fixed data fixed data "
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
fixed_dme.get_match_element("%s/%s/0" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/1" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/2" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/3" % (self.path, self.id_),
DummyMatchContext(data))
])
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"fixed data fixed data fixed data fixed data fixed data "
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
fixed_dme.get_match_element("%s/%s/0" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/1" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/2" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/3" % (self.path, self.id_),
DummyMatchContext(data)),
fixed_dme.get_match_element("%s/%s/4" % (self.path, self.id_),
DummyMatchContext(data))
])
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"fixed data fixed data fixed data fixed data fixed data fixed data "
match_context = DummyMatchContext(data)
match_element = repeated_dme.get_match_element(self.path,
match_context)
self.compare_no_match_results(data, match_element, match_context)
match_context = DummyMatchContext(data)
match_element = same_min_max_repeat_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test3get_match_element_with_different_date_formats(self):
"""Test if different date_formats can be used to match data."""
tz_gmt10 = pytz.timezone("Etc/GMT+10")
en_gb_utf8 = "en_GB.utf8"
en_us_utf8 = "en_US.utf8"
de_at_utf8 = "de_AT.utf8"
multi_locale_dtme = MultiLocaleDateTimeModelElement("path", [
(b"%d.%m.%Y %H:%M:%S.%f", None, None), (b"%d.%m.%Y %H:%M:%S%z", None, None), (b"%d.%m.%Y %H:%M:%S", None, None),
(b"%d.%m.%YT%H:%M:%S", None, None), (b"%d.%m.%Y", None, None), (b"%H:%M:%S:%f", None, de_at_utf8),
(b"%H:%M:%S", None, None), (b"%b %d", tz_gmt10, de_at_utf8), (b"%d %b %Y", None, en_gb_utf8),
(b"%dth %b %Y", None, en_gb_utf8), (b"%d/%m/%Y", None, en_gb_utf8), (b"%m-%d-%Y", None, en_us_utf8),
(b"%d.%m. %H:%M:%S:%f", None, de_at_utf8)])
# test normal date
match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
# test leap year date
match_context = DummyMatchContext(b"29.02.2020 11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"29.02.2020 11:40:00")
self.assertEqual(match_element.match_object, 1582976400)
self.assertEqual(match_context.match_string, b"29.02.2020 11:40:00")
# test normal date with T
match_context = DummyMatchContext(b"07.02.2019T11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019T11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019T11:40:00")
# test normal date with fractions
match_context = DummyMatchContext(b"07.02.2019 11:40:00.123456: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00.123456")
self.assertEqual(match_element.match_object, 1549539600.123456)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00.123456")
# test normal date with z
match_context = DummyMatchContext(b"07.02.2019 11:40:00+0000: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00+0000")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00+0000")
match_context = DummyMatchContext(b"07.02.2019 11:40:00: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019 11:40:00")
self.assertEqual(match_element.match_object, 1549539600)
self.assertEqual(match_context.match_string, b"07.02.2019 11:40:00")
# test with only date defined
match_context = DummyMatchContext(b"07.02.2019: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"07.02.2019")
self.assertEqual(match_element.match_object, 1549497600)
self.assertEqual(match_context.match_string, b"07.02.2019")
# test with only time defined. Here obviously the seconds can not be tested.
match_context = DummyMatchContext(b"11:40:23: it still works")
match_element = multi_locale_dtme.get_match_element("match1", match_context)
self.assertEqual(match_element.match_string, b"11:40:23")
self.assertEqual(match_context.match_string, b"11:40:23")
string = b"Feb 25"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
date = datetime(datetime.now().year, 2, 25, tzinfo=tz_gmt10)
# total_seconds should be in UTC, so the timezones are parsed out.
total_seconds = (date - datetime(1970, 1, 1, tzinfo=tz_gmt10)).days * 86400 - date.utcoffset().total_seconds()
self.assertEqual(match_element.match_string, b"Feb 25")
self.assertEqual(match_element.match_object, total_seconds)
self.assertEqual(match_context.match_string, b"Feb 25")
# British date
string = b"13 Apr 2019"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
self.assertEqual(match_element.match_string, b"13 Apr 2019")
self.assertEqual(match_element.match_object, 1555113600)
self.assertEqual(match_context.match_string, b"13 Apr 2019")
# British date 2
string = b"13th Apr 2019"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
self.assertEqual(match_element.match_string, b"13th Apr 2019")
self.assertEqual(match_element.match_object, 1555113600)
self.assertEqual(match_context.match_string, b"13th Apr 2019")
# British date 3
string = b"13/04/2019"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
self.assertEqual(match_element.match_string, b"13/04/2019")
self.assertEqual(match_element.match_object, 1555113600)
self.assertEqual(match_context.match_string, b"13/04/2019")
# US date
string = b"04-13-2019"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
self.assertEqual(match_element.match_string, b"04-13-2019")
self.assertEqual(match_element.match_object, 1555113600)
self.assertEqual(match_context.match_string, b"04-13-2019")
# Austrian date no year - year should already be learnt.
# start year has to be 2021, because all other formats have defined years.
string = b"13.04. 15:12:54:201"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
self.assertEqual(match_element.match_string, b"13.04. 15:12:54:201")
self.assertEqual(match_element.match_object, 1618326774.201)
self.assertEqual(match_context.match_string, b"13.04. 15:12:54:201")
multi_locale_dtme.latest_parsed_timestamp = None
# Austrian time no date
string = b"15:12:54:201"
match_context = DummyMatchContext(string)
match_element = multi_locale_dtme.get_match_element("match", match_context)
date = datetime(datetime.now().year, datetime.now().month, datetime.now().day, 15, 12, 54, 201, tzinfo=timezone.utc)
# total_seconds should be in UTC, so the timezones are parsed out.
delta = (date - datetime(1970, 1, 1, tzinfo=date.tzinfo))
total_seconds = delta.days * 86400 + delta.seconds + delta.microseconds / 1000
self.assertEqual(match_element.match_string, b"15:12:54:201")
self.assertEqual(match_element.match_object, total_seconds)
self.assertEqual(match_context.match_string, b"15:12:54:201")
def test6get_match_element_with_different_time_zones(self):
"""Test if different time_zones work with the DateTimeModelElement."""
date_time_model_element = DateTimeModelElement(self.id_,
b"%d.%m.%Y %H:%M:%S%z",
timezone.utc)
data = b"07.02.2018 11:40:00 UTC-1200: it still works"
date = b"07.02.2018 11:40:00 UTC-1200"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518046800, None)
data = b"07.02.2018 11:40:00 UTC-12: it still works"
date = b"07.02.2018 11:40:00 UTC-12"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518046800, None)
data = b"07.02.2018 11:40:00 UTC-5: it still works"
date = b"07.02.2018 11:40:00 UTC-5"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518021600, None)
data = b"07.02.2018 11:40:00 UTC-0500: it still works"
date = b"07.02.2018 11:40:00 UTC-0500"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518021600, None)
data = b"07.02.2018 11:40:00-05:00: it still works"
date = b"07.02.2018 11:40:00-05:00"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518021600, None)
data = b"07.02.2018 11:40:00 UTC+0000: it still works"
date = b"07.02.2018 11:40:00 UTC+0000"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518003600, None)
data = b"07.02.2018 11:40:00 UTC+0100: it still works"
date = b"07.02.2018 11:40:00 UTC+0100"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518000000, None)
data = b"07.02.2018 11:40:00+01:00: it still works"
date = b"07.02.2018 11:40:00+01:00"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1518000000, None)
data = b"07.02.2018 11:40:00 UTC+1400: it still works"
date = b"07.02.2018 11:40:00 UTC+1400"
match_context = DummyMatchContext(data)
match_element = date_time_model_element.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, date, 1517953200, None)
def test4get_match_element_with_optional_key(self):
"""Validate optional keys with the optional_key_prefix."""
json_model_element = JsonModelElement(self.id_, self.key_parser_dict)
data = self.single_line_with_optional_key_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(json.loads(match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
json_model_element = JsonModelElement(self.id_, self.empty_key_parser_dict)
data = b"{}"
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(json.loads(data)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
json_model_element = JsonModelElement(self.id_, self.empty_key_parser_dict)
data = b'{"key": "value"}'
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(json.loads(data)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
json_model_element = JsonModelElement(self.id_, self.empty_key_parser_dict)
data = b'{"key": "another not matching value"}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)