def test2optional_key_prefix(self):
"""Validate optional keys with the optional_key_prefix."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_with_optional_key_json))
self.assertEqual(match.match_object,
json.loads(self.single_line_with_optional_key_json))
def test5get_match_element_with_allow_all(self):
"""Test a simplified key_parser_dict with ALLOW_ALL."""
json_model_element = JsonModelElement(self.id_, self.key_parser_dict_allow_all)
data = self.single_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
data = self.multi_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
data = self.everything_new_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
def test2get_child_elements(self):
"""Test if get_child_elements returns None."""
k = self.key_parser_dict
json_me = JsonModelElement(self.id_, self.key_parser_dict)
self.assertEqual(json_me.get_child_elements(), [
k["menu"]["id"], k["menu"]["value"], [
k["menu"]["popup"]["menuitem"][0]["value"], k["menu"]["popup"]["menuitem"][0]["onclick"],
k["menu"]["popup"]["menuitem"][0]["optional_key_clickable"], k["menu"]["popup"]["menuitem"][1]["value"],
k["menu"]["popup"]["menuitem"][1]["onclick"], k["menu"]["popup"]["menuitem"][1]["clickable"]]])
def test7get_match_element_with_umlaut(self):
"""Test if ä ö ü are used correctly."""
key_parser_dict = {"works": DummyFixedDataModelElement("abc", "a ä ü ö z".encode("utf-8"))}
data = """{
"works": "a ä ü ö z"
}""".encode("utf-8")
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, str(value).encode(), value, match_element.children)
def test1get_valid_match_elements(self):
"""Get valid json elements with different formats."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_json))
self.assertEqual(match.match_object, json.loads(self.single_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.multi_line_json))
self.assertEqual(match.match_object, json.loads(self.multi_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.everything_new_line_json))
self.assertEqual(match.match_object,
json.loads(self.everything_new_line_json))
def test4allow_all_dict(self):
"""Test a simplified key_parser_dict with ALLOW_ALL."""
json_model_element = JsonModelElement('json',
self.key_parser_dict_allow_all)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_json))
self.assertEqual(match.match_object, json.loads(self.single_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.multi_line_json))
self.assertEqual(match.match_object, json.loads(self.multi_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.everything_new_line_json))
self.assertEqual(match.match_object,
json.loads(self.everything_new_line_json))
def test6get_match_element_null_value(self):
"""Test if null values are parsed to "null"."""
key_parser_dict = {
"works":
DummyFirstMatchModelElement("id", [
DummyFixedDataModelElement("abc", b"abc"),
DummyFixedDataModelElement("123", b"123")
]),
"null":
DummyFirstMatchModelElement("wordlist", [
DummyFixedDataModelElement("allowed", b"allowed value"),
DummyFixedDataModelElement("problem", b"null")
])
}
data1 = b"""{
"works": "abc",
"null": "allowed value"
}"""
data2 = b"""{
"works": "123",
"null": null
}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
data = data1
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = data2
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test17get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = JsonModelElement(self.id_, self.key_parser_dict)
data = b"abcdefghijklmnopqrstuvwxyz.!?"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test11get_match_element_allow_all_fields(self):
"""Parse matching substring from MatchContext using the allow_all_fields parameter."""
json_model_element = JsonModelElement(
self.id_,
self.key_parser_dict_allow_all_fields,
allow_all_fields=True)
data = self.single_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(
self.id_,
self.key_parser_dict_allow_all_fields,
allow_all_fields=False)
data = self.single_line_json
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test4get_match_element_with_optional_key(self):
"""Validate optional keys with the optional_key_prefix."""
json_model_element = JsonModelElement(self.id_, self.key_parser_dict)
data = self.single_line_with_optional_key_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.empty_key_parser_dict)
data = b"{}"
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(data)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test6null_value(self):
"""Test if null values are parsed to "null"."""
key_parser_dict = {
"works":
VariableByteDataModelElement("id", b"abc123"),
"problem":
FixedWordlistDataModelElement("wordlist",
[b"allowed value", b"null"])
}
data1 = b"""{
"works": "abc",
"problem": "allowed value"
}"""
data2 = b"""{
"works": "123",
"problem": null
}"""
json_model_element = JsonModelElement('json', key_parser_dict)
self.assertIsNotNone(
json_model_element.get_match_element('match', MatchContext(data1)))
self.assertIsNotNone(
json_model_element.get_match_element('match', MatchContext(data2)))
def test5different_order_keys(self):
"""Test if keys differently ordered than in the key_parser_dict are parsed properly."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match',
MatchContext(
self.single_line_different_order_with_optional_key_json))
self.assertEqual(
match.match_object,
json.loads(
self.single_line_different_order_with_optional_key_json))
json_model_element = JsonModelElement('json',
self.key_parser_dict_allow_all)
match = json_model_element.get_match_element(
'match',
MatchContext(
self.single_line_different_order_with_optional_key_json))
self.assertEqual(
match.match_object,
json.loads(
self.single_line_different_order_with_optional_key_json))
def test10get_match_element_float_exponents(self):
"""
Parse float values with exponents.
The principle of only testing dummy classes can not be applied here, as the functionality between the JsonModelElement and
DecimalFloatValueModelElement must be tested directly.
"""
json_model_element = JsonModelElement(
self.id_, {
"a":
DecimalFloatValueModelElement(
self.id_,
exponent_type=DecimalFloatValueModelElement.
EXP_TYPE_OPTIONAL),
"b":
DecimalFloatValueModelElement(
self.id_,
exponent_type=DecimalFloatValueModelElement.
EXP_TYPE_OPTIONAL)
})
def format_float(val):
"""
This function formats the float-value and parses the sign and the exponent
"""
exp = None
if "e" in val:
exp = "e"
elif "E" in val:
exp = "E"
if "+" in val:
sign = "+"
else:
sign = "-"
if exp is not None:
pos_point = val.find(exp)
if "." in val:
pos_point = val.find(".")
if len(val) - val.find(sign) <= 2:
result = format(float(val),
"1.%dE" % (val.find(exp) - pos_point))[:-2]
result += format(float(val),
"1.%dE" % (val.find(exp) - pos_point))[-1]
return result
return format(float(val),
"1.%dE" % (val.find(exp) - pos_point))
return float(val)
data = b'{"a": 111.1, "b": 111.1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 1E-01, "b": 111.1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 111.1, "b": 1E-1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"a": 1E-1, "b": 1E-1}'
value = json.loads(data, parse_float=format_float)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test3missing_key(self):
"""Check if no match is returned if a key is missing."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_missing_key_json))
self.assertEqual(match, None)
def get_model():
"""Return the model."""
name_alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
alphabet = "!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[] °§ß–\n".encode(
)
alphabet_with_newline = alphabet + b"\n"
filename_alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 ._-/"
path = VariableByteDataModelElement("path", filename_alphabet)
apostrophe = FixedDataModelElement("apostrophe", b"'")
repeated_path = RepeatedElementDataModelElement(
"repeated",
SequenceModelElement("sequence", [
apostrophe, path, apostrophe,
OptionalMatchModelElement("optional",
FixedDataModelElement("comma", b","))
]))
rule = SequenceModelElement("rule", [
FixedDataModelElement("open_bracket", b"("), repeated_path,
FixedWordlistDataModelElement("close_bracket", [b")->(", b")<-("]),
repeated_path,
FixedDataModelElement("close_bracket", b")")
])
expected = SequenceModelElement("expected", [
DecimalIntegerValueModelElement("actual"),
FixedDataModelElement("slash", b"/"),
DecimalIntegerValueModelElement("expected")
])
observed = SequenceModelElement("observed", [
DecimalIntegerValueModelElement("actual"),
FixedDataModelElement("slash", b"/"),
DecimalIntegerValueModelElement("expected")
])
has_outlier_bins_flag = FixedWordlistDataModelElement(
"has_outlier_bins_flag", [b"true", b"false"])
model = JsonModelElement(
"aminer", {
"_AnalysisComponent": {
"AnalysisComponentIdentifier":
FirstMatchModelElement("first", [
DecimalIntegerValueModelElement("component_id"),
FixedDataModelElement("null", b"null")
]),
"AnalysisComponentType":
VariableByteDataModelElement("component_type", name_alphabet),
"AnalysisComponentName":
VariableByteDataModelElement("component_name", alphabet),
"Message":
VariableByteDataModelElement("message", alphabet),
"_PersistenceFileName":
VariableByteDataModelElement("persistence_file_name",
filename_alphabet),
"_TrainingMode":
FixedWordlistDataModelElement("training_mode",
[b"true", b"false"]),
"_AffectedLogAtomPaths": [
VariableByteDataModelElement("affected_log_atom_paths",
alphabet)
],
"_AffectedLogAtomValues": [
VariableByteDataModelElement("affected_log_atom_value",
alphabet)
],
"_Metadata": {
"TimeFirstOccurrence":
DecimalFloatValueModelElement("time_first_occurrence"),
"TimeLastOccurrence":
DecimalFloatValueModelElement("time_last_occurrence"),
"NumberOfOccurrences":
DecimalIntegerValueModelElement("number_of_occurrences")
},
"_ParsedLogAtom": {
"ALLOW_ALL_KEYS":
VariableByteDataModelElement("allow_all_keys", alphabet)
},
"_FeatureList": [{
"Rule": {
"type":
VariableByteDataModelElement("type", name_alphabet),
"path":
path,
"_value":
VariableByteDataModelElement("value", alphabet),
"match_action":
VariableByteDataModelElement("match_action",
filename_alphabet),
"log_total":
DecimalIntegerValueModelElement("log_total"),
"log_success":
DecimalIntegerValueModelElement("log_success")
},
"Index":
DecimalIntegerValueModelElement("index"),
"CreationTime":
DecimalFloatValueModelElement("creation_time"),
"LastTriggerTime":
DecimalFloatValueModelElement("last_trigger_time"),
"TriggerCount":
DecimalIntegerValueModelElement("trigger_count")
}],
"_AnalysisStatus":
VariableByteDataModelElement("analysis_status", alphabet),
"_TotalRecords":
DecimalIntegerValueModelElement("total_records"),
"_HistogramData": [{
"TotalElements":
DecimalIntegerValueModelElement("total_elements"),
"BinnedElements":
DecimalIntegerValueModelElement("binned_elements"),
"HasOutlierBinsFlag":
has_outlier_bins_flag,
"Bins": {
"ALLOW_ALL_KEYS":
DecimalIntegerValueModelElement("bin")
},
"BinDefinition": {
"Type":
FixedWordlistDataModelElement("type", [
b"ModuloTimeBinDefinition",
b"LinearNumericBinDefinition"
]),
"LowerLimit":
DecimalIntegerValueModelElement("lower_limit"),
"BinSize":
DecimalIntegerValueModelElement("bin_size"),
"BinCount":
DecimalIntegerValueModelElement("bin_count"),
"OutlierBinsFlag":
has_outlier_bins_flag,
"BinNames": [
SequenceModelElement("bin_names", [
FirstMatchModelElement("first", [
SequenceModelElement(
"lower", [
FixedDataModelElement(
"open_bracket", b"["),
DecimalIntegerValueModelElement(
"value")
]),
FixedDataModelElement("dots", b"...")
]),
FixedDataModelElement("hyphen", b"-"),
FirstMatchModelElement("first", [
SequenceModelElement(
"upper", [
DecimalIntegerValueModelElement(
"value"),
FixedDataModelElement(
"close_bracket", b"]")
]),
FixedDataModelElement("dots", b"...")
]),
])
],
"ExpectedBinRatio":
DecimalFloatValueModelElement("expected_bin_ratio"),
"_ModuloValue":
DecimalIntegerValueModelElement("modulo_value"),
"_TimeUnit":
DecimalIntegerValueModelElement("time_unit")
},
"PropertyPath":
VariableByteDataModelElement("property_path",
filename_alphabet),
}],
"_ReportInterval":
DecimalIntegerValueModelElement("report_interval"),
"_ResetAfterReportFlag":
FixedWordlistDataModelElement("reset_after_report_flag",
[b"true", b"false"]),
"_MissingPaths":
[VariableByteDataModelElement("missing_paths", alphabet)],
"_AnomalyScores": [{
"Path": path,
"AnalysisData": {
"New": {
"N": DecimalIntegerValueModelElement("n"),
"Avg": DecimalFloatValueModelElement("avg"),
"Var": DecimalFloatValueModelElement("var")
},
"Old": {
"N": DecimalIntegerValueModelElement("n"),
"Avg": DecimalFloatValueModelElement("avg"),
"Var": DecimalFloatValueModelElement("var")
}
}
}],
"_MinBinElements":
DecimalIntegerValueModelElement("min_bin_elements"),
"_MinBinTime":
DecimalIntegerValueModelElement("min_bin_time"),
"_DebugMode":
FixedWordlistDataModelElement("debug_mode",
[b"true", b"false"]),
"_Rule": {
"RuleId":
VariableByteDataModelElement("id", filename_alphabet),
"MinTimeDelta":
DecimalIntegerValueModelElement("min_time_delta"),
"MaxTimeDelta":
DecimalIntegerValueModelElement("max_time_delta"),
"MaxArtefactsAForSingleB":
DecimalIntegerValueModelElement(
"max_artefacts_a_for_single_b"),
"ArtefactMatchParameters": [path],
"HistoryAEvents":
"EMPTY_ARRAY",
"HistoryBEvents":
"EMPTY_ARRAY",
"LastTimestampSeen":
DecimalFloatValueModelElement("last_timestamp_seen"),
"correlation_history": {
"MaxItems":
DecimalIntegerValueModelElement("max_items"),
"History":
[VariableByteDataModelElement("value", alphabet)]
}
},
"_CheckResult":
[VariableByteDataModelElement("value", alphabet_with_newline)],
"_NewestTimestamp":
DecimalFloatValueModelElement("newest_timestamp")
},
"_TotalRecords":
DecimalIntegerValueModelElement("total_records"),
"_TypeInfo":
"ALLOW_ALL",
"_RuleInfo": {
"Rule": rule,
"Expected": expected,
"Observed": observed
},
"_LogData": {
"RawLogData":
[VariableByteDataModelElement("raw_log_data", alphabet)],
"Timestamps": [DecimalFloatValueModelElement("timestamp")],
"DetectionTimestamp":
DecimalFloatValueModelElement("detection_timestamp"),
"LogLinesCount":
DecimalIntegerValueModelElement("lines_count"),
"_AnnotatedMatchElement":
VariableByteDataModelElement("annotated_match_element",
alphabet_with_newline),
},
"_StatusInfo": {
"ALLOW_ALL_KEYS": {
"CurrentProcessedLines":
DecimalIntegerValueModelElement("current_processed_lines"),
"TotalProcessedLines":
DecimalIntegerValueModelElement("total_processed_lines")
}
},
"_FromTime":
DecimalFloatValueModelElement("from_time"),
"_ToTime":
DecimalFloatValueModelElement("to_time"),
"_DebugLog": [
OptionalMatchModelElement(
"optional",
VariableByteDataModelElement("debug_log", alphabet))
]
}, "_")
return model
def test12get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
json_model_element = JsonModelElement(self.id_, self.key_parser_dict)
# missing key
data = self.single_line_missing_key_json
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# object instead of array
data = self.single_line_object_instead_of_array
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# invalid json
data = self.single_line_invalid_json
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# child not matching
data = self.single_line_no_match_json
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
# all keys missing
data = b"{}"
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
json_model_element = JsonModelElement(self.id_,
self.empty_key_parser_dict)
data = b"[]"
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"{[]}"
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"key": []}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test1get_id(self):
"""Test if get_id works properly."""
json_me = JsonModelElement(self.id_, self.key_parser_dict)
self.assertEqual(json_me.get_id(), self.id_)
def test9get_match_element_empty_array_empty_object_null(self):
"""Test if the keywords EMPTY_ARRAY, EMPTY_OBJECT, EMPTY_STRING and None (null) work properly."""
key_parser_dict = {
"menu": {
"id": "EMPTY_OBJECT",
"value": "EMPTY_ARRAY",
"popup": {
"menuitem": [{
"value":
DummyFixedDataModelElement("null", b"null"),
"onclick":
DummyFirstMatchModelElement("buttonOnclick", [
DummyFixedDataModelElement("create_new_doc",
b"CreateNewDoc()"),
DummyFixedDataModelElement("open_doc",
b"OpenDoc()"),
DummyFixedDataModelElement("close_doc",
b"CloseDoc()")
]),
"optional_key_clickable":
DummyFirstMatchModelElement("clickable", [
DummyFixedDataModelElement("true", b"true"),
DummyFixedDataModelElement("false", b"false")
])
}]
}
},
"a": "EMPTY_ARRAY",
"b": "EMPTY_OBJECT",
"c": "EMPTY_STRING"
}
json_model_element = JsonModelElement(self.id_, key_parser_dict)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {}, "c": ""}'
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b'{"menu": {"id": {\n}, "value": [\n], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": ' \
b'null, "onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {}, "c": ""}'
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
JsonModelElement(self.id_, {"a": "EMPTY_ARRAY"})
JsonModelElement(self.id_, {"a": "EMPTY_OBJECT"})
JsonModelElement(self.id_, {"a": "EMPTY_STRING"})
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": ["a"], "b": {}, "c": ""}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {"a": "a"}, "c": ""}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": [{"value": null, "onclick": "CreateNewDoc()"}, {"value": null, ' \
b'"onclick": "OpenDoc()"}, {"value": null, "onclick": "CloseDoc()"}]}}, "a": [], "b": {}, "c": "ab"}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b'{"menu": {"id": {}, "value": [], "popup": {"menuitem": []}}, "a": [], "b": {}, "c": ""}'
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test8get_match_element_same_value_as_key(self):
"""Test if object with the same key-value pairs are parsed correctly."""
key_parser_dict = {
"abc":
DummyFirstMatchModelElement("first", [
DummyFixedDataModelElement("abc", b"abc"),
DummyFixedDataModelElement("abc", b"ab"),
DummyFixedDataModelElement("abc", b"bc"),
DummyFixedDataModelElement("abc", b"ba"),
DummyFixedDataModelElement("abc", b"b"),
DummyFixedDataModelElement("abc", b"d")
])
}
data = b"""{"abc":"abc"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"ab"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"bc"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"b"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"d"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = b"""{"abc":"ba"}"""
json_model_element = JsonModelElement(self.id_, key_parser_dict)
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated with all characters."""
json_model_element = JsonModelElement(self.id_, self.key_parser_dict)
data = self.single_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.multi_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.everything_new_line_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
# Test if keys differently ordered than in the key_parser_dict are parsed properly.
data = self.single_line_different_order_with_optional_key_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
data = self.single_line_empty_array
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_allow_all)
data = self.single_line_different_order_with_optional_key_json
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_array)
data = self.single_line_json_array
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(self.id_,
self.key_parser_dict_escaped)
data = self.single_line_escaped_json.decode("unicode-escape").encode()
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(value).encode()
match_context.match_data = data[len(match_context.match_string):]
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
json_model_element = JsonModelElement(
self.id_, self.key_parser_dict_array_of_arrays)
data = self.array_of_arrays
value = json.loads(data)
match_context = DummyMatchContext(data)
match_element = json_model_element.get_match_element(
self.path, match_context)
match_context.match_string = str(json.loads(
match_context.match_string)).encode()
self.compare_match_results(data, match_element, match_context,
self.id_, self.path,
str(value).encode(), value,
match_element.children)
