def test4append_values_bytestring(self):
"""
This unittest checks the append_values method with raw_match_object being a bytestring.
This should trigger a ValueError and append the match_string.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
# initialize all values.
t = time.time()
log_atom = LogAtom(
b'This is a string',
ParserMatch(
MatchElement('path', b'This is a string', b'This is a string',
None)), t, self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[['This is a string']]])
log_atom = LogAtom(
b'24.05.',
ParserMatch(MatchElement('path', b'24.05.', b'24.05.', None)), t,
self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[['24.05.']]])
def test1_child_elements_with_no_path(self):
"""This test case checks, whether a Exception is raised, when the path is None or empty and children are passed."""
self.assertRaises(Exception, MatchElement, None, b'matchString', b'matchString', (
MatchElement('child', b'childMatchString', b'childMatchString', [])))
self.assertRaises(Exception, MatchElement, '', b'matchString', b'matchString', (
MatchElement('child', b'childMatchString', b'childMatchString', [])))
def test5check_value_reduction(self):
"""This unittest checks the functionality of reducing the values when the maxNumVals threshold is reached."""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
t = time.time()
val_list = [[[]]]
for i in range(1, event_type_detector.max_num_vals + 1, 1):
log_atom = LogAtom(
str(i).encode(),
ParserMatch(MatchElement('path',
str(i).encode(), i, None)), t,
self.__class__.__name__)
val_list[0][0].append(float(i))
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.values, val_list)
i += 1
log_atom = LogAtom(
str(i).encode(),
ParserMatch(MatchElement('path',
str(i).encode(), i, None)), t,
self.__class__.__name__)
val_list[0][0].append(float(i))
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(
event_type_detector.values,
[[val_list[0][0][-event_type_detector.min_num_vals:]]])
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated with all characters."""
element_value_branch_me = ElementValueBranchModelElement(
self.id_, self.value_model, None, {
"path: ": self.path_me,
"data: ": self.data_me
})
data = b"path: /model"
match_context = DummyMatchContext(data)
match_element = element_value_branch_me.get_match_element(
self.path, match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
MatchElement("path/value_branch/branch/path", self.path_path,
self.path_path, None),
MatchElement("path/value_branch/value_model",
self.path_fixed_string, self.path_fixed_string,
None)
])
data = b"data: this is some random data: 255."
match_context = DummyMatchContext(data)
match_element = element_value_branch_me.get_match_element(
self.path, match_context)
self.compare_match_results(
data, match_element, match_context, self.id_, self.path, data,
data, [
MatchElement("path/value_branch/branch/data", self.data_path,
self.data_path, None),
MatchElement("path/value_branch/value_model",
self.data_fixed_string, self.data_fixed_string,
None)
])
def test3append_values_float(self):
"""This unittest checks the append_values method with raw_match_object being a float value."""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
# initialize all values.
t = time.time()
log_atom = LogAtom(
b'22.2', ParserMatch(MatchElement('path', b'22.2', 22.2, None)), t,
self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22.2]]])
log_atom = LogAtom(b'22',
ParserMatch(MatchElement('path', b'22', 22, None)),
t, self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22]]])
log_atom = LogAtom(
b'22.2', ParserMatch(MatchElement('path', b'22', b'22', None)), t,
self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22]]])
def get_match_element(self, path, match_context):
"""@return the embedded child match or an empty match."""
current_path = "%s/%s" % (path, self.element_id)
start_data = match_context.match_data
match = self.optional_element.get_match_element(current_path, match_context)
if match is None:
return MatchElement("%s/%s" % (path, self.element_id), '', None, None)
return MatchElement(current_path, start_data[:len(start_data) - len(match_context.match_data)],
start_data[:len(start_data) - len(match_context.match_data)], [match])
def getMatchElement(self, path, matchContext):
"""@return the embedded child match or an empty match."""
currentPath = "%s/%s" % (path, self.elementId)
startData = matchContext.matchData
match = self.optionalElement.getMatchElement(currentPath, matchContext)
if match is None:
return MatchElement("%s/%s" % (path, self.elementId), \
'', None, None)
return MatchElement(currentPath, \
startData[:len(startData)-len(matchContext.matchData)],
startData[:len(startData)-len(matchContext.matchData)], [match])
def test11_path_dependent_histogram_analysis_no_report(self):
"""
This test case aims to test the functionality of the PathDependantHistogramAnalysis.receive_atom method.
No report is expected.
"""
description = "Test11HistogramAnalysis"
start_time = 57600
end_time = 662600
diff = 30000
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, False)
histogram_data = HistogramData(self.match_crontab, modulo_time_bin_definition)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
self.aminer_config, histogram_data.property_path, modulo_time_bin_definition, 604800,
[self.stream_printer_event_handler], True, 'Default')
self.analysis_context.register_component(path_dependent_histogram_analysis, description)
match_element = MatchElement(self.match_crontab, str(start_time).encode(), start_time, None)
t = time.time()
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, path_dependent_histogram_analysis)
path_dependent_histogram_analysis.receive_atom(log_atom)
histogram_data.add_value(start_time)
histogram_data.add_value(end_time)
match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None)
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, path_dependent_histogram_analysis)
path_dependent_histogram_analysis.receive_atom(log_atom)
self.assertEqual(self.output_stream.getvalue(), '')
# resetting the outputStream
start_time = start_time + 3600
end_time = end_time + 3600
self.reset_output_stream()
t = t + diff
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, path_dependent_histogram_analysis)
path_dependent_histogram_analysis.receive_atom(log_atom)
match_element = MatchElement(self.match_crontab, str(start_time).encode(), start_time, None)
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, path_dependent_histogram_analysis)
path_dependent_histogram_analysis.receive_atom(log_atom)
histogram_data.add_value(start_time)
histogram_data.add_value(end_time)
histogram_data.add_value(start_time)
histogram_data.add_value(end_time)
match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None)
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, path_dependent_histogram_analysis)
path_dependent_histogram_analysis.receive_atom(log_atom)
match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None)
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, path_dependent_histogram_analysis)
path_dependent_histogram_analysis.receive_atom(log_atom)
self.assertEqual(self.output_stream.getvalue(), '')
def test2annotate_match(self):
"""This test case checks if all possible annotations are created correctly."""
a3 = MatchElement('a3', b'a3', b'a3', [])
a2 = MatchElement('a2', b'a2', b'a2', [a3])
a1 = MatchElement('a1', b'a1', b'a1', [a2])
b3 = MatchElement('b3', b'b3', b'b3', [])
b2 = MatchElement('b2', b'b2', b'b2', [b3])
b1 = MatchElement('b1', b'b1', b'b1', [b2])
root_element = MatchElement('root', b'root', b'root', [a1, b1])
self.assertEqual(root_element.annotate_match(None), "root: b'root' a1: b'a1' a2: b'a2' a3: b'a3' b1: b'b1' b2: b'b2' b3: b'b3'")
self.assertEqual(root_element.annotate_match(''), "root: b'root'\n a1: b'a1'\n a2: b'a2'\n a3: b'a3'\n b1: b'b1'\n"
" b2: b'b2'\n b3: b'b3'")
def test10_histogram_analysis_receive_atom_report_expected(self):
"""This test case aims to test the functionality of the HistogramAnalysis's receive_atom method, when A report is expected."""
description = "Test10HistogramAnalysis"
start_time = 57600
end_time = 662600
diff = 605000
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, False)
histogram_data = HistogramData(self.match_crontab, modulo_time_bin_definition)
histogram_analysis = HistogramAnalysis(self.aminer_config, [(histogram_data.property_path, modulo_time_bin_definition)], 604800,
[self.stream_printer_event_handler], True, 'Default')
self.analysis_context.register_component(histogram_analysis, description)
match_element = MatchElement(self.match_crontab, str(start_time).encode(), start_time, None)
t = time.time()
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, histogram_analysis)
histogram_analysis.receive_atom(log_atom)
histogram_data.add_value(start_time)
histogram_data.add_value(end_time)
match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None)
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, histogram_analysis)
histogram_analysis.receive_atom(log_atom)
self.assertEqual(self.output_stream.getvalue(), self.__expected_string_histogram_analysis % (
datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string), histogram_analysis.__class__.__name__, description, 2,
datetime.fromtimestamp(t).strftime(self.datetime_format_string),
datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string),
'Property "match/crontab" (2 elements):\n * [16-17]: 2 (ratio = 1.00e+00, p = 1.74e-03)'))
# resetting the outputStream
start_time = start_time + 3600
end_time = end_time + 3600
self.reset_output_stream()
t = t + diff
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, histogram_analysis)
histogram_analysis.receive_atom(log_atom)
histogram_data.add_value(start_time)
histogram_data.add_value(end_time)
match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None)
log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, histogram_analysis)
histogram_analysis.receive_atom(log_atom)
self.assertEqual(self.output_stream.getvalue(), self.__expected_string_histogram_analysis % (
datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string), histogram_analysis.__class__.__name__, description, 2,
datetime.fromtimestamp(t).strftime(self.datetime_format_string),
datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string),
'Property "match/crontab" (2 elements):\n * [16-17]: 1 (ratio = 5.00e-01, p = 8.16e-02)\n * [17-18]: 1 '
'(ratio = 5.00e-01, p = 8.16e-02)'))
def test11get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = Base64StringModelElement(self.id_)
data = b"VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkLg=="
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def getMatchElement(self, path, matchContext):
"""Find the maximum number of bytes forming a integer number
according to the parameters specified
@return a match when at least one byte being a digit was found"""
data = matchContext.matchData
matchLen = 0
atEndFlag = False
for testByte in data:
bVal = testByte
if atEndFlag:
if ((matchLen & 0x3) == 0) or (bVal != 0x3d):
break
elif (not ((bVal >= 0x30) and (bVal <= 0x39))
and not ((bVal >= 0x41) and (bVal <= 0x5a))
and not ((bVal >= 0x61) and (bVal <= 0x7a))
and (bVal not in [0x2b, 0x2f])):
if (bVal != 0x3d) or ((matchLen & 0x2) == 0):
break
atEndFlag = True
matchLen += 1
matchLen = matchLen & (-4)
if matchLen == 0:
return None
matchString = data[:matchLen]
matchContext.update(matchString)
return MatchElement("%s/%s" % (path, self.pathId), matchString,
base64.b64decode(matchString), None)
def test6get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = SequenceModelElement(self.id_, self.children)
data = b"string0 string1 string2"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def getMatchElement(self, path, matchContext):
"""Find the maximum number of bytes forming a integer number
according to the parameters specified
@return a match when at least one byte being a digit was found"""
data = matchContext.matchData
matchLen = 0
for bVal in data:
if ((bVal < 0x30) or
(bVal > 0x39)) and ((bVal < self.charStart) or
(bVal - self.charStart > 5)):
break
matchLen += 1
if matchLen == 0:
return None
matchObject = data[:matchLen]
try:
matchString = bytes.fromhex(matchObject.decode('utf-8'))
except ValueError:
return None
matchContext.update(matchObject)
return MatchElement("%s/%s" % (path, self.elementId), \
matchString, matchObject, None)
def get_match_element(self, path: str, match_context):
"""
Read an IP address at the current data position. When found, the match_object will be.
Allowed formats for IPv6 addresses are defined in RFC4291 section 2.2.
However, trailing IPv4 addresses (for example ::FFFF:129.144.52.38) are not allowed.
"""
data = match_context.match_data
m = self.regex.match(data)
if m is None:
return None
match_len = m.span(0)[1]
if self.extract is extract_ipv6_address and (
b"." in m.group()[:match_len].split(b":")[-1] or
(len(data) > match_len and
(re.compile(br"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}").match(
data[data.rfind(b":", 0, match_len) + 1:]) is not None or
(data.find(b"::", match_len) == match_len and b"::" in data)))):
return None
extracted_address = self.extract(m.group(), match_len)
if extracted_address is None:
return None
match_string = data[:match_len]
match_context.update(match_string)
return MatchElement("%s/%s" % (path, self.element_id), match_string,
extracted_address, None)
def test13get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = DecimalFloatValueModelElement(self.id_)
data = b"123.22"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test9get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = IpAddressDataModelElement(self.id_)
data = b"abcdefghijklmnopqrstuvwxyz.!?"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test7get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = OptionalMatchModelElement(
self.id_, DummyFixedDataModelElement(self.fixed_id,
self.fixed_data))
data = b"fixed data"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def get_match_element(self, path: str, match_context):
"""@return Always return a match."""
msg = 'DebugModelElement path = "%s/%s", unmatched = "%s"' % (
path, self.element_id, repr(match_context.match_data))
logging.getLogger(DEBUG_LOG_NAME).info(msg)
print(msg, file=sys.stderr)
return MatchElement('%s/%s' % (path, self.element_id), b"", b"", None)
def test26get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S")
data = b"07.02.2019 11:40:00: it still works"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def get_match_element(self, path, match_context):
"""
Try to parse all of the match_context against JSON.
When a match is found, the match_context is updated accordingly.
@param path the model path to the parent model element invoking this method.
@param match_context an instance of MatchContext class holding the data context to match against.
@return the matchElement or None if model did not match.
"""
current_path = "%s/%s" % (path, self.element_id)
old_match_data = match_context.match_data
matches = []
try:
json_match_data = json.loads(match_context.match_data)
except JSONDecodeError as e:
logging.getLogger(debug_log_prefix + AminerConfig.DEBUG_LOG_NAME).debug(e)
return None
match_context.match_data = match_context.match_data.decode('unicode-escape').encode()
matches += self.parse_json_dict(self.key_parser_dict, json_match_data, current_path, match_context)
remove_chars = b' }]"\r\n'
match_data = match_context.match_data
for c in remove_chars:
match_data = match_data.replace(bytes(chr(c), encoding="utf-8"), b'')
if None in matches or match_data != b'':
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).debug(
debug_log_prefix + "get_match_element_main NONE RETURNED", match_context.match_data.strip(b' }]"\r\n').decode())
match_context.match_data = old_match_data
return None
# remove all remaining spaces and brackets.
match_context.match_data = b''
return MatchElement(current_path, str(json_match_data), json_match_data, matches)
def get_match_element(self, path, match_context):
"""
Find the maximum number of bytes forming a integer number according to the parameters specified.
@return a match when at least one byte being a digit was found
"""
data = match_context.match_data
match_len = 0
for b_val in data:
if ((b_val < 0x30) or
(b_val > 0x39)) and ((b_val < self.char_start) or
(b_val - self.char_start > 5)):
break
match_len += 1
if match_len == 0:
return None
match_object = data[:match_len]
try:
pad = ''
if len(match_object.decode()) % 2 != 0:
pad = '0'
match_string = bytes.fromhex(pad + match_object.decode())
except ValueError:
return None
match_context.update(match_object)
return MatchElement("%s/%s" % (path, self.element_id), match_string,
match_object, None)
def get_match_element(self, path, match_context):
"""
Find the maximum number of bytes forming a integer number according to the parameters specified.
@return a match when at least one byte being a digit was found.
"""
data = match_context.match_data
match_len = 0
at_end_flag = False
for test_byte in data:
b_val = test_byte
if at_end_flag:
if ((match_len & 0x3) == 0) or (b_val != 0x3d):
break
elif (not (0x30 <= b_val <= 0x39) and not (0x41 <= b_val <= 0x5a)
and not (0x61 <= b_val <= 0x7a)
and (b_val not in [0x2b, 0x2f])):
if (b_val != 0x3d) or ((match_len & 0x2) == 0):
break
at_end_flag = True
match_len += 1
match_len = match_len & (-4)
if match_len == 0:
return None
match_string = data[:match_len]
match_context.update(match_string)
return MatchElement("%s/%s" % (path, self.path_id), match_string,
base64.b64decode(match_string), None)
def get_match_element(self, path, match_context):
"""@return None when there is no match, MatchElement otherwise."""
if not match_context.match_data.startswith(self.fixed_data):
return None
match_context.update(self.fixed_data)
return MatchElement("%s/%s" % (path, self.element_id), self.fixed_data,
self.fixed_data, None)
def __init__(self, match_data):
"""
Create a MatchContext with the full unmatched string data.
@param match_data the data that will be tested by the next model element.
"""
self.match_data = match_data
self.root_match_element = MatchElement('/', None, None, [])
def test12init_child_elements_with_no_path(self):
"""This test case checks, whether an exception is raised, when the path is None and children are passed."""
self.assertRaises(ValueError, MatchElement, None, self.match_string,
self.match_object, [
MatchElement(self.path, self.match_string,
self.match_object, None)
])
def test6get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = WhiteSpaceLimitedDataModelElement(self.id_)
data = b"space: ,tab:\t"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def get_match_element(self, path, match_context):
"""Just return a match including all data from the context"""
match_data = match_context.match_data
if not match_data:
return None
match_context.update(match_data)
return MatchElement("%s/%s" % (path, self.element_id), match_data,
match_data, None)
def getMatchElement(self, path, matchContext):
"""Try to find a match on given data for the test model and
the selected branch.
@param path the model path to the parent model element invoking
this method.
@param matchContext an instance of MatchContext class holding
the data context to match against.
@return the matchElement or None if the test model did not
match, no branch was selected or the branch did not match."""
currentPath = "%s/%s" % (path, self.elementId)
startData = matchContext.matchData
modelMatch = self.valueModel.getMatchElement(currentPath, matchContext)
if modelMatch is None:
return None
# Now extract the test path value from the modelMatch. From here
# on, the matchContext is already modified so we must NEVER just
# return but revert the changes in the context first.
remainingValuePath = self.valuePath
testMatch = modelMatch
currentTestPath = testMatch.getPath()
while remainingValuePath is not None:
nextPartPos = remainingValuePath.find('/')
if nextPartPos <= 0:
currentTestPath += '/' + remainingValuePath
remainingValuePath = None
else:
currentTestPath += '/' + remainingValuePath[:nextPartPos]
remainingValuePath = remainingValuePath[nextPartPos + 1:]
matchChildren = testMatch.getChildren()
testMatch = None
if matchChildren is None:
break
for child in matchChildren:
if child.getPath() == currentTestPath:
testMatch = child
break
branchMatch = None
if testMatch is not None:
if isinstance(testMatch.getMatchObject(), bytes):
branchModel = self.branchModelDict.get(testMatch.getMatchObject().decode(), \
self.defaultBranch)
else:
branchModel = self.branchModelDict.get(testMatch.getMatchObject(), \
self.defaultBranch)
if branchModel is not None:
branchMatch = branchModel.getMatchElement(
currentPath, matchContext)
if branchMatch is None:
matchContext.matchData = startData
return None
return MatchElement(currentPath, \
startData[:len(startData)-len(matchContext.matchData)], \
startData[:len(startData)-len(matchContext.matchData)], [modelMatch, branchMatch])
def get_match_element(self, path: str, match_context):
"""
Try to parse all of the match_context against JSON.
When a match is found, the match_context is updated accordingly.
@param path the model path to the parent model element invoking this method.
@param match_context an instance of MatchContext class holding the data context to match against.
@return the matchElement or None if model did not match.
"""
current_path = "%s/%s" % (path, self.element_id)
old_match_data = match_context.match_data
matches: Union[List[Union[MatchElement, None]]] = []
try:
index = 0
while index != -1:
index = match_context.match_data.find(rb"\x", index)
if index != -1 and index != match_context.match_data.find(
b"\\x", index - 1):
match_context.match_data = match_context.match_data.decode(
"unicode-escape").encode()
break
json_match_data = json.loads(match_context.match_data,
parse_float=format_float)
if not isinstance(json_match_data, dict):
return None
except JSONDecodeError as e:
logging.getLogger(debug_log_prefix + DEBUG_LOG_NAME).debug(e)
return None
self.dec_escapes = True
if self.is_escaped_unicode(match_context.match_data.decode()):
match_context.match_data = match_context.match_data.decode(
"unicode-escape").encode()
self.dec_escapes = False
matches += self.parse_json_dict(self.key_parser_dict, json_match_data,
current_path, match_context)
remove_chars = b' }]"\r\n'
match_data = match_context.match_data
for c in remove_chars:
match_data = match_data.replace(bytes(chr(c), encoding="utf-8"),
b"")
if None in matches or (match_data != b"" and len(matches) > 0):
logging.getLogger(DEBUG_LOG_NAME).debug(
debug_log_prefix + "get_match_element_main NONE RETURNED",
match_context.match_data.strip(b' }]"\r\n').decode())
match_context.match_data = old_match_data
return None
# remove all remaining spaces and brackets.
match_context.update(match_context.match_data)
if len(matches) == 0:
resulting_matches = None
else:
resulting_matches = matches
return MatchElement(current_path,
str(json_match_data).encode(), json_match_data,
resulting_matches)
