def test4append_values_bytestring(self): """ This unittest checks the append_values method with raw_match_object being a bytestring. This should trigger a ValueError and append the match_string. """ event_type_detector = EventTypeDetector( self.aminer_config, [self.stream_printer_event_handler]) # initialize all values. t = time.time() log_atom = LogAtom( b'This is a string', ParserMatch( MatchElement('path', b'This is a string', b'This is a string', None)), t, self.__class__.__name__) event_type_detector.receive_atom(log_atom) event_type_detector.values = [[[]]] event_type_detector.append_values(log_atom, 0) self.assertEqual(event_type_detector.values, [[['This is a string']]]) log_atom = LogAtom( b'24.05.', ParserMatch(MatchElement('path', b'24.05.', b'24.05.', None)), t, self.__class__.__name__) event_type_detector.values = [[[]]] event_type_detector.append_values(log_atom, 0) self.assertEqual(event_type_detector.values, [[['24.05.']]])
def test1_child_elements_with_no_path(self): """This test case checks, whether a Exception is raised, when the path is None or empty and children are passed.""" self.assertRaises(Exception, MatchElement, None, b'matchString', b'matchString', ( MatchElement('child', b'childMatchString', b'childMatchString', []))) self.assertRaises(Exception, MatchElement, '', b'matchString', b'matchString', ( MatchElement('child', b'childMatchString', b'childMatchString', [])))
def test5check_value_reduction(self): """This unittest checks the functionality of reducing the values when the maxNumVals threshold is reached.""" event_type_detector = EventTypeDetector( self.aminer_config, [self.stream_printer_event_handler]) t = time.time() val_list = [[[]]] for i in range(1, event_type_detector.max_num_vals + 1, 1): log_atom = LogAtom( str(i).encode(), ParserMatch(MatchElement('path', str(i).encode(), i, None)), t, self.__class__.__name__) val_list[0][0].append(float(i)) self.assertTrue(event_type_detector.receive_atom(log_atom)) self.assertEqual(event_type_detector.values, val_list) i += 1 log_atom = LogAtom( str(i).encode(), ParserMatch(MatchElement('path', str(i).encode(), i, None)), t, self.__class__.__name__) val_list[0][0].append(float(i)) self.assertTrue(event_type_detector.receive_atom(log_atom)) self.assertEqual( event_type_detector.values, [[val_list[0][0][-event_type_detector.min_num_vals:]]])
def test3get_match_element_valid_match(self): """Parse matching substring from MatchContext and check if the MatchContext was updated with all characters.""" element_value_branch_me = ElementValueBranchModelElement( self.id_, self.value_model, None, { "path: ": self.path_me, "data: ": self.data_me }) data = b"path: /model" match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_match_results( data, match_element, match_context, self.id_, self.path, data, data, [ MatchElement("path/value_branch/branch/path", self.path_path, self.path_path, None), MatchElement("path/value_branch/value_model", self.path_fixed_string, self.path_fixed_string, None) ]) data = b"data: this is some random data: 255." match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_match_results( data, match_element, match_context, self.id_, self.path, data, data, [ MatchElement("path/value_branch/branch/data", self.data_path, self.data_path, None), MatchElement("path/value_branch/value_model", self.data_fixed_string, self.data_fixed_string, None) ])
def test3append_values_float(self): """This unittest checks the append_values method with raw_match_object being a float value.""" event_type_detector = EventTypeDetector( self.aminer_config, [self.stream_printer_event_handler]) # initialize all values. t = time.time() log_atom = LogAtom( b'22.2', ParserMatch(MatchElement('path', b'22.2', 22.2, None)), t, self.__class__.__name__) event_type_detector.receive_atom(log_atom) event_type_detector.values = [[[]]] event_type_detector.append_values(log_atom, 0) self.assertEqual(event_type_detector.values, [[[22.2]]]) log_atom = LogAtom(b'22', ParserMatch(MatchElement('path', b'22', 22, None)), t, self.__class__.__name__) event_type_detector.values = [[[]]] event_type_detector.append_values(log_atom, 0) self.assertEqual(event_type_detector.values, [[[22]]]) log_atom = LogAtom( b'22.2', ParserMatch(MatchElement('path', b'22', b'22', None)), t, self.__class__.__name__) event_type_detector.values = [[[]]] event_type_detector.append_values(log_atom, 0) self.assertEqual(event_type_detector.values, [[[22]]])
def get_match_element(self, path, match_context): """@return the embedded child match or an empty match.""" current_path = "%s/%s" % (path, self.element_id) start_data = match_context.match_data match = self.optional_element.get_match_element(current_path, match_context) if match is None: return MatchElement("%s/%s" % (path, self.element_id), '', None, None) return MatchElement(current_path, start_data[:len(start_data) - len(match_context.match_data)], start_data[:len(start_data) - len(match_context.match_data)], [match])
def getMatchElement(self, path, matchContext): """@return the embedded child match or an empty match.""" currentPath = "%s/%s" % (path, self.elementId) startData = matchContext.matchData match = self.optionalElement.getMatchElement(currentPath, matchContext) if match is None: return MatchElement("%s/%s" % (path, self.elementId), \ '', None, None) return MatchElement(currentPath, \ startData[:len(startData)-len(matchContext.matchData)], startData[:len(startData)-len(matchContext.matchData)], [match])
def test11_path_dependent_histogram_analysis_no_report(self): """ This test case aims to test the functionality of the PathDependantHistogramAnalysis.receive_atom method. No report is expected. """ description = "Test11HistogramAnalysis" start_time = 57600 end_time = 662600 diff = 30000 modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, False) histogram_data = HistogramData(self.match_crontab, modulo_time_bin_definition) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( self.aminer_config, histogram_data.property_path, modulo_time_bin_definition, 604800, [self.stream_printer_event_handler], True, 'Default') self.analysis_context.register_component(path_dependent_histogram_analysis, description) match_element = MatchElement(self.match_crontab, str(start_time).encode(), start_time, None) t = time.time() log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, path_dependent_histogram_analysis) path_dependent_histogram_analysis.receive_atom(log_atom) histogram_data.add_value(start_time) histogram_data.add_value(end_time) match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None) log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, path_dependent_histogram_analysis) path_dependent_histogram_analysis.receive_atom(log_atom) self.assertEqual(self.output_stream.getvalue(), '') # resetting the outputStream start_time = start_time + 3600 end_time = end_time + 3600 self.reset_output_stream() t = t + diff log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, path_dependent_histogram_analysis) path_dependent_histogram_analysis.receive_atom(log_atom) match_element = MatchElement(self.match_crontab, str(start_time).encode(), start_time, None) log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, path_dependent_histogram_analysis) path_dependent_histogram_analysis.receive_atom(log_atom) histogram_data.add_value(start_time) histogram_data.add_value(end_time) histogram_data.add_value(start_time) histogram_data.add_value(end_time) match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None) log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, path_dependent_histogram_analysis) path_dependent_histogram_analysis.receive_atom(log_atom) match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None) log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, path_dependent_histogram_analysis) path_dependent_histogram_analysis.receive_atom(log_atom) self.assertEqual(self.output_stream.getvalue(), '')
def test2annotate_match(self): """This test case checks if all possible annotations are created correctly.""" a3 = MatchElement('a3', b'a3', b'a3', []) a2 = MatchElement('a2', b'a2', b'a2', [a3]) a1 = MatchElement('a1', b'a1', b'a1', [a2]) b3 = MatchElement('b3', b'b3', b'b3', []) b2 = MatchElement('b2', b'b2', b'b2', [b3]) b1 = MatchElement('b1', b'b1', b'b1', [b2]) root_element = MatchElement('root', b'root', b'root', [a1, b1]) self.assertEqual(root_element.annotate_match(None), "root: b'root' a1: b'a1' a2: b'a2' a3: b'a3' b1: b'b1' b2: b'b2' b3: b'b3'") self.assertEqual(root_element.annotate_match(''), "root: b'root'\n a1: b'a1'\n a2: b'a2'\n a3: b'a3'\n b1: b'b1'\n" " b2: b'b2'\n b3: b'b3'")
def test10_histogram_analysis_receive_atom_report_expected(self): """This test case aims to test the functionality of the HistogramAnalysis's receive_atom method, when A report is expected.""" description = "Test10HistogramAnalysis" start_time = 57600 end_time = 662600 diff = 605000 modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, False) histogram_data = HistogramData(self.match_crontab, modulo_time_bin_definition) histogram_analysis = HistogramAnalysis(self.aminer_config, [(histogram_data.property_path, modulo_time_bin_definition)], 604800, [self.stream_printer_event_handler], True, 'Default') self.analysis_context.register_component(histogram_analysis, description) match_element = MatchElement(self.match_crontab, str(start_time).encode(), start_time, None) t = time.time() log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, histogram_analysis) histogram_analysis.receive_atom(log_atom) histogram_data.add_value(start_time) histogram_data.add_value(end_time) match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None) log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, histogram_analysis) histogram_analysis.receive_atom(log_atom) self.assertEqual(self.output_stream.getvalue(), self.__expected_string_histogram_analysis % ( datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string), histogram_analysis.__class__.__name__, description, 2, datetime.fromtimestamp(t).strftime(self.datetime_format_string), datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string), 'Property "match/crontab" (2 elements):\n * [16-17]: 2 (ratio = 1.00e+00, p = 1.74e-03)')) # resetting the outputStream start_time = start_time + 3600 end_time = end_time + 3600 self.reset_output_stream() t = t + diff log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t, histogram_analysis) histogram_analysis.receive_atom(log_atom) histogram_data.add_value(start_time) histogram_data.add_value(end_time) match_element = MatchElement(self.match_crontab, str(end_time).encode(), end_time, None) log_atom = LogAtom(histogram_data.bin_data, ParserMatch(match_element), t + diff, histogram_analysis) histogram_analysis.receive_atom(log_atom) self.assertEqual(self.output_stream.getvalue(), self.__expected_string_histogram_analysis % ( datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string), histogram_analysis.__class__.__name__, description, 2, datetime.fromtimestamp(t).strftime(self.datetime_format_string), datetime.fromtimestamp(t + diff).strftime(self.datetime_format_string), 'Property "match/crontab" (2 elements):\n * [16-17]: 1 (ratio = 5.00e-01, p = 8.16e-02)\n * [17-18]: 1 ' '(ratio = 5.00e-01, p = 8.16e-02)'))
def test11get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = Base64StringModelElement(self.id_) data = b"VGhpcyBpcyBzb21lIHN0cmluZyB0byBiZSBlbmNvZGVkLg==" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(self.path, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def getMatchElement(self, path, matchContext): """Find the maximum number of bytes forming a integer number according to the parameters specified @return a match when at least one byte being a digit was found""" data = matchContext.matchData matchLen = 0 atEndFlag = False for testByte in data: bVal = testByte if atEndFlag: if ((matchLen & 0x3) == 0) or (bVal != 0x3d): break elif (not ((bVal >= 0x30) and (bVal <= 0x39)) and not ((bVal >= 0x41) and (bVal <= 0x5a)) and not ((bVal >= 0x61) and (bVal <= 0x7a)) and (bVal not in [0x2b, 0x2f])): if (bVal != 0x3d) or ((matchLen & 0x2) == 0): break atEndFlag = True matchLen += 1 matchLen = matchLen & (-4) if matchLen == 0: return None matchString = data[:matchLen] matchContext.update(matchString) return MatchElement("%s/%s" % (path, self.pathId), matchString, base64.b64decode(matchString), None)
def test6get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = SequenceModelElement(self.id_, self.children) data = b"string0 string1 string2" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(None, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def getMatchElement(self, path, matchContext): """Find the maximum number of bytes forming a integer number according to the parameters specified @return a match when at least one byte being a digit was found""" data = matchContext.matchData matchLen = 0 for bVal in data: if ((bVal < 0x30) or (bVal > 0x39)) and ((bVal < self.charStart) or (bVal - self.charStart > 5)): break matchLen += 1 if matchLen == 0: return None matchObject = data[:matchLen] try: matchString = bytes.fromhex(matchObject.decode('utf-8')) except ValueError: return None matchContext.update(matchObject) return MatchElement("%s/%s" % (path, self.elementId), \ matchString, matchObject, None)
def get_match_element(self, path: str, match_context): """ Read an IP address at the current data position. When found, the match_object will be. Allowed formats for IPv6 addresses are defined in RFC4291 section 2.2. However, trailing IPv4 addresses (for example ::FFFF:129.144.52.38) are not allowed. """ data = match_context.match_data m = self.regex.match(data) if m is None: return None match_len = m.span(0)[1] if self.extract is extract_ipv6_address and ( b"." in m.group()[:match_len].split(b":")[-1] or (len(data) > match_len and (re.compile(br"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}").match( data[data.rfind(b":", 0, match_len) + 1:]) is not None or (data.find(b"::", match_len) == match_len and b"::" in data)))): return None extracted_address = self.extract(m.group(), match_len) if extracted_address is None: return None match_string = data[:match_len] match_context.update(match_string) return MatchElement("%s/%s" % (path, self.element_id), match_string, extracted_address, None)
def test13get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = DecimalFloatValueModelElement(self.id_) data = b"123.22" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(self.path, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def test9get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = IpAddressDataModelElement(self.id_) data = b"abcdefghijklmnopqrstuvwxyz.!?" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(None, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def test7get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = OptionalMatchModelElement( self.id_, DummyFixedDataModelElement(self.fixed_id, self.fixed_data)) data = b"fixed data" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(None, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def get_match_element(self, path: str, match_context): """@return Always return a match.""" msg = 'DebugModelElement path = "%s/%s", unmatched = "%s"' % ( path, self.element_id, repr(match_context.match_data)) logging.getLogger(DEBUG_LOG_NAME).info(msg) print(msg, file=sys.stderr) return MatchElement('%s/%s' % (path, self.element_id), b"", b"", None)
def test26get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = DateTimeModelElement(self.id_, b"%d.%m.%Y %H:%M:%S") data = b"07.02.2019 11:40:00: it still works" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(self.path, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def get_match_element(self, path, match_context): """ Try to parse all of the match_context against JSON. When a match is found, the match_context is updated accordingly. @param path the model path to the parent model element invoking this method. @param match_context an instance of MatchContext class holding the data context to match against. @return the matchElement or None if model did not match. """ current_path = "%s/%s" % (path, self.element_id) old_match_data = match_context.match_data matches = [] try: json_match_data = json.loads(match_context.match_data) except JSONDecodeError as e: logging.getLogger(debug_log_prefix + AminerConfig.DEBUG_LOG_NAME).debug(e) return None match_context.match_data = match_context.match_data.decode('unicode-escape').encode() matches += self.parse_json_dict(self.key_parser_dict, json_match_data, current_path, match_context) remove_chars = b' }]"\r\n' match_data = match_context.match_data for c in remove_chars: match_data = match_data.replace(bytes(chr(c), encoding="utf-8"), b'') if None in matches or match_data != b'': logging.getLogger(AminerConfig.DEBUG_LOG_NAME).debug( debug_log_prefix + "get_match_element_main NONE RETURNED", match_context.match_data.strip(b' }]"\r\n').decode()) match_context.match_data = old_match_data return None # remove all remaining spaces and brackets. match_context.match_data = b'' return MatchElement(current_path, str(json_match_data), json_match_data, matches)
def get_match_element(self, path, match_context): """ Find the maximum number of bytes forming a integer number according to the parameters specified. @return a match when at least one byte being a digit was found """ data = match_context.match_data match_len = 0 for b_val in data: if ((b_val < 0x30) or (b_val > 0x39)) and ((b_val < self.char_start) or (b_val - self.char_start > 5)): break match_len += 1 if match_len == 0: return None match_object = data[:match_len] try: pad = '' if len(match_object.decode()) % 2 != 0: pad = '0' match_string = bytes.fromhex(pad + match_object.decode()) except ValueError: return None match_context.update(match_object) return MatchElement("%s/%s" % (path, self.element_id), match_string, match_object, None)
def get_match_element(self, path, match_context): """ Find the maximum number of bytes forming a integer number according to the parameters specified. @return a match when at least one byte being a digit was found. """ data = match_context.match_data match_len = 0 at_end_flag = False for test_byte in data: b_val = test_byte if at_end_flag: if ((match_len & 0x3) == 0) or (b_val != 0x3d): break elif (not (0x30 <= b_val <= 0x39) and not (0x41 <= b_val <= 0x5a) and not (0x61 <= b_val <= 0x7a) and (b_val not in [0x2b, 0x2f])): if (b_val != 0x3d) or ((match_len & 0x2) == 0): break at_end_flag = True match_len += 1 match_len = match_len & (-4) if match_len == 0: return None match_string = data[:match_len] match_context.update(match_string) return MatchElement("%s/%s" % (path, self.path_id), match_string, base64.b64decode(match_string), None)
def get_match_element(self, path, match_context): """@return None when there is no match, MatchElement otherwise.""" if not match_context.match_data.startswith(self.fixed_data): return None match_context.update(self.fixed_data) return MatchElement("%s/%s" % (path, self.element_id), self.fixed_data, self.fixed_data, None)
def __init__(self, match_data): """ Create a MatchContext with the full unmatched string data. @param match_data the data that will be tested by the next model element. """ self.match_data = match_data self.root_match_element = MatchElement('/', None, None, [])
def test12init_child_elements_with_no_path(self): """This test case checks, whether an exception is raised, when the path is None and children are passed.""" self.assertRaises(ValueError, MatchElement, None, self.match_string, self.match_object, [ MatchElement(self.path, self.match_string, self.match_object, None) ])
def test6get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = WhiteSpaceLimitedDataModelElement(self.id_) data = b"space: ,tab:\t" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(None, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def get_match_element(self, path, match_context): """Just return a match including all data from the context""" match_data = match_context.match_data if not match_data: return None match_context.update(match_data) return MatchElement("%s/%s" % (path, self.element_id), match_data, match_data, None)
def getMatchElement(self, path, matchContext): """Try to find a match on given data for the test model and the selected branch. @param path the model path to the parent model element invoking this method. @param matchContext an instance of MatchContext class holding the data context to match against. @return the matchElement or None if the test model did not match, no branch was selected or the branch did not match.""" currentPath = "%s/%s" % (path, self.elementId) startData = matchContext.matchData modelMatch = self.valueModel.getMatchElement(currentPath, matchContext) if modelMatch is None: return None # Now extract the test path value from the modelMatch. From here # on, the matchContext is already modified so we must NEVER just # return but revert the changes in the context first. remainingValuePath = self.valuePath testMatch = modelMatch currentTestPath = testMatch.getPath() while remainingValuePath is not None: nextPartPos = remainingValuePath.find('/') if nextPartPos <= 0: currentTestPath += '/' + remainingValuePath remainingValuePath = None else: currentTestPath += '/' + remainingValuePath[:nextPartPos] remainingValuePath = remainingValuePath[nextPartPos + 1:] matchChildren = testMatch.getChildren() testMatch = None if matchChildren is None: break for child in matchChildren: if child.getPath() == currentTestPath: testMatch = child break branchMatch = None if testMatch is not None: if isinstance(testMatch.getMatchObject(), bytes): branchModel = self.branchModelDict.get(testMatch.getMatchObject().decode(), \ self.defaultBranch) else: branchModel = self.branchModelDict.get(testMatch.getMatchObject(), \ self.defaultBranch) if branchModel is not None: branchMatch = branchModel.getMatchElement( currentPath, matchContext) if branchMatch is None: matchContext.matchData = startData return None return MatchElement(currentPath, \ startData[:len(startData)-len(matchContext.matchData)], \ startData[:len(startData)-len(matchContext.matchData)], [modelMatch, branchMatch])
def get_match_element(self, path: str, match_context): """ Try to parse all of the match_context against JSON. When a match is found, the match_context is updated accordingly. @param path the model path to the parent model element invoking this method. @param match_context an instance of MatchContext class holding the data context to match against. @return the matchElement or None if model did not match. """ current_path = "%s/%s" % (path, self.element_id) old_match_data = match_context.match_data matches: Union[List[Union[MatchElement, None]]] = [] try: index = 0 while index != -1: index = match_context.match_data.find(rb"\x", index) if index != -1 and index != match_context.match_data.find( b"\\x", index - 1): match_context.match_data = match_context.match_data.decode( "unicode-escape").encode() break json_match_data = json.loads(match_context.match_data, parse_float=format_float) if not isinstance(json_match_data, dict): return None except JSONDecodeError as e: logging.getLogger(debug_log_prefix + DEBUG_LOG_NAME).debug(e) return None self.dec_escapes = True if self.is_escaped_unicode(match_context.match_data.decode()): match_context.match_data = match_context.match_data.decode( "unicode-escape").encode() self.dec_escapes = False matches += self.parse_json_dict(self.key_parser_dict, json_match_data, current_path, match_context) remove_chars = b' }]"\r\n' match_data = match_context.match_data for c in remove_chars: match_data = match_data.replace(bytes(chr(c), encoding="utf-8"), b"") if None in matches or (match_data != b"" and len(matches) > 0): logging.getLogger(DEBUG_LOG_NAME).debug( debug_log_prefix + "get_match_element_main NONE RETURNED", match_context.match_data.strip(b' }]"\r\n').decode()) match_context.match_data = old_match_data return None # remove all remaining spaces and brackets. match_context.update(match_context.match_data) if len(matches) == 0: resulting_matches = None else: resulting_matches = matches return MatchElement(current_path, str(json_match_data).encode(), json_match_data, resulting_matches)