def test3append_values_float(self):
"""This unittest checks the append_values method with raw_match_object being a float value."""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
# initialize all values.
t = time.time()
log_atom = LogAtom(
b'22.2', ParserMatch(MatchElement('path', '22.2', 22.2, None)), t,
self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22.2]]])
log_atom = LogAtom(b'22',
ParserMatch(MatchElement('path', '22', 22, None)),
t, self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22]]])
log_atom = LogAtom(
b'22.2', ParserMatch(MatchElement('path', '22', b'22', None)), t,
self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22]]])
def test4append_values_bytestring(self):
"""
This unittest checks the append_values method with raw_match_object being a bytestring.
This should trigger a ValueError and append the match_string.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
# initialize all values.
t = time.time()
log_atom = LogAtom(
b'This is a string',
ParserMatch(
MatchElement('path', 'This is a string', b'This is a string',
None)), t, self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[['This is a string']]])
log_atom = LogAtom(
b'24.05.',
ParserMatch(MatchElement('path', '24.05.', b'24.05.', None)), t,
self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[['24.05.']]])
def test1_log_atom_not_known(self):
"""This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found. The
output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be repeatable
on second run."""
description = "Test1NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t, new_match_path_detector)
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# repeating should produce the same result
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# other MatchElement
self.assertTrue(new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_d1, self.uid))
def test2_log_atom_known(self):
"""This test case checks the functionality of the autoIncludeFlag. If the same MatchElement is processed a second time and the
auto_include_flag was True, no event must be triggered."""
description = "Test2NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t, new_match_path_detector)
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# repeating should NOT produce the same result
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
# other MatchElement
self.assertTrue(new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_d1, self.uid))
def test4_whitelist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test4EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertEqual(
enhanced_new_match_path_value_combo_detector.whitelist_event(
'Analysis.%s' % enhanced_new_match_path_value_combo_detector.
__class__.__name__, [
log_atom_sequence_me,
[self.match_element_sequence_me.get_path()]
], [
log_atom_sequence_me,
self.match_element_sequence_me.get_path()
], None), self.__expected_whitelisting_string %
(', '.join(
enhanced_new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me.get_path(), log_atom_sequence_me))
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector)
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
self.assertEqual(
enhanced_new_match_path_value_combo_detector.whitelist_event(
'Analysis.%s' % enhanced_new_match_path_value_combo_detector.
__class__.__name__, [
log_atom_sequence_me2,
[self.match_element_sequence_me2.get_path()]
], [
log_atom_sequence_me2,
self.match_element_sequence_me2.get_path()
], None), self.__expected_whitelisting_string %
(', '.join(
enhanced_new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me2.path, log_atom_sequence_me2))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(
new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
new_match_path_value_combo_detector.do_persist()
self.reset_output_stream()
other_new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
other_new_match_path_value_combo_detector, description + "2")
otherLogAtomFixedDME = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
other_new_match_path_value_combo_detector)
self.assertTrue(
other_new_match_path_value_combo_detector.receive_atom(
otherLogAtomFixedDME))
self.assertEqual(self.output_stream.getvalue(), '')
def test10_allowlist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test10NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector,
description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
self.assertEqual(
new_match_path_detector.allowlist_event(
self.analysis % new_match_path_detector.__class__.__name__,
self.match_element_fixed_dme.get_path(),
None), 'Allowlisted path(es) %s in %s.' %
(self.match_element_fixed_dme.get_path(),
self.analysis % new_match_path_detector.__class__.__name__))
new_match_path_detector.auto_include_flag = False
self.assertEqual(
new_match_path_detector.allowlist_event(
self.analysis % new_match_path_detector.__class__.__name__,
self.match_element_decimal_integer_value_me.get_path(),
None), 'Allowlisted path(es) %s in %s.' %
(self.match_element_decimal_integer_value_me.path,
self.analysis % new_match_path_detector.__class__.__name__))
def run_time_correlation_detector(self, number_of_rules):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
time_correlation_detector = TimeCorrelationDetector(
self.aminer_config,
2,
number_of_rules,
0, [self.stream_printer_event_handler],
record_count_before_event=self.waiting_time * 9000)
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
time_correlation_detector)
time_correlation_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
time_correlation_detector.__class__.__name__, avg, results,
'testCount=%d.' % number_of_rules)
def test2receive_event_with_same_event_data_attributes(self):
"""In this test case an attribute of AnalysisComponent is overwritten and an JsonError attribute is expected."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2'],
'Message': 'An other event happened too!'
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
self.assertEqual(
self.output_stream.getvalue(), self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(float("%.2f" % self.t), 2),
',\n "JsonError": "AnalysisComponent attribute \'Message\' is already in use and can not be overwritten!\\n"'
))
def test2receive_atoms_with_defined_path_list(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is set to a static list of paths and variable_key_list should not be used.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler],
path_list=['parser/type/path/nametype'])
results = [
True, False, True, False, True, False, True, True, False, False,
True, True, False, True, False, True, False, True, False, False,
True
]
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(
line,
ParserMatch(
self.parsing_model.get_match_element(
'parser', MatchContext(line))), t,
self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
old_vals = (event_type_detector.num_events,
event_type_detector.num_eventlines,
event_type_detector.total_records,
event_type_detector.longest_path)
self.assertEqual(event_type_detector.receive_atom(log_atom),
not results[i], i)
if results[i]:
self.assertEqual(old_vals, (event_type_detector.num_events,
event_type_detector.num_eventlines,
event_type_detector.total_records,
event_type_detector.longest_path))
def run_new_match_path_detector(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
t = round(time.time(), 3)
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
new_match_path_detector)
new_match_path_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
new_match_path_detector.__class__.__name__, avg, results,
self.different_pathes % number_of_pathes)
def test1receive_serialized_data(self):
"""This unittest tests the receive_event method with serialized data from the JsonConverterHandler."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
output = self.output_stream.getvalue()
kafka_event_handler = KafkaEventHandler(
self.analysis_context, self.kafka_topic, {
'bootstrap_servers': ['localhost:9092'],
'api_version': (2, 0, 1)
})
self.assertTrue(
kafka_event_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, output,
log_atom, self))
self.assertEqual(
self.consumer.__next__().value, self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), ""))
def test1log_atom_not_in_path_list(self):
"""This unittest checks if no action happens, when no path in the match_dictionary matches a target_path."""
parser_count = ParserCount(self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2'], [self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.fixed_dme_m3.fixed_data, ParserMatch(self.match_element_m3), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test6receive_atom_without_target_paths(self):
"""This unittest tests the receive_atom method with multiple paths matching without having target_paths specified."""
parser_count = ParserCount(self.aminer_config, None, [self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.match_context_seq.match_data, ParserMatch(self.match_element_seq), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/seq'] = {current_processed_lines_str: 1, total_processed_lines_str: 1}
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test9WhitelistEventWhitelistingDataException(self):
"""The NewMatchPathDetector can not handle whitelisting data and therefore an exception is expected."""
description = "Test9NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
self.assertRaises(Exception, new_match_path_detector.whitelist_event, self.analysis % new_match_path_detector.__class__.__name__,
log_atom_fixed_dme.raw_data, self.output_stream.getvalue(), ['random', 'Data'])
def test2log_atom_matches_single_path(self):
"""This unittest tests the receive_atom method with a single path matching."""
parser_count = ParserCount(self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.fixed_dme_m3.fixed_data, ParserMatch(self.match_element_m3), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/m3'][current_processed_lines_str] = 1
old_count_dict['fixed/m3'][total_processed_lines_str] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test4do_timer(self):
"""This unittest checks if the do_timer method works properly."""
parser_count = ParserCount(self.aminer_config, ['fixed/m3'], [self.stream_printer_event_handler], 600)
t = time.time()
self.assertEqual(int(parser_count.do_timer(t + 100)), 600)
self.assertEqual(self.output_stream.getvalue(), "")
log_atom = LogAtom(self.match_context_seq.match_data, ParserMatch(self.match_element_seq), t, parser_count)
parser_count.receive_atom(log_atom)
self.assertEqual(int(parser_count.do_timer(t + 100)), 500)
self.assertEqual(self.output_stream.getvalue(), "")
self.assertEqual(parser_count.do_timer(t + 601), 600)
self.assertNotEqual(self.output_stream.getvalue(), "")
self.reset_output_stream()
def run_match_value_stream_writer(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
path_list = []
parsing_model = []
while i < number_of_pathes / 2:
path_list.append('match/integer/d' + str(i % number_of_pathes))
path_list.append('match/integer/s' + str(i % number_of_pathes))
parsing_model.append(
DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE))
parsing_model.append(
FixedDataModelElement('s' + str(i % number_of_pathes),
b' Euro '))
i = i + 1
sequence_model_element = SequenceModelElement(
'integer', parsing_model)
match_value_stream_writer = MatchValueStreamWriter(
self.output_stream, path_list, b';', b'-')
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
data = b''
p = process_time()
for j in range(
1,
int(number_of_pathes / 2) + number_of_pathes % 2 + 1):
data = data + str(j).encode() + b' Euro '
seconds = seconds + process_time() - p
match_context = MatchContext(data)
match_element = sequence_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
match_value_stream_writer.__class__.__name__, avg, results,
self.different_pathes % number_of_pathes)
def test8_whitelist_event_type_exception(self):
"""This test case checks whether an exception is thrown when entering an event of another class."""
description = "Test8NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [], [self.stream_printer_event_handler],
'Default', True, True)
self.assertRaises(
Exception, new_match_path_detector.whitelist_event, self.analysis % new_match_path_value_combo_detector.__class__.__name__,
log_atom_fixed_dme.raw_data, self.output_stream.getvalue(), None)
def test5check_value_reduction(self):
"""This unittest checks the functionality of reducing the values when the maxNumVals threshold is reached."""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
t = time.time()
val_list = [[[]]]
for i in range(1, event_type_detector.max_num_vals + 1, 1):
log_atom = LogAtom(
str(i).encode(),
ParserMatch(MatchElement('path', str(i), i, None)), t,
self.__class__.__name__)
val_list[0][0].append(float(i))
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.values, val_list)
i += 1
log_atom = LogAtom(
str(i).encode(), ParserMatch(MatchElement('path', str(i), i,
None)), t,
self.__class__.__name__)
val_list[0][0].append(float(i))
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(
event_type_detector.values,
[[val_list[0][0][-event_type_detector.min_num_vals:]]])
def run_atom_filters_match_value_filter(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
subhandler_filter = SubhandlerFilter([],
stop_when_handled_flag=True)
i = 0
dictionary = {}
while i < 1000000:
dictionary[i] = new_match_path_detector
i = i + 1
i = 0
while i < number_of_pathes:
match_value_filter = MatchValueFilter(
self.integerd + str(i % number_of_pathes), dictionary,
None)
subhandler_filter.add_handler(match_value_filter,
stop_when_handled_flag=True)
i = i + 1
t = round(time.time(), 3)
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
match_value_filter)
subhandler_filter.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
subhandler_filter.__class__.__name__, avg, results,
'%d different %ss with a dictionary of %ss.' %
(number_of_pathes, match_value_filter.__class__.__name__,
new_match_path_detector.__class__.__name__))
def test3log_atom_matches_multiple_paths(self):
"""This unittest tests the receive_atom method with multiple paths matching."""
parser_count = ParserCount(
self.aminer_config,
['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.match_context_seq.match_data,
ParserMatch(self.match_element_seq), t,
parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/seq'] = 1
old_count_dict['fixed/seq/m1'] = 1
old_count_dict['fixed/seq/m2'] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def run_whitelist_violation_detector(self, number_of_pathes,
modulo_factor):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
rules = []
while i < number_of_pathes:
rules.append(
PathExistsMatchRule(
self.integerd + str(i % number_of_pathes), None))
i = i + 1
whitelist_violation_detector = WhitelistViolationDetector(
self.aminer_config, rules, [self.stream_printer_event_handler])
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
p = process_time()
r = random.randint(1, 100)
if r >= modulo_factor:
r = 2
else:
r = 1
seconds = seconds + process_time() - p
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % (number_of_pathes * r)),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
whitelist_violation_detector)
whitelist_violation_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
whitelist_violation_detector.__class__.__name__, avg, results,
'%d different PathExistsMatchRules and a moduloFactor of %d.' %
(number_of_pathes, modulo_factor))
def run_missing_match_path_value_detector(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
path_list = []
while i < number_of_pathes:
path_list.append(self.integerd + str(i % number_of_pathes))
i = i + 1
missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, path_list,
[self.stream_printer_event_handler], 'Default', True, 3600,
86400)
seconds = time.time()
t = seconds
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(1).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
p = process_time()
r = random.randint(0, 100)
seconds = seconds + process_time() - p
delta = int(r / 100)
t = t + 4000 * delta
log_atom = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
missing_match_path_list_value_detector)
missing_match_path_list_value_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
missing_match_path_list_value_detector.__class__.__name__, avg,
results, self.different_pathes % number_of_pathes)
def test1receive_atoms_with_default_values(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is empty and all paths are learned dynamically in variable_key_list.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(
line,
ParserMatch(
self.parsing_model.get_match_element(
'parser', MatchContext(line))), t,
self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.total_records, i + 1)
def test2receive_non_serialized_data(self):
"""This unittest tests the receive_event method with not serialized data"""
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
kafka_event_handler = KafkaEventHandler(
self.analysis_context, self.kafka_topic, {
'bootstrap_servers': ['localhost:9092'],
'api_version': (2, 0, 1)
})
self.assertFalse(
kafka_event_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines,
event_data, log_atom, self))
self.assertRaises(StopIteration, self.consumer.__next__)
def test5save_metadata(self):
"""This test case checks the correctness of the metadata informations"""
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['first/f1/s1'],
[self.stream_printer_event_handler],
'Default',
False,
True,
None,
output_log_line=False)
t = 1
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me)
self.assertEqual(
enhanced_new_match_path_value_combo_detector.known_values_dict.get(
(self.fixed_dme.fixed_data, (t, t, 1))), None)
def test1receive_expected_event(self):
"""In this test case a normal Event happens and the json output should be sent to a StreamPrinterEventHandler."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
self.assertEqual(
self.output_stream.getvalue(), self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), ""))
def test6persist_and_load_data(self):
"""This unittest checks the functionality of the persistence by persisting and reloading values."""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(
b'22.2', ParserMatch(MatchElement('path', '22.2', 22.2, None)), t,
self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.do_persist()
event_type_detector_loaded = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
self.assertEqual(event_type_detector.variable_key_list,
event_type_detector_loaded.variable_key_list)
self.assertEqual(event_type_detector.values,
event_type_detector_loaded.values)
self.assertEqual(event_type_detector.longest_path,
event_type_detector_loaded.longest_path)
self.assertEqual(event_type_detector.check_variables,
event_type_detector_loaded.check_variables)
self.assertEqual(event_type_detector.num_eventlines,
event_type_detector_loaded.num_eventlines)
def test3receive_match_in_time_without_auto_include_flag(self):
"""This test case checks if log_atoms are accepted as expected with the auto_include_flag=False."""
description = 'test3newMatchIdValueComboDetectorTest'
output_stream_empty_results = [True, False, True, False, True, False, True, True, False, True, False, True, True, True, False,
False, False, True, False, True, False]
id_dict_current_results = [
{1: {'parser/type/syscall/syscall': 1}}, {}, {2: {'parser/type/syscall/syscall': 2}}, {},
{3: {'parser/type/syscall/syscall': 3}}, {}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 4: {'parser/type/syscall/syscall': 1}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 5: {'parser/type/path/name': 'two'}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 6: {'parser/type/syscall/syscall': 4}},
{100: {'parser/type/syscall/syscall': 1}, 6: {'parser/type/syscall/syscall': 4}, 7: {'parser/type/path/name': 'five'}},
{100: {'parser/type/syscall/syscall': 1}, 6: {'parser/type/syscall/syscall': 4}, 7: {'parser/type/path/name': 'five'},
8: {'parser/type/syscall/syscall': 6}},
{100: {'parser/type/syscall/syscall': 1}, 7: {'parser/type/path/name': 'five'}, 8: {'parser/type/syscall/syscall': 6}},
{100: {'parser/type/syscall/syscall': 1}, 8: {'parser/type/syscall/syscall': 6}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 9: {'parser/type/syscall/syscall': 2}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 10: {'parser/type/path/name': 'one'}}, {100: {'parser/type/syscall/syscall': 1}}]
id_dict_old_results = [{}] * 21
min_allowed_time_diff = 0.1
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(line, ParserMatch(self.parsing_model.get_match_element('parser', MatchContext(line))), t, self.__class__.__name__))
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(self.aminer_config, [
'parser/type/path/name', 'parser/type/syscall/syscall'], [self.stream_printer_event_handler],
id_path_list=['parser/type/path/id', 'parser/type/syscall/id'], min_allowed_time_diff=min_allowed_time_diff,
auto_include_flag=False, allow_missing_values_flag=True, persistence_id='audit_type_path', output_log_line=False)
self.analysis_context.register_component(new_match_id_value_combo_detector, description)
for i, log_atom in enumerate(log_atoms):
self.assertTrue(new_match_id_value_combo_detector.receive_atom(log_atom))
self.assertEqual(self.output_stream.getvalue() == "", output_stream_empty_results[i], log_atom.raw_data)
self.assertEqual(new_match_id_value_combo_detector.id_dict_current, id_dict_current_results[i])
self.assertEqual(new_match_id_value_combo_detector.id_dict_old, id_dict_old_results[i])
self.assertEqual(new_match_id_value_combo_detector.known_values, [])
self.reset_output_stream()
