class TimeCorrelationDetectorTest(TestBase):
__expected_string = '%s Correlation report\nTimeCorrelationDetector: "%s" (%d lines)\n '
string = b'25537 uid=2'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
fixed_dme = FixedDataModelElement('s1', string)
decimal_integer_value_me = DecimalIntegerValueModelElement('d1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_first_match_me = MatchContext(string)
first_match_me = FirstMatchModelElement('f1', [fixed_dme, decimal_integer_value_me])
match_element_first_match_me = first_match_me.get_match_element('first', match_context_first_match_me)
match_context_first_match_me2 = MatchContext(string)
first_match_me2 = FirstMatchModelElement('f2', [decimal_integer_value_me, fixed_dme])
match_element_first_match_me2 = first_match_me2.get_match_element('second', match_context_first_match_me2)
def test1_normal_report(self):
"""This test case unit the creation of a report. As the rules are chosen randomly this test can not be very specific in checking
the actual values of the report."""
description = "Test1TimeCorrelationDetector"
time_correlation_detector = TimeCorrelationDetector(self.aminer_config, 2, 1, 0, [self.stream_printer_event_handler],
record_count_before_event=10)
self.analysis_context.register_component(time_correlation_detector, component_name=description)
t = time.time()
for i in range(0, 10):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t).strftime(self.datetime_format_string), description, 10)))
self.reset_output_stream()
for i in range(0, 10):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t + 9).strftime(self.datetime_format_string), description, 20)))
self.reset_output_stream()
for i in range(10, 15):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
logAtomSequenceME2 = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me2), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME2)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t + 14).strftime(self.datetime_format_string), description, 30)))
def run_time_correlation_detector(self, number_of_rules):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
time_correlation_detector = TimeCorrelationDetector(
self.aminer_config,
2,
number_of_rules,
0, [self.stream_printer_event_handler],
record_count_before_event=self.waiting_time * 9000)
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
time_correlation_detector)
time_correlation_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
time_correlation_detector.__class__.__name__, avg, results,
'testCount=%d.' % number_of_rules)
def test2receive_atoms_with_defined_path_list(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is set to a static list of paths and variable_key_list should not be used.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler],
path_list=['parser/type/path/nametype'])
results = [
True, False, True, False, True, False, True, True, False, False,
True, True, False, True, False, True, False, True, False, False,
True
]
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(
line,
ParserMatch(
self.parsing_model.get_match_element(
'parser', MatchContext(line))), t,
self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
old_vals = (event_type_detector.num_events,
event_type_detector.num_eventlines,
event_type_detector.total_records,
event_type_detector.longest_path)
self.assertEqual(event_type_detector.receive_atom(log_atom),
not results[i], i)
if results[i]:
self.assertEqual(old_vals, (event_type_detector.num_events,
event_type_detector.num_eventlines,
event_type_detector.total_records,
event_type_detector.longest_path))
def run_new_match_path_detector(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
t = round(time.time(), 3)
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
new_match_path_detector)
new_match_path_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
new_match_path_detector.__class__.__name__, avg, results,
self.different_pathes % number_of_pathes)
def run_match_value_stream_writer(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
path_list = []
parsing_model = []
while i < number_of_pathes / 2:
path_list.append('match/integer/d' + str(i % number_of_pathes))
path_list.append('match/integer/s' + str(i % number_of_pathes))
parsing_model.append(
DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE))
parsing_model.append(
FixedDataModelElement('s' + str(i % number_of_pathes),
b' Euro '))
i = i + 1
sequence_model_element = SequenceModelElement(
'integer', parsing_model)
match_value_stream_writer = MatchValueStreamWriter(
self.output_stream, path_list, b';', b'-')
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
data = b''
p = process_time()
for j in range(
1,
int(number_of_pathes / 2) + number_of_pathes % 2 + 1):
data = data + str(j).encode() + b' Euro '
seconds = seconds + process_time() - p
match_context = MatchContext(data)
match_element = sequence_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
match_value_stream_writer.__class__.__name__, avg, results,
self.different_pathes % number_of_pathes)
def run_atom_filters_match_value_filter(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
subhandler_filter = SubhandlerFilter([],
stop_when_handled_flag=True)
i = 0
dictionary = {}
while i < 1000000:
dictionary[i] = new_match_path_detector
i = i + 1
i = 0
while i < number_of_pathes:
match_value_filter = MatchValueFilter(
self.integerd + str(i % number_of_pathes), dictionary,
None)
subhandler_filter.add_handler(match_value_filter,
stop_when_handled_flag=True)
i = i + 1
t = round(time.time(), 3)
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
match_value_filter)
subhandler_filter.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
subhandler_filter.__class__.__name__, avg, results,
'%d different %ss with a dictionary of %ss.' %
(number_of_pathes, match_value_filter.__class__.__name__,
new_match_path_detector.__class__.__name__))
def run_whitelist_violation_detector(self, number_of_pathes,
modulo_factor):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
rules = []
while i < number_of_pathes:
rules.append(
PathExistsMatchRule(
self.integerd + str(i % number_of_pathes), None))
i = i + 1
whitelist_violation_detector = WhitelistViolationDetector(
self.aminer_config, rules, [self.stream_printer_event_handler])
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
p = process_time()
r = random.randint(1, 100)
if r >= modulo_factor:
r = 2
else:
r = 1
seconds = seconds + process_time() - p
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % (number_of_pathes * r)),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
whitelist_violation_detector)
whitelist_violation_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
whitelist_violation_detector.__class__.__name__, avg, results,
'%d different PathExistsMatchRules and a moduloFactor of %d.' %
(number_of_pathes, modulo_factor))
def run_missing_match_path_value_detector(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
path_list = []
while i < number_of_pathes:
path_list.append(self.integerd + str(i % number_of_pathes))
i = i + 1
missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, path_list,
[self.stream_printer_event_handler], 'Default', True, 3600,
86400)
seconds = time.time()
t = seconds
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(1).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
p = process_time()
r = random.randint(0, 100)
seconds = seconds + process_time() - p
delta = int(r / 100)
t = t + 4000 * delta
log_atom = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
missing_match_path_list_value_detector)
missing_match_path_list_value_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
missing_match_path_list_value_detector.__class__.__name__, avg,
results, self.different_pathes % number_of_pathes)
def test1receive_atoms_with_default_values(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is empty and all paths are learned dynamically in variable_key_list.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler])
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(
line,
ParserMatch(
self.parsing_model.get_match_element(
'parser', MatchContext(line))), t,
self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.total_records, i + 1)
def run_timestamps_unsorted_detector(self, reset_factor):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
timestamps_unsorted_detector = TimestampsUnsortedDetector(
self.aminer_config, [self.stream_printer_event_handler])
seconds = time.time()
s = seconds
i = 0
mini = 100
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
p = process_time()
r = random.randint(1, 100)
seconds = seconds + process_time() - p
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element),
s + min(r, mini),
timestamps_unsorted_detector)
timestamps_unsorted_detector.receive_atom(log_atom)
if mini > r:
mini = r
else:
mini = mini + reset_factor
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
timestamps_unsorted_detector.__class__.__name__, avg, results,
'a resetFactor of %f.' % reset_factor)
def run_timestamp_correction_filters(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[new_match_path_detector])
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
p = process_time()
r = random.randint(1, 1000000)
seconds = seconds + process_time() - p
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), seconds - r,
simple_monotonic_timestamp_adjust)
simple_monotonic_timestamp_adjust.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
simple_monotonic_timestamp_adjust.__class__.__name__, avg, results,
'a %s and %d different path(es).' %
(new_match_path_detector.__class__.__name__, number_of_pathes))
class EnhancedNewMatchPathValueComboDetectorTest(TestBase):
__expected_string = '%s New value combination(s) detected\n%s: "%s" (%d lines)\n%s\n\n'
__expected_whitelisting_string = 'Whitelisted path(es) %s with %s in %s'
fixed_dme = FixedDataModelElement('s1', b'25537 uid=')
fixed_dme2 = FixedDataModelElement('s2', b' uid=2')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_sequence_me = MatchContext(b'25537 uid=2')
seq = SequenceModelElement('seq', [fixed_dme, decimal_integer_value_me])
match_element_sequence_me = seq.get_match_element(
'first', match_context_sequence_me)
match_context_sequence_me2 = MatchContext(b'25537 uid=2')
seq2 = SequenceModelElement('seq2', [decimal_integer_value_me, fixed_dme2])
match_element_sequence_me2 = seq2.get_match_element(
'second', match_context_sequence_me2)
first_seq_s1 = 'first/seq/s1'
first_seq_d1 = 'first/seq/d1'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
exp_str = " first/seq: b'25537 uid=2'\n " + first_seq_s1 + ": b'25537 uid='\n " + first_seq_d1 + \
": 2\n{(b'25537 uid=', 2): [%s, %s, 1]}"
exp_str2 = " {(b'25537 uid=', 2): [%s, %s, 1]}\nb'25537 uid=2'"
def test1_log_atom_not_known(self):
"""This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found. The
output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be repeatable
on second run."""
description = "Test1EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.exp_str2 % (t, t)))
self.reset_output_stream()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t + 2,
enhanced_new_match_path_value_combo_detector)
# repeating should produce the same result with new extraData.
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 2).strftime(
self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1,
" {(b'25537 uid=', 2): [%s, %s, 2]}\nb'25537 uid=2'" %
(t, t + 2)))
self.reset_output_stream()
enhanced_new_match_path_value_combo_detector2 = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(
enhanced_new_match_path_value_combo_detector2.receive_atom(
log_atom_sequence_me2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(25537, b' uid=2'): [%s, %s, 1]}\nb'25537 uid=2'" % (t, t)))
def test2_log_atom_known(self):
"""This test case checks the functionality of the auto_include_flag. If the same MatchElement is processed a second time and the
auto_include_flag was True, no event must be triggered."""
description = "Test2EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.exp_str2 % (t, t)))
self.reset_output_stream()
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
# repeating should NOT produce the same result, only persist the new extraData.
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
enhanced_new_match_path_value_combo_detector2 = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(
enhanced_new_match_path_value_combo_detector2.receive_atom(
log_atom_sequence_me2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(25537, b' uid=2'): [%s, %s, 1]}\nb'25537 uid=2'" % (t, t)))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.exp_str2 % (t, t)))
enhanced_new_match_path_value_combo_detector.do_persist()
self.reset_output_stream()
other_enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
other_enhanced_new_match_path_value_combo_detector,
description + "2")
other_log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t + 2,
other_enhanced_new_match_path_value_combo_detector)
self.assertTrue(
other_enhanced_new_match_path_value_combo_detector.receive_atom(
other_log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 2).strftime(
self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(b'25537 uid=', 2): [%s, %s, 2]}\nb'25537 uid=2'" %
(t, t + 2)))
self.reset_output_stream()
other_log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t + 5,
other_enhanced_new_match_path_value_combo_detector)
self.assertTrue(
other_enhanced_new_match_path_value_combo_detector.receive_atom(
other_log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 5).strftime(
self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(b'25537 uid=', 2): [%s, %s, 3]}\nb'25537 uid=2'" %
(t, t + 5)))
def test4_whitelist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test4EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertEqual(
enhanced_new_match_path_value_combo_detector.whitelist_event(
'Analysis.%s' % enhanced_new_match_path_value_combo_detector.
__class__.__name__, [
log_atom_sequence_me,
[self.match_element_sequence_me.get_path()]
], [
log_atom_sequence_me,
self.match_element_sequence_me.get_path()
], None), self.__expected_whitelisting_string %
(', '.join(
enhanced_new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me.get_path(), log_atom_sequence_me))
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector)
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
self.assertEqual(
enhanced_new_match_path_value_combo_detector.whitelist_event(
'Analysis.%s' % enhanced_new_match_path_value_combo_detector.
__class__.__name__, [
log_atom_sequence_me2,
[self.match_element_sequence_me2.get_path()]
], [
log_atom_sequence_me2,
self.match_element_sequence_me2.get_path()
], None), self.__expected_whitelisting_string %
(', '.join(
enhanced_new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me2.path, log_atom_sequence_me2))
def test5save_metadata(self):
"""This test case checks the correctness of the metadata informations"""
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['first/f1/s1'],
[self.stream_printer_event_handler],
'Default',
False,
True,
None,
output_log_line=False)
t = 1
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me)
self.assertEqual(
enhanced_new_match_path_value_combo_detector.known_values_dict.get(
(self.fixed_dme.fixed_data, (t, t, 1))), None)
def consumeData(self, streamData, endOfStreamFlag=False):
"""Consume data from the underlying stream for atomizing.
@return the number of consumed bytes, 0 if the atomizer would
need more data for a complete atom or -1 when no data was
consumed at the moment but data might be consumed later on."""
# Loop until as much streamData as possible was processed and
# then return a result. The correct processing of endOfStreamFlag
# is tricky: by default, even when all data was processed, do
# one more iteration to handle also the flag.
consumedLength = 0
while True:
if self.lastUnconsumedLogAtom != None:
# Keep length before dispatching: dispatch will reset the field.
dataLength = len(self.lastUnconsumedLogAtom.rawData)
if self.dispatchAtom(self.lastUnconsumedLogAtom):
consumedLength += dataLength + 1
continue
# Nothing consumed, tell upstream to wait if appropriate.
if consumedLength == 0:
consumedLength = -1
break
lineEnd = streamData.find(b'\n', consumedLength)
if self.inOverlongLineFlag:
if lineEnd < 0:
consumedLength = len(streamData)
if endOfStreamFlag:
self.dispatchEvent(
'Overlong line terminated by end of stream',
streamData)
self.inOverlongLineFlag = False
break
consumedLength = lineEnd + 1
self.inOverlongLineFlag = False
continue
# This is the valid start of a normal/incomplete/overlong line.
if lineEnd < 0:
tailLength = len(streamData) - consumedLength
if tailLength > self.maxLineLength:
self.dispatchEvent('Start of overlong line detected',
streamData[consumedLength:])
self.inOverlongLineFlag = True
consumedLength = len(streamData)
# Stay in loop to handle also endOfStreamFlag!
continue
if endOfStreamFlag and (tailLength != 0):
self.dispatchEvent('Incomplete last line',
streamData[consumedLength:])
consumedLength = len(streamData)
break
# This is at least a complete/overlong line.
lineLength = lineEnd + 1 - consumedLength
if lineLength > self.maxLineLength:
self.dispatchEvent('Overlong line detected',
streamData[consumedLength:lineEnd])
consumedLength = lineEnd + 1
continue
# This is a normal line.
lineData = streamData[consumedLength:lineEnd]
logAtom = LogAtom(lineData, None, None, self)
if self.parsingModel != None:
matchContext = MatchContext(lineData)
matchElement = self.parsingModel.getMatchElement(
'', matchContext)
if (matchElement != None) and not matchContext.matchData:
logAtom.parserMatch = ParserMatch(matchElement)
if self.defaultTimestampPath != None:
tsMatch = logAtom.parserMatch.getMatchDictionary().get(
self.defaultTimestampPath, None)
if tsMatch != None:
logAtom.setTimestamp(tsMatch.matchObject[1])
if self.dispatchAtom(logAtom):
consumedLength = lineEnd + 1
continue
if consumedLength == 0:
# Downstream did not want the data, so tell upstream to block
# for a while.
consumedLength = -1
break
return consumedLength
class ParserCountTest(TestBase):
match_context_m1 = MatchContext(b'First string')
match_context_m2 = MatchContext(b' to match.')
match_context_m3 = MatchContext(b'some completely other string to match.')
match_context_seq = MatchContext(b'First string to match.')
fixed_dme_m1 = FixedDataModelElement('m1', b'First string')
fixed_dme_m2 = FixedDataModelElement('m2', b' to match.')
seq = SequenceModelElement('seq', [fixed_dme_m1, fixed_dme_m2])
fixed_dme_m3 = FixedDataModelElement(
'm3', b'some completely other string to match.')
match_element_m1 = fixed_dme_m1.get_match_element('fixed',
match_context_m1)
match_element_m2 = fixed_dme_m2.get_match_element('fixed',
match_context_m2)
match_element_m3 = fixed_dme_m3.get_match_element('fixed',
match_context_m3)
match_element_seq = seq.get_match_element('fixed', match_context_seq)
def test1log_atom_not_in_path_list(self):
"""This unittest checks if no action happens, when no path in the match_dictionary matches a target_path."""
parser_count = ParserCount(
self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.fixed_dme_m3.fixed_data,
ParserMatch(self.match_element_m3), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test2log_atom_matches_single_path(self):
"""This unittest tests the receive_atom method with a single path matching."""
parser_count = ParserCount(
self.aminer_config,
['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.fixed_dme_m3.fixed_data,
ParserMatch(self.match_element_m3), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/m3'] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test3log_atom_matches_multiple_paths(self):
"""This unittest tests the receive_atom method with multiple paths matching."""
parser_count = ParserCount(
self.aminer_config,
['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.match_context_seq.match_data,
ParserMatch(self.match_element_seq), t,
parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/seq'] = 1
old_count_dict['fixed/seq/m1'] = 1
old_count_dict['fixed/seq/m2'] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test4do_timer(self):
"""This unittest checks if the do_timer method works properly."""
parser_count = ParserCount(self.aminer_config, ['fixed/m3'],
[self.stream_printer_event_handler], 600)
t = time.time()
self.assertEqual(int(parser_count.do_timer(t + 100)), 600)
self.assertEqual(self.output_stream.getvalue(), "")
log_atom = LogAtom(self.match_context_seq.match_data,
ParserMatch(self.match_element_seq), t,
parser_count)
parser_count.receive_atom(log_atom)
self.assertEqual(int(parser_count.do_timer(t + 100)), 500)
self.assertEqual(self.output_stream.getvalue(), "")
self.assertEqual(parser_count.do_timer(t + 601), 600)
self.assertNotEqual(self.output_stream.getvalue(), "")
self.reset_output_stream()
def test5reset_after_report_flag(self):
"""This unittest tests the functionality of the reset_after_report flag."""
parser_count = ParserCount(
self.aminer_config,
['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler], 600, False)
parser_count.count_dict['fixed/seq'] = 5
parser_count.count_dict['fixed/seq/m1'] = 5
parser_count.count_dict['fixed/seq/m2'] = 5
parser_count.count_dict['fixed/m3'] = 17
old_count_dict = dict(parser_count.count_dict)
parser_count.send_report()
self.assertEqual(parser_count.count_dict, old_count_dict)
parser_count.reset_after_report_flag = True
parser_count.send_report()
old_count_dict['fixed/seq'] = 0
old_count_dict['fixed/seq/m1'] = 0
old_count_dict['fixed/seq/m2'] = 0
old_count_dict['fixed/m3'] = 0
self.assertEqual(parser_count.count_dict, old_count_dict)
def test6receive_atom_without_target_paths(self):
"""This unittest tests the receive_atom method with multiple paths matching without having target_paths specified."""
parser_count = ParserCount(self.aminer_config, None,
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.match_context_seq.match_data,
ParserMatch(self.match_element_seq), t,
parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/seq'] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test1receive_match_in_time_with_auto_include_flag(self):
"""This test case checks if log_atoms are accepted as expected with the auto_include_flag=True."""
description = 'test1newMatchIdValueComboDetectorTest'
output_stream_empty_results = [True, False, True, False, True, False, True, True, True, True, True, True, True, True, False, False,
False, True, False, True, False]
id_dict_current_results = [
{1: {'parser/type/syscall/syscall': 1}}, {}, {2: {'parser/type/syscall/syscall': 2}}, {},
{3: {'parser/type/syscall/syscall': 3}}, {}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 4: {'parser/type/syscall/syscall': 1}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 5: {'parser/type/path/name': 'two'}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 6: {'parser/type/syscall/syscall': 4}},
{100: {'parser/type/syscall/syscall': 1}, 6: {'parser/type/syscall/syscall': 4}, 7: {'parser/type/path/name': 'five'}},
{100: {'parser/type/syscall/syscall': 1}, 6: {'parser/type/syscall/syscall': 4}, 7: {'parser/type/path/name': 'five'},
8: {'parser/type/syscall/syscall': 6}},
{100: {'parser/type/syscall/syscall': 1}, 7: {'parser/type/path/name': 'five'}, 8: {'parser/type/syscall/syscall': 6}},
{100: {'parser/type/syscall/syscall': 1}, 8: {'parser/type/syscall/syscall': 6}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 9: {'parser/type/syscall/syscall': 2}}, {100: {'parser/type/syscall/syscall': 1}},
{100: {'parser/type/syscall/syscall': 1}, 10: {'parser/type/path/name': 'one'}}, {100: {'parser/type/syscall/syscall': 1}}]
id_dict_old_results = [{}] * 21
min_allowed_time_diff = 0.1
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(line, ParserMatch(self.parsing_model.get_match_element('parser', MatchContext(line))), t, self.__class__.__name__))
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(self.aminer_config, [
'parser/type/path/name', 'parser/type/syscall/syscall'], [self.stream_printer_event_handler],
id_path_list=['parser/type/path/id', 'parser/type/syscall/id'], min_allowed_time_diff=min_allowed_time_diff,
auto_include_flag=True, allow_missing_values_flag=True, persistence_id='audit_type_path', output_log_line=False)
self.analysis_context.register_component(new_match_id_value_combo_detector, description)
for i, log_atom in enumerate(log_atoms):
self.assertTrue(new_match_id_value_combo_detector.receive_atom(log_atom))
self.assertEqual(self.output_stream.getvalue() == "", output_stream_empty_results[i], log_atom.raw_data)
self.assertEqual(new_match_id_value_combo_detector.id_dict_current, id_dict_current_results[i])
self.assertEqual(new_match_id_value_combo_detector.id_dict_old, id_dict_old_results[i])
self.reset_output_stream()
class NewMatchPathDetectorTest(TestBase):
__expected_string = '%s New path(es) detected\n%s: "%s" (%d lines)\n %s\n%s\n\n'
match_path_s1 = "['/s1']"
match_path_d1 = "['/d1']"
datetime_format_string = '%Y-%m-%d %H:%M:%S'
analysis = 'Analysis.%s'
pid = "b' pid='"
uid = "b' uid=2'"
match_context_fixed_dme = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
match_element_fixed_dme = fixed_dme.get_match_element("", match_context_fixed_dme)
match_context_decimal_integer_value_me = MatchContext(b'25537 uid=2')
decimal_integer_value_me = DecimalIntegerValueModelElement('d1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_element_decimal_integer_value_me = decimal_integer_value_me.get_match_element("", match_context_decimal_integer_value_me)
def test1_log_atom_not_known(self):
"""This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found. The
output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be repeatable
on second run."""
description = "Test1NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t, new_match_path_detector)
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# repeating should produce the same result
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# other MatchElement
self.assertTrue(new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_d1, self.uid))
def test2_log_atom_known(self):
"""This test case checks the functionality of the autoIncludeFlag. If the same MatchElement is processed a second time and the
auto_include_flag was True, no event must be triggered."""
description = "Test2NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t, new_match_path_detector)
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
self.reset_output_stream()
# repeating should NOT produce the same result
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
# other MatchElement
self.assertTrue(new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_d1, self.uid))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
self.assertTrue(new_match_path_detector.receive_atom(log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_detector.__class__.__name__, description, 1,
self.match_path_s1, self.pid))
new_match_path_detector.do_persist()
self.reset_output_stream()
otherNewMatchPathDetector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False,
output_log_line=False)
otherLogAtomFixedDME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, otherNewMatchPathDetector)
self.assertTrue(otherNewMatchPathDetector.receive_atom(otherLogAtomFixedDME))
self.assertEqual(self.output_stream.getvalue(), '')
def test4GetTimeTriggerClass(self):
"""The known paths are to be periodically stored after a certain time. This requires a synchronization class. The return of the
correct class is to be checked in this test case."""
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.assertEqual(new_match_path_detector.get_time_trigger_class(), 1)
"""The following test cases should check if the doTimer() method is working properly.This includes the updating of nextPersistTime.
As it is not updated directly in the method this test cases are not correct. Due to that they are commented."""
# '''
# During initialization, the next time is not determined (the value is initialized with None).
# In this case, the persistence is expected to occur after 600 milliseconds.
# '''
# def test5_do_timer_next_persist_time_none(self):
# self.new_match_path_detector = NewMatchPathDetector(self.aminer_config,
# [self.stream_printer_event_handler], 'Default', True, output_log_line=False)
# self.assertEqual(self.new_match_path_detector.do_timer(200), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(400), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(10000), 600)
#
# '''
# If the NextPersistTime is less than or equal to zero, the data must be saved.
# '''
# def test6_do_timer_delta_smaller_or_equal_zero(self):
# self.new_match_path_detector = NewMatchPathDetector(self.aminer_config,
# [self.stream_printer_event_handler], 'Default', True, output_log_line=False)
# self.new_match_path_detector.nextPersistTime = 400
# self.assertEqual(self.new_match_path_detector.do_timer(400), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(1000), 600)
#
# '''
# If the delta does not fall below the limit value, only the delta value should be returned.
# '''
# def test7_do_timer_delta_greater_zero(self):
# #this test fails due to the missing update of the nextPersistTime variable in the doTimer method
# self.new_match_path_detector = NewMatchPathDetector(self.aminer_config,
# [self.stream_printer_event_handler], 'Default', True, output_log_line=False)
# self.new_match_path_detector.nextPersistTime = 400
# self.assertEqual(self.new_match_path_detector.do_timer(200), 200)
# self.assertEqual(self.new_match_path_detector.do_timer(200), 600)
# self.assertEqual(self.new_match_path_detector.do_timer(100), 500)
def test8_whitelist_event_type_exception(self):
"""This test case checks whether an exception is thrown when entering an event of another class."""
description = "Test8NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [], [self.stream_printer_event_handler],
'Default', True, True)
self.assertRaises(
Exception, new_match_path_detector.whitelist_event, self.analysis % new_match_path_value_combo_detector.__class__.__name__,
log_atom_fixed_dme.raw_data, self.output_stream.getvalue(), None)
def test9WhitelistEventWhitelistingDataException(self):
"""The NewMatchPathDetector can not handle whitelisting data and therefore an exception is expected."""
description = "Test9NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
self.assertRaises(Exception, new_match_path_detector.whitelist_event, self.analysis % new_match_path_detector.__class__.__name__,
log_atom_fixed_dme.raw_data, self.output_stream.getvalue(), ['random', 'Data'])
def test10WhitelistEventWithKnownAndUnknownPaths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test10NewMatchPathDetector"
new_match_path_detector = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', True,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector, description)
t = round(time.time(), 3)
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_fixed_dme), t, new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
self.assertEqual(new_match_path_detector.whitelist_event(self.analysis % new_match_path_detector.__class__.__name__, [
log_atom_fixed_dme, [self.match_element_fixed_dme.get_path()]],
[log_atom_fixed_dme, [self.match_element_fixed_dme.get_path()]], None), 'Whitelisted path(es) in %s' % log_atom_fixed_dme)
log_atom_decimal_integer_value_me = LogAtom(self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t, new_match_path_detector)
new_match_path_detector.auto_include_flag = False
self.assertEqual(new_match_path_detector.whitelist_event(self.analysis % new_match_path_detector.__class__.__name__, [
log_atom_decimal_integer_value_me, [self.match_element_decimal_integer_value_me.get_path()]],
[log_atom_decimal_integer_value_me, [self.match_element_decimal_integer_value_me.get_path()]], None),
'Whitelisted path(es) %s in %s' % (self.match_element_decimal_integer_value_me.path, log_atom_decimal_integer_value_me))
class ParserCountTest(TestBase):
"""Unittests for the ParserCount."""
match_context_m1 = MatchContext(b'First string')
match_context_m2 = MatchContext(b' to match.')
match_context_m3 = MatchContext(b'some completely other string to match.')
match_context_seq = MatchContext(b'First string to match.')
fixed_dme_m1 = FixedDataModelElement('m1', b'First string')
fixed_dme_m2 = FixedDataModelElement('m2', b' to match.')
seq = SequenceModelElement('seq', [fixed_dme_m1, fixed_dme_m2])
fixed_dme_m3 = FixedDataModelElement('m3', b'some completely other string to match.')
match_element_m1 = fixed_dme_m1.get_match_element('fixed', match_context_m1)
match_element_m2 = fixed_dme_m2.get_match_element('fixed', match_context_m2)
match_element_m3 = fixed_dme_m3.get_match_element('fixed', match_context_m3)
match_element_seq = seq.get_match_element('fixed', match_context_seq)
def test1log_atom_not_in_path_list(self):
"""This unittest checks if no action happens, when no path in the match_dictionary matches a target_path."""
parser_count = ParserCount(self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2'], [self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.fixed_dme_m3.fixed_data, ParserMatch(self.match_element_m3), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test2log_atom_matches_single_path(self):
"""This unittest tests the receive_atom method with a single path matching."""
parser_count = ParserCount(self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.fixed_dme_m3.fixed_data, ParserMatch(self.match_element_m3), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/m3'][current_processed_lines_str] = 1
old_count_dict['fixed/m3'][total_processed_lines_str] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test3log_atom_matches_multiple_paths(self):
"""This unittest tests the receive_atom method with multiple paths matching."""
parser_count = ParserCount(self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.match_context_seq.match_data, ParserMatch(self.match_element_seq), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/seq'][current_processed_lines_str] = 1
old_count_dict['fixed/seq'][total_processed_lines_str] = 1
old_count_dict['fixed/seq/m1'][current_processed_lines_str] = 1
old_count_dict['fixed/seq/m1'][total_processed_lines_str] = 1
old_count_dict['fixed/seq/m2'][current_processed_lines_str] = 1
old_count_dict['fixed/seq/m2'][total_processed_lines_str] = 1
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test4do_timer(self):
"""This unittest checks if the do_timer method works properly."""
parser_count = ParserCount(self.aminer_config, ['fixed/m3'], [self.stream_printer_event_handler], 600)
t = time.time()
self.assertEqual(int(parser_count.do_timer(t + 100)), 600)
self.assertEqual(self.output_stream.getvalue(), "")
log_atom = LogAtom(self.match_context_seq.match_data, ParserMatch(self.match_element_seq), t, parser_count)
parser_count.receive_atom(log_atom)
self.assertEqual(int(parser_count.do_timer(t + 100)), 500)
self.assertEqual(self.output_stream.getvalue(), "")
self.assertEqual(parser_count.do_timer(t + 601), 600)
self.assertNotEqual(self.output_stream.getvalue(), "")
self.reset_output_stream()
def test5resetting(self):
"""This unittest tests the functionality of resetting the counts."""
parser_count = ParserCount(self.aminer_config, ['fixed/seq', 'fixed/seq/m1', 'fixed/seq/m2', 'fixed/m3'],
[self.stream_printer_event_handler], 600)
parser_count.count_dict['fixed/seq'][current_processed_lines_str] = 5
parser_count.count_dict['fixed/seq'][total_processed_lines_str] = 5
parser_count.count_dict['fixed/seq/m1'][current_processed_lines_str] = 5
parser_count.count_dict['fixed/seq/m1'][total_processed_lines_str] = 5
parser_count.count_dict['fixed/seq/m2'][current_processed_lines_str] = 5
parser_count.count_dict['fixed/seq/m2'][total_processed_lines_str] = 5
parser_count.count_dict['fixed/m3'][current_processed_lines_str] = 17
parser_count.count_dict['fixed/m3'][total_processed_lines_str] = 17
old_count_dict = dict(parser_count.count_dict)
parser_count.send_report()
self.assertEqual(parser_count.count_dict, old_count_dict)
parser_count.send_report()
old_count_dict['fixed/seq'][current_processed_lines_str] = 0
old_count_dict['fixed/seq/m1'][current_processed_lines_str] = 0
old_count_dict['fixed/seq/m2'][current_processed_lines_str] = 0
old_count_dict['fixed/m3'][current_processed_lines_str] = 0
self.assertEqual(parser_count.count_dict, old_count_dict)
def test6receive_atom_without_target_paths(self):
"""This unittest tests the receive_atom method with multiple paths matching without having target_paths specified."""
parser_count = ParserCount(self.aminer_config, None, [self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(self.match_context_seq.match_data, ParserMatch(self.match_element_seq), t, parser_count)
old_count_dict = dict(parser_count.count_dict)
old_count_dict['fixed/seq'] = {current_processed_lines_str: 1, total_processed_lines_str: 1}
parser_count.receive_atom(log_atom)
self.assertEqual(parser_count.count_dict, old_count_dict)
def test7initialize_errored_target_label_list(self):
"""Initialize the ParserCount class with errored target_label_list parameters and check if an error is raised."""
self.assertRaises(ValueError, ParserCount, self.aminer_config, None, [self.stream_printer_event_handler], target_label_list=['p'])
self.assertRaises(ValueError, ParserCount, self.aminer_config, ['path1', 'path2'], [self.stream_printer_event_handler],
target_label_list=['p'])
self.assertRaises(ValueError, ParserCount, self.aminer_config, ['path1'], [self.stream_printer_event_handler],
target_label_list=['p1', 'p2'])
ParserCount(self.aminer_config, ['path'], [self.stream_printer_event_handler], target_label_list=['p'])
def run_time_correlation_violation_detector(self, chance):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
correlation_rule = CorrelationRule('Correlation',
0,
chance,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[
('/integer/d0',
'/integer/d1')
])
a_class_selector = EventClassSelector('Selector1',
[correlation_rule], None)
b_class_selector = EventClassSelector('Selector2', None,
[correlation_rule])
rules = [
Rules.PathExistsMatchRule('/integer/d0', a_class_selector),
Rules.PathExistsMatchRule('/integer/d1', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
self.analysis_context.aminer_config, rules,
[self.stream_printer_event_handler])
seconds = time.time()
s = seconds
i = 0
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd0', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
while int(time.time() - seconds) < self.waiting_time:
integer = '/integer'
p = process_time()
r = random.randint(1, 100)
seconds = seconds + process_time() - p
decimal_integer_value_me1 = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
integer, match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), s,
time_correlation_violation_detector)
time_correlation_violation_detector.receive_atom(log_atom)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me1.get_match_element(
integer, match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), s + r / 100,
time_correlation_violation_detector)
time_correlation_violation_detector.receive_atom(log_atom)
s = s + r / 100
if r / 100 >= chance:
p = process_time()
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
integer, match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), s,
time_correlation_violation_detector)
time_correlation_violation_detector.receive_atom(log_atom)
seconds = seconds + process_time() - p
time_correlation_violation_detector.do_timer(s)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
time_correlation_violation_detector.__class__.__name__, avg,
results, '%d%% chance of not finding an element' %
((1 - chance) * 100))
class KafkaEventHandlerTest(TestBase):
output_log_line = True
kafka_topic = 'test_topic'
kafka_group = 'test_group'
consumer = None
match_context = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
other_data = 4
match_element = fixed_dme.get_match_element("match", match_context)
description = 'jsonConverterHandlerDescription'
t = time.time()
persistence_id = 'Default'
test_detector = 'Analysis.TestDetector'
event_message = 'An event happened!'
sorted_log_lines = ['Event happend at /path/ 5 times.', '', '', '', '']
expected_string = '%s %s\n%s: "%s" (5 lines)\n {\n "AnalysisComponent": {\n "AnalysisComponentIdentifier": 0,\n' \
' "AnalysisComponentType": "%s",\n "AnalysisComponentName": "%s",\n "Message": "%s",\n' \
' "PersistenceFileName": "%s",\n "AffectedParserPaths": [\n "test/path/1",\n' \
' "test/path/2"\n ]\n },\n "LogData": {\n "RawLogData": [\n " pid="\n ],\n ' \
'"Timestamps": [\n %s\n ],\n "LogLinesCount": 5,\n' \
' "AnnotatedMatchElement": "match/s1: b\' pid=\'"\n }%s\n}\n\n'
@classmethod
def setUpClass(cls):
cls.consumer = KafkaConsumer(cls.kafka_topic,
bootstrap_servers=['localhost:9092'],
enable_auto_commit=True,
consumer_timeout_ms=10000,
group_id=cls.kafka_group,
value_deserializer=lambda x: x.decode(),
api_version=(2, 0, 1),
auto_offset_reset='earliest')
@classmethod
def tearDownClass(cls):
cls.consumer.close()
def test1receive_serialized_data(self):
"""This unittest tests the receive_event method with serialized data from the JsonConverterHandler."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
output = self.output_stream.getvalue()
kafka_event_handler = KafkaEventHandler(
self.analysis_context, self.kafka_topic, {
'bootstrap_servers': ['localhost:9092'],
'api_version': (2, 0, 1)
})
self.assertTrue(
kafka_event_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, output,
log_atom, self))
self.assertEqual(
self.consumer.__next__().value, self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), ""))
def test2receive_non_serialized_data(self):
"""This unittest tests the receive_event method with not serialized data"""
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
kafka_event_handler = KafkaEventHandler(
self.analysis_context, self.kafka_topic, {
'bootstrap_servers': ['localhost:9092'],
'api_version': (2, 0, 1)
})
self.assertFalse(
kafka_event_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines,
event_data, log_atom, self))
self.assertRaises(StopIteration, self.consumer.__next__)
def run_new_match_id_value_combo_detector(self, min_allowed_time_diff):
log_lines = [
b'type=SYSCALL msg=audit(1580367384.000:1): arch=c000003e syscall=1 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367385.000:1): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 '
b'rdev=00:00 nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367386.000:2): arch=c000003e syscall=2 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367387.000:2): item=0 name="two" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367388.000:3): arch=c000003e syscall=3 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367389.000:3): item=0 name="three" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00'
b' nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367388.500:100): arch=c000003e syscall=1 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=SYSCALL msg=audit(1580367390.000:4): arch=c000003e syscall=1 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367391.000:4): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=PATH msg=audit(1580367392.000:5): item=0 name="two" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367393.000:5): arch=c000003e syscall=2 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=SYSCALL msg=audit(1580367394.000:6): arch=c000003e syscall=4 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367395.000:7): item=0 name="five" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367396.000:8): arch=c000003e syscall=6 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367397.000:6): item=0 name="four" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367398.000:7): arch=c000003e syscall=5 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367399.000:8): item=0 name="six" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367400.000:9): arch=c000003e syscall=2 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f '
b'items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) '
b'ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367401.000:9): item=0 name="three" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 '
b'rdev=00:00 nametype=NORMAL',
b'type=PATH msg=audit(1580367402.000:10): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 '
b'rdev=00:00 nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367403.000:10): arch=c000003e syscall=3 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 '
b'a3=4f items=1 ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 '
b'tty=(none) ses=4294967295 comm="apache2" exe="/usr/sbin/apache2" key=(null)'
]
parsing_model = FirstMatchModelElement('type', [
SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '),
FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'),
FixedDataModelElement('placeholder', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='),
DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'),
DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='),
DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='),
DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode'),
FixedDataModelElement('ouid_string', b' ouid='),
DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='),
DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='),
DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='),
FixedWordlistDataModelElement('nametype',
[b'NORMAL', b'ERROR'])
]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '),
FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'),
FixedDataModelElement('placeholder', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='),
DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='),
DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='),
FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='),
DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')
])
])
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
self.aminer_config,
['parser/type/path/name', 'parser/type/syscall/syscall'],
[self.stream_printer_event_handler],
id_path_list=['parser/type/path/id', 'parser/type/syscall/id'],
min_allowed_time_diff=min_allowed_time_diff,
auto_include_flag=False,
allow_missing_values_flag=True,
persistence_id='audit_type_path',
output_log_line=False)
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
p = process_time()
r = random.randint(0, len(log_lines) - 1)
seconds = seconds + process_time() - p
# this code just creates some data to be able to compare with other analysis components.
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
_match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
########################################################################################
line = log_lines[r]
log_atom = LogAtom(
line,
ParserMatch(
parsing_model.get_match_element(
'parser', MatchContext(line))), t,
self.__class__.__name__)
new_match_id_value_combo_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
new_match_id_value_combo_detector.__class__.__name__, avg, results,
'%.2f seconds min_allowed_time_diff.' % min_allowed_time_diff)
class JsonConverterHandlerTest(TestBase):
"""Unittests for the JsonConverterHandler."""
output_log_line = True
match_context = MatchContext(b' pid=')
fixed_dme = FixedDataModelElement('s1', b' pid=')
match_element = fixed_dme.get_match_element("match", match_context)
t = time.time()
test_detector = 'Analysis.TestDetector'
event_message = 'An event happened!'
sorted_log_lines = ['Event happend at /path/ 5 times.', '', '', '', '']
persistence_id = 'Default'
description = 'jsonConverterHandlerDescription'
expected_string = '%s %s\n%s: "%s" (5 lines)\n {\n "AnalysisComponent": {\n "AnalysisComponentIdentifier": 0,\n' \
' "AnalysisComponentType": "%s",\n "AnalysisComponentName": "%s",\n "Message": "%s",\n' \
' "PersistenceFileName": "%s",\n "AffectedParserPaths": [\n "test/path/1",\n' \
' "test/path/2"\n ]\n },\n "LogData": {\n "RawLogData": [\n " pid="\n ],\n ' \
'"Timestamps": [\n %s\n ],\n "LogLinesCount": 5,\n' \
' "AnnotatedMatchElement": "match/s1: b\' pid=\'"\n }%s\n}\n\n'
def test1receive_expected_event(self):
"""In this test case a normal Event happens and the json output should be sent to a StreamPrinterEventHandler."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2']
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
self.assertEqual(
self.output_stream.getvalue(), self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(self.t, 2), ""))
def test2receive_event_with_same_event_data_attributes(self):
"""In this test case an attribute of AnalysisComponent is overwritten and an JsonError attribute is expected."""
json_converter_handler = JsonConverterHandler(
[self.stream_printer_event_handler], self.analysis_context)
log_atom = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element), self.t, self)
self.analysis_context.register_component(self, self.description)
event_data = {
'AnalysisComponent': {
'AffectedParserPaths': ['test/path/1', 'test/path/2'],
'Message': 'An other event happened too!'
}
}
json_converter_handler.receive_event(self.test_detector,
self.event_message,
self.sorted_log_lines, event_data,
log_atom, self)
self.assertEqual(
self.output_stream.getvalue(), self.expected_string %
(datetime.fromtimestamp(self.t).strftime("%Y-%m-%d %H:%M:%S"),
self.event_message, self.__class__.__name__, self.description,
self.__class__.__name__, self.description, self.event_message,
self.persistence_id, round(float("%.2f" % self.t), 2),
',\n "JsonError": "AnalysisComponent attribute \'Message\' is already in use and can not be overwritten!\\n"'
))
class NewMatchPathValueComboDetectorTest(TestBase):
"""Unittests for the NewMatchPathValueComboDetector."""
__expected_string = '%s New value combination(s) detected\n%s: "%s" (%d lines)\n%s\n\n'
fixed_dme = FixedDataModelElement('s1', b'25537 uid=')
fixed_dme2 = FixedDataModelElement('s2', b' uid=2')
datetime_format_string = '%Y-%m-%d %H:%M:%S'
first_seq_s1 = 'first/seq/s1'
first_seq_d1 = 'first/seq/d1'
string = " first/seq: b'25537 uid=2'\n " + first_seq_s1 + ": b'25537 uid='\n " + first_seq_d1 + ": 2\n(b'25537 uid=', 2)"
string2 = " (b'25537 uid=', 2)\nb'25537 uid=2'"
decimal_integer_value_me = DecimalIntegerValueModelElement('d1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_sequence_me = MatchContext(b'25537 uid=2')
seq = SequenceModelElement('seq', [fixed_dme, decimal_integer_value_me])
match_element_sequence_me = seq.get_match_element('first', match_context_sequence_me)
match_context_sequence_me2 = MatchContext(b'25537 uid=2')
seq2 = SequenceModelElement('seq2', [decimal_integer_value_me, fixed_dme2])
match_element_sequence_me2 = seq2.get_match_element('second', match_context_sequence_me2)
def test1_log_atom_not_known(self):
"""
This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found.
The output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be
repeatable on second run.
"""
description = "Test1NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [self.first_seq_s1, self.first_seq_d1], [
self.stream_printer_event_handler], 'Default', False, False, output_log_line=False)
self.analysis_context.register_component(new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(self.match_element_sequence_me.get_match_string(), ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
self.reset_output_stream()
# repeating should produce the same result
self.assertTrue(new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
self.reset_output_stream()
new_match_path_value_combo_detector2 = NewMatchPathValueComboDetector(self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'], [
self.stream_printer_event_handler], 'Default', False, False, output_log_line=False)
self.analysis_context.register_component(new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(self.match_element_sequence_me2.get_match_string(), ParserMatch(self.match_element_sequence_me2), t,
new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(new_match_path_value_combo_detector2.receive_atom(log_atom_sequence_me2))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1, " (25537, b' uid=2')\nb'25537 uid=2'"))
def test2_log_atom_known(self):
"""
This test case checks the functionality of the auto_include_flag.
If the same MatchElement is processed a second time and the auto_include_flag was True, no event must be triggered.
"""
description = "Test2NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [self.first_seq_s1, self.first_seq_d1], [
self.stream_printer_event_handler], 'Default', False, True, output_log_line=False)
self.analysis_context.register_component(new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(self.match_element_sequence_me.get_match_string(), ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
self.reset_output_stream()
# repeating should NOT produce the same result
self.assertTrue(new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
new_match_path_value_combo_detector2 = NewMatchPathValueComboDetector(self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'], [
self.stream_printer_event_handler], 'Default', False, False, output_log_line=False)
self.analysis_context.register_component(new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(self.match_element_sequence_me2.get_match_string(), ParserMatch(self.match_element_sequence_me2), t,
new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(new_match_path_value_combo_detector2.receive_atom(log_atom_sequence_me2))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1, " (25537, b' uid=2')\nb'25537 uid=2'"))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [self.first_seq_s1, self.first_seq_d1], [
self.stream_printer_event_handler], 'Default', False, True, output_log_line=False)
self.analysis_context.register_component(new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(self.match_element_sequence_me.get_match_string(), ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
self.assertTrue(new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), self.__expected_string % (
datetime.fromtimestamp(t).strftime(self.datetime_format_string), new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.string2))
new_match_path_value_combo_detector.do_persist()
self.reset_output_stream()
other_new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [
self.first_seq_s1, self.first_seq_d1], [self.stream_printer_event_handler], 'Default', False, True, output_log_line=False)
self.analysis_context.register_component(other_new_match_path_value_combo_detector, description + "2")
other_log_atom_fixed_dme = LogAtom(self.match_element_sequence_me.get_match_string(), ParserMatch(self.match_element_sequence_me),
t, other_new_match_path_value_combo_detector)
self.assertTrue(other_new_match_path_value_combo_detector.receive_atom(other_log_atom_fixed_dme))
self.assertEqual(self.output_stream.getvalue(), '')
def test4_allowlist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test4NewMatchPathValueComboDetector"
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(self.aminer_config, [self.first_seq_s1, self.first_seq_d1], [
self.stream_printer_event_handler], 'Default', False, True, output_log_line=False)
self.analysis_context.register_component(new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(self.match_element_sequence_me.get_match_string(), ParserMatch(self.match_element_sequence_me), t,
new_match_path_value_combo_detector)
new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me)
self.assertEqual(
new_match_path_value_combo_detector.allowlist_event(
'Analysis.%s' % new_match_path_value_combo_detector.__class__.__name__,
self.match_element_sequence_me.get_path(), None), 'Allowlisted path(es) %s with %s.' % (
", ".join(new_match_path_value_combo_detector.target_path_list), self.match_element_sequence_me.get_path()))
new_match_path_value_combo_detector.auto_include_flag = False
self.assertEqual(
new_match_path_value_combo_detector.allowlist_event(
'Analysis.%s' % new_match_path_value_combo_detector.__class__.__name__, self.match_element_sequence_me2.get_path(),
None), 'Allowlisted path(es) %s with %s.' % (
", ".join(new_match_path_value_combo_detector.target_path_list), self.match_element_sequence_me2.path))