def test2no_timestamp_log_atom(self):
"""In this test case a LogAtom with no timestamp is received by the class."""
description = "Test2SimpleMultisourceAtomSync"
sync_wait_time = 3
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector1,
description)
simple_multisource_atom_sync = SimpleMultisourceAtomSync(
[new_match_path_detector1], sync_wait_time)
t = time()
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element(
'match', match_context)
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), None,
new_match_path_detector1)
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description, 1,
self.match_path, self.calculation))
def get_model():
"""Return a model to parse messages from kernel logging."""
type_children = [
SequenceModelElement('ipv4-martian', [
FixedDataModelElement('s0', b'IPv4: martian '),
FixedWordlistDataModelElement('direction',
[b'source', b'destination']),
FixedDataModelElement('s1', b' '),
IpAddressDataModelElement('destination'),
FixedDataModelElement('s2', b' from '),
IpAddressDataModelElement('source'),
FixedDataModelElement('s3', b', on dev '),
AnyByteDataModelElement('interface')
]),
SequenceModelElement('net-llheader', [
FixedDataModelElement('s0', b'll header: '),
AnyByteDataModelElement('data')
]),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('kernel', [
FixedDataModelElement('sname', b'kernel: ['),
DelimitedDataModelElement('timestamp', b']'),
FixedDataModelElement('s0', b'] '),
FirstMatchModelElement('msg', type_children)
])
return model
def test7string_regex_match_rule(self):
"""This case unit the StringRegexMatchRule."""
description = "Test7Rules"
string_regex_match_rule = StringRegexMatchRule(self.match_any,
re.compile(r'\w'), None)
self.analysis_context.register_component(string_regex_match_rule,
description)
any_byte_date_me = AnyByteDataModelElement('any')
match_context = MatchContext(self.alphabet)
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
string_regex_match_rule)
self.assertTrue(string_regex_match_rule.match(log_atom))
match_context = MatchContext(
'--> There are 26 letters in the english alphabet')
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
string_regex_match_rule)
self.assertTrue(not string_regex_match_rule.match(log_atom))
def get_model():
"""Return a model to parse messages from kernel logging."""
type_children = [
SequenceModelElement("ipv4-martian", [
FixedDataModelElement("s0", b"IPv4: martian "),
FixedWordlistDataModelElement("direction", [b"source", b"destination"]),
FixedDataModelElement("s1", b" "),
IpAddressDataModelElement("destination"),
FixedDataModelElement("s2", b" from "),
IpAddressDataModelElement("source"),
FixedDataModelElement("s3", b", on dev "),
AnyByteDataModelElement("interface")]),
SequenceModelElement("net-llheader", [
FixedDataModelElement("s0", b"ll header: "),
AnyByteDataModelElement("data")
]),
AnyByteDataModelElement("unparsed")
]
model = SequenceModelElement("kernel", [
FixedDataModelElement("sname", b"kernel: "),
OptionalMatchModelElement("opt", SequenceModelElement("seq", [
FixedDataModelElement("opt_s0", b"]"),
DelimitedDataModelElement("timestamp", b"]"),
FixedDataModelElement("opt_s1", b"] "),
])),
FirstMatchModelElement("msg", type_children)
])
return model
def test4get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
data = b""
match_context = DummyMatchContext(data)
any_dme = AnyByteDataModelElement(self.id_)
match_element = any_dme.get_match_element(self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def test3unsorted_log_atom(self):
"""In this test case multiple, UNSORTED LogAtoms of different sources are received by the class."""
description = "Test3SimpleMultisourceAtomSync"
sync_wait_time = 3
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector1,
description)
new_match_path_detector2 = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default',
False,
output_log_line=False)
self.analysis_context.register_component(new_match_path_detector2,
description + "2")
simple_multisource_atom_sync = SimpleMultisourceAtomSync(
[new_match_path_detector1, new_match_path_detector2],
sync_wait_time)
t = time()
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element(
'match', match_context)
log_atom1 = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
new_match_path_detector1)
log_atom2 = LogAtom(match_element.match_object,
ParserMatch(match_element), t - 1,
new_match_path_detector1)
self.assertTrue(
not simple_multisource_atom_sync.receive_atom(log_atom1))
sleep(sync_wait_time)
# unsorted, should be accepted
self.reset_output_stream()
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom2))
self.assertTrue(simple_multisource_atom_sync.receive_atom(log_atom1))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t - 1).strftime(
self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description, 1,
self.match_path, self.calculation) + self.__expected_string %
(datetime.fromtimestamp(t - 1).strftime(
self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description + "2", 1,
self.match_path, self.calculation) + self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description, 1,
self.match_path, self.calculation) + self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
new_match_path_detector1.__class__.__name__, description + "2", 1,
self.match_path, self.calculation))
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated with all characters."""
data = b"abcdefghijklmnopqrstuvwxyz.!?"
match_context = DummyMatchContext(data)
any_dme = AnyByteDataModelElement(self.id_)
match_element = any_dme.get_match_element(self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, data, data, None)
def test2get_match_element_invalid_input(self):
"""This test case checks if INVALID inputs are accepted."""
match_context = MatchContext(b'')
any_byte_date_me = AnyByteDataModelElement('id')
self.assertEqual(any_byte_date_me.get_match_element('match1', match_context), None)
match_context = MatchContext(None)
self.assertEqual(any_byte_date_me.get_match_element('match1', match_context), None)
def test1get_match_element_valid_input(self):
"""This test case checks if VALID inputs are accepted."""
match_context = MatchContext(b' some input 123 }')
any_byte_date_me = AnyByteDataModelElement('id')
self.assertEqual(any_byte_date_me.get_match_element('match1', match_context).get_match_string(), b' some input 123 }')
self.assertEqual(any_byte_date_me.get_match_element('match1', match_context), None)
match_context = MatchContext(b'1 ')
self.assertEqual(any_byte_date_me.get_match_element('match1', match_context).get_match_string(), b'1 ')
def test4get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
no_match_string = b""
match_context = DummyMatchContext(no_match_string)
any_dme = AnyByteDataModelElement("s0")
match_element = any_dme.get_match_element("any", match_context)
self.assertIsNone(match_element, None)
self.assertEqual(match_context.match_string, no_match_string)
def test13value_dependent_delegated_match_rule(self):
"""This case unit the ValueDependentDelegatedMatchRule."""
description = "Test13Rules"
string_regex_match_rule = StringRegexMatchRule(self.match_any,
re.compile(rb'\w'),
None)
self.analysis_context.register_component(string_regex_match_rule,
description)
any_byte_date_me = AnyByteDataModelElement('any')
i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4)
self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule,
description + "2")
ip_address_data_model_element = IpAddressDataModelElement('IPv4')
value_dependent_delegated_match_rule = ValueDependentDelegatedMatchRule(
[self.match_any, self.match_ipv4], {
(self.alphabet, ): string_regex_match_rule,
(3232235520, ): i_pv4_in_rfc1918_match_rule
})
self.analysis_context.register_component(
value_dependent_delegated_match_rule, description + "3")
match_context = MatchContext(self.alphabet)
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(value_dependent_delegated_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.0.0')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(value_dependent_delegated_match_rule.match(log_atom))
# not matching values
match_context = MatchContext(
b'.There are 26 letters in the english alphabet')
match_element = any_byte_date_me.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(
not value_dependent_delegated_match_rule.match(log_atom))
match_context = MatchContext(b'192.168.0.1')
match_element = ip_address_data_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
value_dependent_delegated_match_rule)
self.assertTrue(
not value_dependent_delegated_match_rule.match(log_atom))
def test3normal_incomplete_overlong_line_stream_not_ended(self):
"""A incomplete, overlong line, with the stream NOT ended, is tested as Input of the Class."""
match_context_fixed_dme = MatchContext(self.illegal_access1)
any_dme = AnyByteDataModelElement('s')
_match_element_fixed_dme = any_dme.get_match_element("match1", match_context_fixed_dme)
byte_stream_line_atomizer = ByteStreamLineAtomizer(
any_dme, [], [self.stream_printer_event_handler], sys.getsizeof(match_context_fixed_dme.match_data) - 1, [])
self.assertGreater(byte_stream_line_atomizer.consume_data(self.illegal_access1, False), 0)
self.assertEqual(self.output_stream.getvalue(), 'Start of overlong line detected (1 lines)\n %s' % self.illegal_access2)
def get_model():
"""Return a model to parse Apache Error logs from the AIT-LDS."""
model = SequenceModelElement('model', [
FixedDataModelElement('sp1', b'['),
FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']),
FixedDataModelElement('sp2', b' '),
DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'),
FixedDataModelElement('error_str', b'] [:error] [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('sp3', b'] [client '),
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('client_port'),
FixedDataModelElement('php', b'] PHP '),
FirstMatchModelElement('fphp', [
SequenceModelElement('warning', [
FixedDataModelElement('warning_str', b'Warning: '),
FirstMatchModelElement('warning', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function', b')'),
FixedDataModelElement('compatible_str', b') should be compatible with '),
DelimitedDataModelElement('function2', b')'),
FixedDataModelElement('compatible_str', b') in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer')]),
SequenceModelElement('system', [
FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')])])]),
SequenceModelElement('notice', [
FixedDataModelElement('notice_str', b'Notice: Undefined index: '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('sp', b' in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')]),
SequenceModelElement('deprecated', [
FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class '
b'will not be constructors in a future version of PHP; '),
DelimitedDataModelElement('class', b' '),
FixedDataModelElement('constructor_str', b' has a deprecated constructor in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer'),
])])])
return model
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated accordingly."""
data = b'abcdefghijklmnopqrstuvwxyz.!?'
match_context = DummyMatchContext(data)
any_dme = AnyByteDataModelElement("s0")
match_element = any_dme.get_match_element("any", match_context)
self.assertEqual(match_element.path, "any/s0")
self.assertEqual(match_element.match_string, data)
self.assertEqual(match_element.match_object, data)
self.assertIsNone(match_element.children, None)
self.assertEqual(match_context.match_string, data)
def test3_parser_match_is_other_element(self):
"""In this test case the ParserMatch actually is no instance of ParserMatch. The atom should still be considered to be parsed."""
description = "Test3SimpleUnparsedAtomHandler"
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False)
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element('match', match_context)
log_atom = LogAtom(match_element.match_object, any_byte_data_model_element, time(), new_match_path_detector1)
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler([self.stream_printer_event_handler])
self.analysis_context.register_component(simple_unparsed_atom_handler, description)
self.assertTrue(not simple_unparsed_atom_handler.receive_atom(log_atom))
def test2_atom_is_parsed(self):
"""The atom in this test case has no ParserMatch."""
description = "Test2SimpleUnparsedAtomHandler"
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(self.aminer_config, [self.stream_printer_event_handler], 'Default', False)
match_context = MatchContext(self.calculation)
match_element = any_byte_data_model_element.get_match_element('match', match_context)
log_atom = LogAtom(match_element.match_object, None, time(), new_match_path_detector1)
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler([self.stream_printer_event_handler])
self.analysis_context.register_component(simple_unparsed_atom_handler, description)
self.assertTrue(simple_unparsed_atom_handler.receive_atom(log_atom))
def get_model():
"""Return the model."""
type_children = [
SequenceModelElement("sent", [
FixedDataModelElement("s0", b"Sent mail for "),
DelimitedDataModelElement("to-addr", b" ("),
FixedDataModelElement("s1", b" ("),
DelimitedDataModelElement("status", b") uid="),
FixedDataModelElement("s2", b") uid="),
DecimalIntegerValueModelElement("uid"),
FixedDataModelElement("s3", b" username="******"username", b" outbytes="),
FixedDataModelElement("s4", b" outbytes="),
DecimalIntegerValueModelElement("bytes")
]),
SequenceModelElement("sent", [
DelimitedDataModelElement("program", b" "),
FixedDataModelElement("s0", b" sent mail for "),
AnyByteDataModelElement("user")
])
]
model = SequenceModelElement("ssmtp", [
FixedDataModelElement("sname", b"sSMTP["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return the model."""
type_children = [
FixedDataModelElement(
'warn-no-openat',
b'WARNING: SECURITY: No secure open yet due to missing openat in python!'
),
FixedDataModelElement(
'warn-no-OPATH',
b'WARNING: SECURITY: Open should use O_PATH, but not yet available in python'
),
FixedDataModelElement(
'warn-POSIX-acls',
b'WARNING: SECURITY: No checking for backdoor access via \
POSIX ACLs, use "getfacl" from "acl" package to check manually.'),
FixedDataModelElement(
'warn-no-linkat',
b'WARNING: SECURITY: unsafe unlink (unavailable unlinkat/linkat \
should be used, but not available in python)'),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('aminer', [
FixedDataModelElement('sname', b'aminer['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def test6json_format(self):
"""Check if json formatted log data can be processed with the json_format flag."""
json_data = b'{\n\t"a": 1,\n\t"b": {"x": 2}}'
data = b'some log line.'
any_dme = AnyByteDataModelElement('s')
byte_stream_line_atomizer = ByteStreamLineAtomizer(any_dme, [], [self.stream_printer_event_handler], 300, [], json_format=True)
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data + data, False), len(json_data))
# this is no valid json and should process only data until the last \n
self.assertEqual(byte_stream_line_atomizer.consume_data(data + json_data + data, False), len(data) + json_data.rfind(b'\n') + 1)
json_data = b'{"a": 1, "b": {"c": 2}, "d": 3}\n{"a": 1, "b": {"c": 2}, "d": 3}'
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, False), len(json_data))
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data + data, False), len(json_data))
json_data = b'{\n\t"a": 1,\n\t"b": {\n\t\t"c": 2},\n\t"d": 3}{\n"a": 1,\n\t"b": {\n\t\t"c": 2},\n\t"d": 3}'
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data + data, False), len(json_data))
self.assertEqual(byte_stream_line_atomizer.consume_data(data + json_data, False), len(data) + json_data.rfind(b'\n') + 1)
# even when the first json data gets invalidated, the second one starts after an empty line and is therefore valid until the end.
json_data = b'{\n\t"a": 1,\n\t"b": {\n\t\t"c": 2},\n\t"d": 3}\n\n{\n"a": 1,\n\t"b": {\n\t\t"c": 2},\n\t"d": 3}'
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data + data, False), len(json_data))
self.assertEqual(byte_stream_line_atomizer.consume_data(data + json_data + data, False), len(data) + len(json_data))
# this is an incomplete json, but it still can be valid.
json_data = b'{"a": 1, "b": {"c": 2}, "d": 3}\n{"a": 1, "b": {"c": 2}, "d'
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, False), json_data.rfind(b'\n') + 1)
# this is an incomplete json and the end can not be valid.
json_data = b'{"a": 1, "b": {"c": 2}, "d": 3}\n{"a": 1, "b": {"c": 2}, d'
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, False), json_data.rfind(b'\n') + 1)
def test7json_max_line_length(self):
"""Check if json data is not parsed over the max_line_length."""
json_data = b'{\n\t"a": 1,\n\t"b": {\n\t\t"c": 2},\n\t"d": 3}\n{\n"a": 1,\n\t"b": {"c": 2},"d": 3}\n'
single_line_json_data = b'{"a": 1,"b": {"c": 2},"d": 3}{"a": 1,"b": {"c": 2},"d": 3'
any_dme = AnyByteDataModelElement('s')
byte_stream_line_atomizer = ByteStreamLineAtomizer(any_dme, [], [self.stream_printer_event_handler], 25, [], json_format=True)
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, False), json_data.rfind(b'\n') + 1)
self.assertEqual(self.output_stream.getvalue(), '')
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, True), len(json_data))
self.assertEqual(self.output_stream.getvalue(), '')
byte_stream_line_atomizer = ByteStreamLineAtomizer(any_dme, [], [self.stream_printer_event_handler], 100, [], json_format=True)
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, False), len(json_data))
self.assertEqual(self.output_stream.getvalue(), '')
self.assertEqual(byte_stream_line_atomizer.consume_data(json_data, True), len(json_data))
self.assertEqual(self.output_stream.getvalue(), '')
byte_stream_line_atomizer = ByteStreamLineAtomizer(any_dme, [], [self.stream_printer_event_handler], 25, [], json_format=True)
self.assertEqual(byte_stream_line_atomizer.consume_data(single_line_json_data, True), len(single_line_json_data))
self.assertEqual(self.output_stream.getvalue(), 'Overlong line terminated by end of stream (1 lines)\n {"a": 1,"b": {"c": 2},"d":'
' 3}{"a": 1,"b": {"c": 2},"d": 3\n\n')
self.reset_output_stream()
self.assertEqual(byte_stream_line_atomizer.consume_data(single_line_json_data, False), len(single_line_json_data))
self.assertEqual(self.output_stream.getvalue(), '')
byte_stream_line_atomizer = ByteStreamLineAtomizer(any_dme, [], [self.stream_printer_event_handler], 100, [], json_format=True)
self.assertEqual(byte_stream_line_atomizer.consume_data(single_line_json_data, True), len(single_line_json_data))
self.assertEqual(self.output_stream.getvalue(), 'Incomplete last line (1 lines)\n {"a": 1,"b": {"c": 2},"d": 3\n\n')
self.reset_output_stream()
self.assertEqual(byte_stream_line_atomizer.consume_data(single_line_json_data, False), len(
single_line_json_data.rsplit(b'}', 2)[0]) + 1)
self.assertEqual(self.output_stream.getvalue(), '')
def test5eol_sep(self):
"""Test the eol_sep parameter."""
searched = b'WARNING: All illegal access operations will be denied in a future release\nAnother line of data\nThe third line of ' \
b'data.'
any_dme = AnyByteDataModelElement('s')
byte_stream_line_atomizer = ByteStreamLineAtomizer(any_dme, [], [self.stream_printer_event_handler], 300, [], eol_sep=b'\n\n')
data = searched + b'\n\nother line'
self.assertEqual(byte_stream_line_atomizer.consume_data(data, False), len(data)-len(b'other line'))
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
# Build the parsing model:
from aminer.parsing.AnyByteDataModelElement import AnyByteDataModelElement
parsing_model = AnyByteDataModelElement('AnyByteDataModelElement')
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
anomaly_event_handlers = [stream_printer_event_handler]
# Now define the AtomizerFactory using the model. A simple line
# based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [atom_filter], anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.input.SimpleUnparsedAtomHandler import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config, ['/AnyByteDataModelElement'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
def test6get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = AnyByteDataModelElement(self.id_)
data = b"abcdefghijklmnopqrstuvwxyz.!?"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(self.path, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test6_log_stream_handle_streams(self):
"""
This unit case verifies the functionality of the LogStream class. Different FileLogDataResources are added to the stream.
The handling of not existing sources is also tested.
"""
stream_printer_event_handler = StreamPrinterEventHandler(
self.analysis_context, self.output_stream)
any_byte_data_me = AnyByteDataModelElement('a1')
byte_stream_line_atomizer = ByteStreamLineAtomizer(
any_byte_data_me, [], [stream_printer_event_handler], 300, [])
file_log_data_resource = FileLogDataResource(self.file + self.logfile,
-1)
self.assertEqual(file_log_data_resource.buffer, b'')
log_stream = LogStream(file_log_data_resource,
byte_stream_line_atomizer)
file_log_data_resource.open(False)
log_stream.handle_stream()
self.assertEqual(
file_log_data_resource.total_consumed_length +
len(file_log_data_resource.buffer),
file_log_data_resource.default_buffer_size)
log_stream.handle_stream()
self.assertEqual(
file_log_data_resource.total_consumed_length +
len(file_log_data_resource.buffer),
file_log_data_resource.default_buffer_size)
fileLogDataResource2 = FileLogDataResource(b'file:///var/log/auth.log',
-1)
self.assertEqual(fileLogDataResource2.buffer, b'')
fileLogDataResource2.open(False)
log_stream.add_next_resource(fileLogDataResource2)
log_stream.roll_over()
log_stream.handle_stream()
self.assertTrue(file_log_data_resource.total_consumed_length > 0)
self.assertEqual(file_log_data_resource.total_consumed_length,
file_log_data_resource.default_buffer_size)
self.assertTrue(fileLogDataResource2.total_consumed_length > 0)
log_stream.roll_over()
fileLogDataResource3 = FileLogDataResource(
b'file:///var/log/123example.log', -1)
fileLogDataResource3.open(False)
log_stream.add_next_resource(fileLogDataResource3)
self.assertRaises(OSError, log_stream.roll_over)
def get_model():
"""Return the model."""
type_children = [
FixedDataModelElement('start',
b' * Starting Tomcat servlet engine tomcat7'),
FixedDataModelElement('stop',
b' * Stopping Tomcat servlet engine tomcat7'),
FixedDataModelElement('done', b' ...done.'),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('tomcat7', [
FixedDataModelElement('sname', b'tomcat7['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return the model."""
type_children = [
FixedDataModelElement("start",
b" * Starting Tomcat servlet engine tomcat7"),
FixedDataModelElement("stop",
b" * Stopping Tomcat servlet engine tomcat7"),
FixedDataModelElement("done", b" ...done."),
AnyByteDataModelElement("unparsed")
]
model = SequenceModelElement("tomcat7", [
FixedDataModelElement("sname", b"tomcat7["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def test1get_atomizer(self):
any_byte_data_model_element = AnyByteDataModelElement('a1')
new_match_path_detector1 = NewMatchPathDetector(
self.aminer_config, [], 'Default', False)
new_match_path_detector2 = NewMatchPathDetector(
self.aminer_config, [], 'Default', False)
simple_byte_stream_line_atomizer_factory = SimpleByteStreamLineAtomizerFactory(
any_byte_data_model_element,
[new_match_path_detector1, new_match_path_detector2],
[self.stream_printer_event_handler], None)
byte_stream_line_atomizer = simple_byte_stream_line_atomizer_factory.get_atomizer_for_resource(
None)
self.assertEqual(byte_stream_line_atomizer.atom_handler_list,
[new_match_path_detector1, new_match_path_detector2])
self.assertEqual(byte_stream_line_atomizer.event_handler_list,
[self.stream_printer_event_handler])
self.assertEqual(byte_stream_line_atomizer.default_timestamp_paths, [])
self.assertEqual(byte_stream_line_atomizer.parsing_model,
any_byte_data_model_element)
self.assertEqual(byte_stream_line_atomizer.max_line_length, 65536)
def get_model(user_name_model=None):
"""Return a model to parse a cron message logged via syslog after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
type_children = [
SequenceModelElement('exec', [
FixedDataModelElement('s0', b'('), user_name_model,
FixedDataModelElement('s1', b') CMD '),
AnyByteDataModelElement('command')
]),
SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(cron:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '), user_name_model,
OptionalMatchModelElement(
'openby', FixedDataModelElement('default', b' by (uid=0)'))
])
]
model = FirstMatchModelElement('cron', [
SequenceModelElement('std', [
FixedDataModelElement('sname', b'CRON['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msgtype', type_children)
]),
SequenceModelElement('low', [
FixedDataModelElement('sname', b'cron['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: (*system*'),
DelimitedDataModelElement('rname', b') RELOAD ('),
FixedDataModelElement('s1', b') RELOAD ('),
DelimitedDataModelElement('fname', b')'),
FixedDataModelElement('s2', b')'),
])
])
return model
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPUWorkload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S')
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filters = AtomFilters.SubhandlerFilter(None)
analysis_context.register_component(atom_filters,
component_name="AtomFilter")
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filters])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(
analysis_context) # skipcq: BAN-B108
from aminer.events.Utils import VolatileLogarithmicBackoffEventHistory
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(
100)
anomaly_event_handlers = [
stream_printer_event_handler,
volatile_logarithmic_backoff_event_history
]
analysis_context.register_component(
volatile_logarithmic_backoff_event_history,
component_name="VolatileLogarithmicBackoffEventHistory")
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filters.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filters.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filters.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None,
anomaly_event_handlers, 10)
analysis_context.register_component(parser_count,
component_name="ParserCount")
atom_filters.add_handler(parser_count)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
check_rules_flag=True,
hypothesis_max_delta_time=1.0,
auto_include_flag=True)
analysis_context.register_component(
ecd, component_name="EventCorrelationDetector")
atom_filters.add_handler(ecd)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filters.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=False,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filters.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filters.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filters.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filters.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filters.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=False)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filters.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
analysis_context.aminer_config,
['/model/type/path/name', '/model/type/syscall/syscall'],
anomaly_event_handlers,
id_path_list=['/model/type/path/id', '/model/type/syscall/id'],
min_allowed_time_diff=5,
auto_include_flag=True,
allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(
new_match_id_value_combo_detector,
component_name="NewMatchIdValueComboDetector")
atom_filters.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/Job Number', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=False)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filters.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'],
anomaly_event_handlers,
auto_include_flag=False,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filters.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filters.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filters.add_handler(time_correlation_violation_detector)
from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers.append(mail_notification_handler)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement('Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%'),
DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')]
service_children_login_details = [
FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']),
OptionalMatchModelElement('PastTime', SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')]))]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')]
service_children_random_time = [FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random')]
service_children_sensors = [SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')]), FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)]
service_children_user_ip_address = [
FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')]
service_children_audit = [SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO),
FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')])]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'),
SequenceModelElement('se', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';')])]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement("wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement("decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement('ElementValueBranchModelElement', FirstMatchModelElement("first", [
SequenceModelElement("seq1", [fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])]), "wordlist",
{0: decimal_integer_value_model_element, 1: fixed_data_me2}))
service_children_parsing_model_element.append(HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(SequenceModelElement('se2', [
FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement')]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1))
service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(SequenceModelElement('se', [
WhiteSpaceLimitedDataModelElement('WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ')]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement('/', FirstMatchModelElement('FirstMatchModelElement//optional', [
FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [
FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])])))
alphabet = b'ghijkl'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement),
SequenceModelElement('CronExecution', service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element)])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust([atom_filter])
analysis_context.register_component(simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
from aminer.events.JsonConverterHandler import JsonConverterHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
json_converter_handler = JsonConverterHandler([stream_printer_event_handler], analysis_context)
anomaly_event_handlers = [json_converter_handler]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True)
analysis_context.register_component(verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.AndMatchRule([
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails'),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10)
analysis_context.register_component(parser_count, component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers)
analysis_context.register_component(etd, component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False,
output_log_line=False, ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5,
ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True,
hypothesis_max_delta_time=1.0)
analysis_context.register_component(ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector
efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1)
analysis_context.register_component(efd, component_name="EventFrequencyDetector")
atom_filter.add_handler(efd)
from aminer.analysis.EventSequenceDetector import EventSequenceDetector
esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random',
'/model/RandomTime', '/model/DailyCron'])
analysis_context.register_component(esd, component_name="EventSequenceDetector")
atom_filter.add_handler(esd)
from aminer.analysis.MatchFilter import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[
1, 10, 100], output_log_line=True)
analysis_context.register_component(match_filter, component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(analysis_context.aminer_config, [
'/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function, output_log_line=True)
analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
import re
ip_match_action = Rules.EventGenerationMatchAction(
"Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers)
vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3])
mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None)
time_allowlist_rules = [
Rules.AndMatchRule([
Rules.ParallelMatchRule([
Rules.ValueDependentDelegatedMatchRule([
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], {
(b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt),
Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action),
Rules.DebugHistoryMatchRule(debug_match_result=True)
]),
# IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed
Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])),
Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)),
Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$")))
])
]
time_allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist")
atom_filter.add_handler(time_allowlist_violation_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(analysis_context.aminer_config, [
('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(analysis_context.aminer_config, anomaly_event_handlers, None, [
'/model/Random'], 100, 10, output_log_line=True)
analysis_context.register_component(match_value_average_change_detector, component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(
sys.stdout, ['/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM'], b';', b'')
analysis_context.register_component(match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(analysis_context.aminer_config, [
'/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=[
'/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(analysis_context.aminer_config, [
'/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True)
analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2,
realert_interval=5, output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5,
record_count_before_event=10000, output_log_line=True)
analysis_context.register_component(time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule('CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[
('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement])
rules = [Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)]
time_correlation_violation_detector = TimeCorrelationViolationDetector(analysis_context.aminer_config, rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)