def get_model():
"""Return a parser for apache2 access.log."""
new_time_model = SequenceModelElement('time_model', [
DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S '),
FixedWordlistDataModelElement('sign', [b'+', b'-']),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('bracket', b']')
])
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identity_model = VariableByteDataModelElement(
'ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
request_method_model = FixedWordlistDataModelElement(
'method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT',
b'OPTIONS', b'TRACE', b'PATCH'
])
request_model = VariableByteDataModelElement(
'request',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+'
)
version_model = VariableByteDataModelElement('version', b'0123456789.')
status_code_model = DecimalIntegerValueModelElement('status')
size_model = DecimalIntegerValueModelElement('size')
user_agent_model = VariableByteDataModelElement(
'useragent',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+;:_ '
)
whitespace_str = b' '
model = SequenceModelElement('accesslog', [
host_name_model,
FixedDataModelElement('sp0', whitespace_str),
identity_model,
FixedDataModelElement('sp1', whitespace_str),
user_name_model,
FixedDataModelElement('sp2', whitespace_str),
new_time_model,
FixedDataModelElement('sp3', b' "'),
request_method_model,
FixedDataModelElement('sp4', whitespace_str),
request_model,
FixedDataModelElement('sp5', b' HTTP/'),
version_model,
FixedDataModelElement('sp6', b'" '),
status_code_model,
FixedDataModelElement('sp7', whitespace_str),
size_model,
FixedDataModelElement('sp8', b' "-" "'),
user_agent_model,
FixedDataModelElement('sp9', b'"'),
])
return model
def get_model():
"""Return a model for su session information messages after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('build-stack', [
FixedDataModelElement('s0', b'building new pluginstance stack: \''),
DelimitedDataModelElement('stack', b'\''),
FixedDataModelElement('s1', b'\'')
]),
SequenceModelElement('nfct-event', [
FixedDataModelElement('s0', b'[DESTROY] ORIG: SRC='),
IpAddressDataModelElement('osrcip'),
FixedDataModelElement('s1', b' DST='),
IpAddressDataModelElement('odstip'),
FixedDataModelElement('s2', b' PROTO='),
FixedWordlistDataModelElement('proto', [b'TCP', b'UDP']),
FixedDataModelElement('s3', b' SPT='),
DecimalIntegerValueModelElement('ospt'),
FixedDataModelElement('s4', b' DPT='),
DecimalIntegerValueModelElement('odpt'),
FixedDataModelElement('s5', b' PKTS='),
DecimalIntegerValueModelElement('opkts'),
FixedDataModelElement('s6', b' BYTES='),
DecimalIntegerValueModelElement('obytes'),
FixedDataModelElement('s7', b' , REPLY: SRC='),
IpAddressDataModelElement('rsrcip'),
FixedDataModelElement('s8', b' DST='),
IpAddressDataModelElement('rdstip'),
FixedDataModelElement('s9', b' PROTO='),
FixedWordlistDataModelElement('rproto', [b'TCP', b'UDP']),
FixedDataModelElement('s10', b' SPT='),
DecimalIntegerValueModelElement('rspt'),
FixedDataModelElement('s11', b' DPT='),
DecimalIntegerValueModelElement('rdpt'),
FixedDataModelElement('s12', b' PKTS='),
DecimalIntegerValueModelElement('rpkts'),
FixedDataModelElement('s13', b' BYTES='),
DecimalIntegerValueModelElement('rbytes'),
# No additional whitespace from Ubuntu Trusty 14.04 on.
OptionalMatchModelElement('tail', FixedDataModelElement('s0', b' '))
]),
FixedDataModelElement('nfct-plugin', b'NFCT plugin working in event mode'),
FixedDataModelElement('reopen', b'reopening capture file'),
FixedDataModelElement('signal', b'signal received, calling pluginstances'),
FixedDataModelElement('uidchange', b'Changing UID / GID')
]
# Netflow entry
model = SequenceModelElement('ulogd', [
FixedDataModelElement('sname', b'ulogd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
class SequenceModelElementTest(unittest.TestCase):
sequence_start = b'The sequence starts with a number: '
fixed_data_model_element = FixedDataModelElement('fixed', sequence_start)
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
'decimal', DecimalIntegerValueModelElement.SIGN_TYPE_NONE, DecimalIntegerValueModelElement.PAD_TYPE_NONE)
fixed_wordlist_data_model_element = FixedWordlistDataModelElement('wordlist', [b' Euro', b' Dollar', b' Pfund'])
sequence_model_element = SequenceModelElement(
'sequence', [fixed_data_model_element, decimal_integer_value_model_element, fixed_wordlist_data_model_element])
def test1sequence_of_matching_elements(self):
"""A normal sequence of matching elements is tested in this example test case"""
match_context = MatchContext(b'The sequence starts with a number: 25538 Euro')
self.assertEqual(self.sequence_model_element.get_match_element(
'match', match_context).get_match_string(), b'The sequence starts with a number: 25538 Euro')
self.assertEqual(match_context.match_data, b'')
def test2sequence_not_matching(self):
"""A normal sequence of elements, which do not match with the expected sequence_model is tested."""
match_context = MatchContext(b'The sequence starts with a number: 25538 US-Dollar')
self.assertEqual(self.sequence_model_element.get_match_element('match', match_context), None)
self.assertEqual(match_context.match_data, b'The sequence starts with a number: 25538 US-Dollar')
def test3match_context_shorter_than_sequence(self):
"""This test case unit if the sequence_model returns None, when the match_context is too short for a match."""
match_context = MatchContext(self.sequence_start)
self.assertEqual(self.sequence_model_element.get_match_element('match', match_context), None)
self.assertEqual(match_context.match_data, self.sequence_start)
def get_model():
"""Return a model to parse messages from kernel logging."""
type_children = [
SequenceModelElement("ipv4-martian", [
FixedDataModelElement("s0", b"IPv4: martian "),
FixedWordlistDataModelElement("direction", [b"source", b"destination"]),
FixedDataModelElement("s1", b" "),
IpAddressDataModelElement("destination"),
FixedDataModelElement("s2", b" from "),
IpAddressDataModelElement("source"),
FixedDataModelElement("s3", b", on dev "),
AnyByteDataModelElement("interface")]),
SequenceModelElement("net-llheader", [
FixedDataModelElement("s0", b"ll header: "),
AnyByteDataModelElement("data")
]),
AnyByteDataModelElement("unparsed")
]
model = SequenceModelElement("kernel", [
FixedDataModelElement("sname", b"kernel: "),
OptionalMatchModelElement("opt", SequenceModelElement("seq", [
FixedDataModelElement("opt_s0", b"]"),
DelimitedDataModelElement("timestamp", b"]"),
FixedDataModelElement("opt_s1", b"] "),
])),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return a model to parse messages from kernel logging."""
type_children = [
SequenceModelElement('ipv4-martian', [
FixedDataModelElement('s0', b'IPv4: martian '),
FixedWordlistDataModelElement('direction',
[b'source', b'destination']),
FixedDataModelElement('s1', b' '),
IpAddressDataModelElement('destination'),
FixedDataModelElement('s2', b' from '),
IpAddressDataModelElement('source'),
FixedDataModelElement('s3', b', on dev '),
AnyByteDataModelElement('interface')
]),
SequenceModelElement('net-llheader', [
FixedDataModelElement('s0', b'll header: '),
AnyByteDataModelElement('data')
]),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('kernel', [
FixedDataModelElement('sname', b'kernel: ['),
DelimitedDataModelElement('timestamp', b']'),
FixedDataModelElement('s0', b'] '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return a model to parse Apache Error logs from the AIT-LDS."""
model = SequenceModelElement('model', [
FixedDataModelElement('sp1', b'['),
FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']),
FixedDataModelElement('sp2', b' '),
DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'),
FixedDataModelElement('error_str', b'] [:error] [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('sp3', b'] [client '),
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('client_port'),
FixedDataModelElement('php', b'] PHP '),
FirstMatchModelElement('fphp', [
SequenceModelElement('warning', [
FixedDataModelElement('warning_str', b'Warning: '),
FirstMatchModelElement('warning', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function', b')'),
FixedDataModelElement('compatible_str', b') should be compatible with '),
DelimitedDataModelElement('function2', b')'),
FixedDataModelElement('compatible_str', b') in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer')]),
SequenceModelElement('system', [
FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')])])]),
SequenceModelElement('notice', [
FixedDataModelElement('notice_str', b'Notice: Undefined index: '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('sp', b' in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')]),
SequenceModelElement('deprecated', [
FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class '
b'will not be constructors in a future version of PHP; '),
DelimitedDataModelElement('class', b' '),
FixedDataModelElement('constructor_str', b' has a deprecated constructor in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer'),
])])])
return model
def test7get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = FixedWordlistDataModelElement(self.id_, self.wordlist)
data = b"abcdefghijklmnopqrstuvwxyz.!?"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def test1sorted_list(self):
"""A valid wordlist is used in this test case. """
match_context = MatchContext(b'wordlist started with "wordlist"')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
'wordlist', [b'wordlist', b'word'])
match_element = fixed_wordlist_data_model_element.get_match_element(
'match', match_context)
self.assertEqual(match_element.get_match_string(), b'wordlist')
match_context = MatchContext(b'words started with "wordlist"')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
'wordlist', [b'wordlist', b'word'])
match_element = fixed_wordlist_data_model_element.get_match_element(
'match', match_context)
self.assertEqual(match_element.get_match_string(), b'word')
match_context = MatchContext(b'list started with "wordlist"')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
'wordlist', [b'wordlist', b'word'])
match_element = fixed_wordlist_data_model_element.get_match_element(
'match', match_context)
self.assertEqual(match_element, None)
def get_model():
"""Return a parser for apache2 access.log."""
new_time_model = DateTimeModelElement("time", b"[%d/%b/%Y:%H:%M:%S%z")
host_name_model = VariableByteDataModelElement(
"host", b"-.01234567890abcdefghijklmnopqrstuvwxyz:")
identity_model = VariableByteDataModelElement(
"ident", b"-.01234567890abcdefghijklmnopqrstuvwxyz:")
user_name_model = VariableByteDataModelElement(
"user", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
request_method_model = FirstMatchModelElement("fm", [
FixedDataModelElement("dash", b"-"),
SequenceModelElement("request", [
FixedWordlistDataModelElement("method", [
b"GET", b"POST", b"PUT", b"HEAD", b"DELETE", b"CONNECT",
b"OPTIONS", b"TRACE", b"PATCH"
]),
FixedDataModelElement("sp5", b" "),
DelimitedDataModelElement("request", b" ", b"\\"),
FixedDataModelElement("sp6", b" "),
DelimitedDataModelElement("version", b'"'),
])
])
status_code_model = DecimalIntegerValueModelElement("status")
size_model = DecimalIntegerValueModelElement("size")
whitespace_str = b" "
model = SequenceModelElement("accesslog", [
host_name_model,
FixedDataModelElement("sp0", whitespace_str), identity_model,
FixedDataModelElement("sp1", whitespace_str), user_name_model,
FixedDataModelElement("sp2", whitespace_str), new_time_model,
FixedDataModelElement("sp3", b'] "'), request_method_model,
FixedDataModelElement("sp6", b'" '), status_code_model,
FixedDataModelElement("sp7", whitespace_str), size_model,
OptionalMatchModelElement(
"combined",
SequenceModelElement("combined", [
FixedDataModelElement("sp9", b' "'),
DelimitedDataModelElement("referer", b'"', b"\\"),
FixedDataModelElement("sp10", b'" "'),
DelimitedDataModelElement("user_agent", b'"', b"\\"),
FixedDataModelElement("sp11", b'"')
]))
])
return model
def get_model():
"""Return a model to parse Apache Access logs from the AIT-LDS."""
alphabet = b"!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]"
model = SequenceModelElement("model", [
FirstMatchModelElement("client_ip", [
IpAddressDataModelElement("client_ip"),
FixedDataModelElement("localhost", b"::1")
]),
FixedDataModelElement("sp1", b" "),
VariableByteDataModelElement("client_id", alphabet),
FixedDataModelElement("sp2", b" "),
VariableByteDataModelElement("user_id", alphabet),
FixedDataModelElement("sp3", b" ["),
DateTimeModelElement("time", b"%d/%b/%Y:%H:%M:%S%z"),
FixedDataModelElement("sp4", b'] "'),
FirstMatchModelElement("fm", [
FixedDataModelElement("dash", b"-"),
SequenceModelElement("request", [
FixedWordlistDataModelElement("method", [
b"GET", b"POST", b"PUT", b"HEAD", b"DELETE", b"CONNECT",
b"OPTIONS", b"TRACE", b"PATCH"
]),
FixedDataModelElement("sp5", b" "),
DelimitedDataModelElement("request", b" ", b"\\"),
FixedDataModelElement("sp6", b" "),
DelimitedDataModelElement("version", b'"'),
])
]),
FixedDataModelElement("sp7", b'" '),
DecimalIntegerValueModelElement("status_code"),
FixedDataModelElement("sp8", b" "),
DecimalIntegerValueModelElement("content_size"),
OptionalMatchModelElement(
"combined",
SequenceModelElement("combined", [
FixedDataModelElement("sp9", b' "'),
DelimitedDataModelElement("referer", b'"', b"\\"),
FixedDataModelElement("sp10", b'" "'),
DelimitedDataModelElement("user_agent", b'"', b"\\"),
FixedDataModelElement("sp11", b'"'),
]))
])
return model
def get_model():
"""Return a model to parse Apache Access logs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
model = SequenceModelElement('model', [
FirstMatchModelElement('client_ip', [
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('localhost', b'::1')
]),
FixedDataModelElement('sp1', b' '),
VariableByteDataModelElement('client_id', alphabet),
FixedDataModelElement('sp2', b' '),
VariableByteDataModelElement('user_id', alphabet),
FixedDataModelElement('sp3', b' ['),
DateTimeModelElement('time', b'%d/%b/%Y:%H:%M:%S'),
FixedDataModelElement('sp4', b' +'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('sp5', b'] "'),
FirstMatchModelElement('fm', [
FixedDataModelElement('dash', b'-'),
SequenceModelElement('request', [
FixedWordlistDataModelElement('method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH']),
FixedDataModelElement('sp6', b' '),
DelimitedDataModelElement('request', b' ', b'\\'),
FixedDataModelElement('sp7', b' '),
DelimitedDataModelElement('version', b'"'),
])
]),
FixedDataModelElement('sp8', b'" '),
DecimalIntegerValueModelElement('status_code'),
FixedDataModelElement('sp9', b' '),
DecimalIntegerValueModelElement('content_size'),
OptionalMatchModelElement(
'combined', SequenceModelElement('combined', [
FixedDataModelElement('sp10', b' "'),
DelimitedDataModelElement('referer', b'"', b'\\'),
FixedDataModelElement('sp11', b'" "'),
DelimitedDataModelElement('user_agent', b'"', b'\\'),
FixedDataModelElement('sp12', b'"'),
])),
])
return model
def get_model(user_name_model=None):
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
"user", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
srcuser_name_model = VariableByteDataModelElement(
"srcuser", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
type_children = [
SequenceModelElement("su-good", [
FixedDataModelElement("s0", b"Successful su for "),
user_name_model,
FixedDataModelElement("s1", b" by "), srcuser_name_model
]),
SequenceModelElement("su-good", [
FixedDataModelElement("s0", b"+ "),
DelimitedDataModelElement("terminal", b" "),
FixedDataModelElement("s1", b" "), srcuser_name_model,
FixedDataModelElement("s2", b":"), user_name_model
]),
SequenceModelElement("pam", [
FixedDataModelElement("s0", b"pam_unix(su:session): session "),
FixedWordlistDataModelElement("change", [b"opened", b"closed"]),
FixedDataModelElement("s1", b" for user "), user_name_model,
OptionalMatchModelElement(
"openby",
SequenceModelElement("userinfo", [
FixedDataModelElement("s0", b" by (uid="),
DecimalIntegerValueModelElement("uid"),
FixedDataModelElement("s1", b")")
]))
])
]
model = SequenceModelElement("su", [
FixedDataModelElement("sname", b"su["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model(user_name_model=None):
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
srcuser_name_model = VariableByteDataModelElement(
'srcuser', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
type_children = [
SequenceModelElement('su-good', [
FixedDataModelElement('s0', b'Successful su for '),
user_name_model,
FixedDataModelElement('s1', b' by '), srcuser_name_model
]),
SequenceModelElement('su-good', [
FixedDataModelElement('s0', b'+ '),
DelimitedDataModelElement('terminal', b' '),
FixedDataModelElement('s1', b' '), srcuser_name_model,
FixedDataModelElement('s2', b':'), user_name_model
]),
SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(su:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '), user_name_model,
OptionalMatchModelElement(
'openby',
SequenceModelElement('userinfo', [
FixedDataModelElement('s0', b' by (uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s1', b')')
]))
])
]
model = SequenceModelElement('su', [
FixedDataModelElement('sname', b'su['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def test6null_value(self):
"""Test if null values are parsed to "null"."""
key_parser_dict = {
"works":
VariableByteDataModelElement("id", b"abc123"),
"problem":
FixedWordlistDataModelElement("wordlist",
[b"allowed value", b"null"])
}
data1 = b"""{
"works": "abc",
"problem": "allowed value"
}"""
data2 = b"""{
"works": "123",
"problem": null
}"""
json_model_element = JsonModelElement('json', key_parser_dict)
self.assertIsNotNone(
json_model_element.get_match_element('match', MatchContext(data1)))
self.assertIsNotNone(
json_model_element.get_match_element('match', MatchContext(data2)))
def get_model(user_name_model=None):
"""Return a model to parse a cron message logged via syslog after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
type_children = [
SequenceModelElement('exec', [
FixedDataModelElement('s0', b'('), user_name_model,
FixedDataModelElement('s1', b') CMD '),
AnyByteDataModelElement('command')
]),
SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(cron:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '), user_name_model,
OptionalMatchModelElement(
'openby', FixedDataModelElement('default', b' by (uid=0)'))
])
]
model = FirstMatchModelElement('cron', [
SequenceModelElement('std', [
FixedDataModelElement('sname', b'CRON['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msgtype', type_children)
]),
SequenceModelElement('low', [
FixedDataModelElement('sname', b'cron['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: (*system*'),
DelimitedDataModelElement('rname', b') RELOAD ('),
FixedDataModelElement('s1', b') RELOAD ('),
DelimitedDataModelElement('fname', b')'),
FixedDataModelElement('s2', b')'),
])
])
return model
def test3get_match_element_valid_match(self):
"""Parse matching substring from MatchContext and check if the MatchContext was updated with all characters."""
data = b"wordlist, word"
index = 0
value = b"wordlist"
match_context = DummyMatchContext(data)
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
match_element = fixed_wordlist_dme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, index, None)
data = b"word, wordlist"
index = 1
value = b"word"
match_context = DummyMatchContext(data)
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
match_element = fixed_wordlist_dme.get_match_element(
self.path, match_context)
self.compare_match_results(data, match_element, match_context,
self.id_, self.path, value, index, None)
def test2get_child_elements(self):
"""Test if get_child_elements returns None."""
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
self.assertEqual(fixed_wordlist_dme.get_child_elements(), None)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('Fixed Workload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Start Time', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', "de_AT.utf8", None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
AnyByteDataModelElement('AnyByteDataModelElement')
])))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers
# later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler
syslog_event_handler = SyslogWriterEventHandler(analysis_context)
from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers = [
stream_printer_event_handler, syslog_event_handler,
mail_notification_handler
]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'],
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
class ElementValueBranchModelElementTest(unittest.TestCase):
"""Unittests for the ElementValueBranchModelElement."""
match = b'match '
fixed_string = b'fixed String'
path_to_match = b'match path: /path/to/match'
match_data = b'match data: 25000'
match_string = b'match string: fixed String'
string_path_to_match = b'match string: /path/to/match'
fixed_data_me1 = FixedDataModelElement("fixed 1", match)
fixed_data_me2 = FixedDataModelElement("fixed 2", fixed_string)
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
seq1 = SequenceModelElement(
"seq1", [fixed_data_me1, fixed_wordlist_data_model_element])
seq2 = SequenceModelElement(
"seq2",
[fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])
first_match_me = FirstMatchModelElement("first", [seq1, seq2])
fixed_data_me3 = FixedDataModelElement("fixed 3", path_to_match)
def test1_match_element_found(self):
"""In this test case different possible parameters are used to obtain a MatchElement successfully."""
match_context = MatchContext(self.match_data)
element_value_branch_model_element = ElementValueBranchModelElement(
"id", self.first_match_me, "wordlist", {
0: self.decimal_integer_value_model_element,
1: self.fixed_data_me2
})
match_element = element_value_branch_model_element.get_match_element(
"elementValueBranchME match", match_context)
self.assertEqual(match_element.get_path(),
'elementValueBranchME match/id')
self.assertEqual(match_element.get_match_string(), self.match_data)
self.assertEqual(match_element.get_match_object(), self.match_data)
self.assertEqual(match_element.get_children()[0].get_path(),
'elementValueBranchME match/id/first/seq1')
self.assertEqual(match_element.get_children()[0].get_match_string(),
b'match data: ')
self.assertEqual(match_element.get_children()[0].get_match_object(),
b'match data: ')
self.assertEqual(
match_element.get_children()[0].get_children()[0].get_path(),
'elementValueBranchME match/id/first/seq1/fixed 1')
self.assertEqual(
match_element.get_children()[0].get_children()
[0].get_match_string(), self.match)
self.assertEqual(
match_element.get_children()[0].get_children()
[0].get_match_object(), self.match)
self.assertEqual(
match_element.get_children()[0].get_children()[0].get_children(),
None)
self.assertEqual(
match_element.get_children()[0].get_children()[1].get_path(),
'elementValueBranchME match/id/first/seq1/wordlist')
self.assertEqual(
match_element.get_children()[0].get_children()
[1].get_match_string(), b'data: ')
self.assertEqual(
match_element.get_children()[0].get_children()
[1].get_match_object(), 0)
self.assertEqual(
match_element.get_children()[0].get_children()[1].get_children(),
None)
self.assertEqual(match_element.get_children()[1].get_path(),
'elementValueBranchME match/id/decimal')
self.assertEqual(match_element.get_children()[1].get_match_string(),
b'25000')
self.assertEqual(match_element.get_children()[1].get_match_object(),
25000)
self.assertEqual(match_element.get_children()[1].get_children(), None)
match_context = MatchContext(self.match_string)
match_element = element_value_branch_model_element.get_match_element(
"elementValueBranchME match", match_context)
self.assertEqual(match_element.get_path(),
'elementValueBranchME match/id')
self.assertEqual(match_element.get_match_string(), self.match_string)
self.assertEqual(match_element.get_match_object(), self.match_string)
self.assertEqual(match_element.get_children()[0].get_path(),
'elementValueBranchME match/id/first/seq1')
self.assertEqual(match_element.get_children()[0].get_match_string(),
b'match string: ')
self.assertEqual(match_element.get_children()[0].get_match_object(),
b'match string: ')
self.assertEqual(
match_element.get_children()[0].get_children()[0].get_path(),
'elementValueBranchME match/id/first/seq1/fixed 1')
self.assertEqual(
match_element.get_children()[0].get_children()
[0].get_match_string(), self.match)
self.assertEqual(
match_element.get_children()[0].get_children()
[0].get_match_object(), self.match)
self.assertEqual(
match_element.get_children()[0].get_children()[0].get_children(),
None)
self.assertEqual(
match_element.get_children()[0].get_children()[1].get_path(),
'elementValueBranchME match/id/first/seq1/wordlist')
self.assertEqual(
match_element.get_children()[0].get_children()
[1].get_match_string(), b'string: ')
self.assertEqual(
match_element.get_children()[0].get_children()
[1].get_match_object(), 1)
self.assertEqual(
match_element.get_children()[0].get_children()[1].get_children(),
None)
self.assertEqual(match_element.get_children()[1].get_path(),
'elementValueBranchME match/id/fixed 2')
self.assertEqual(match_element.get_children()[1].get_match_string(),
self.fixed_string)
self.assertEqual(match_element.get_children()[1].get_match_object(),
self.fixed_string)
self.assertEqual(match_element.get_children()[1].get_children(), None)
def test2_match_element_not_found(self):
"""In this test case all possible ways of not getting a MatchElement successfully are tested."""
# no modelMatch
element_value_branch_model_element = ElementValueBranchModelElement(
"id", self.first_match_me, "wordlist", {
0: self.decimal_integer_value_model_element,
1: self.fixed_data_me2
})
match_context = MatchContext(self.path_to_match)
self.assertEqual(
element_value_branch_model_element.get_match_element(
'match', match_context), None)
self.assertEqual(match_context.match_data, self.path_to_match)
# no matchChildren
element_value_branch_model_element = ElementValueBranchModelElement(
"id", self.fixed_data_me3, "wordlist", {
0: self.decimal_integer_value_model_element,
1: self.fixed_data_me2
})
match_context = MatchContext(self.path_to_match)
self.assertEqual(
element_value_branch_model_element.get_match_element(
'match', match_context), None)
self.assertEqual(match_context.match_data, self.path_to_match)
# no branchModel
element_value_branch_model_element = ElementValueBranchModelElement(
"id", self.first_match_me, "wordlist",
{0: self.decimal_integer_value_model_element})
match_context = MatchContext(self.string_path_to_match)
self.assertEqual(
element_value_branch_model_element.get_match_element(
'match', match_context), None)
self.assertEqual(match_context.match_data, self.string_path_to_match)
# no branchMatch
element_value_branch_model_element = ElementValueBranchModelElement(
"id", self.first_match_me, "wordlist", {
0: self.decimal_integer_value_model_element,
1: self.fixed_data_me2
})
match_context = MatchContext(self.string_path_to_match)
self.assertEqual(
element_value_branch_model_element.get_match_element(
'match', match_context), None)
self.assertEqual(match_context.match_data, self.string_path_to_match)
def test3_get_child_elements(self):
"""In this test case the functionality of the get_child_elements-method is tested."""
element_value_branch_model_element = ElementValueBranchModelElement(
"id", self.first_match_me, "wordlist", {
0: self.decimal_integer_value_model_element,
1: self.fixed_data_me2
})
self.assertEqual(
element_value_branch_model_element.get_child_elements(), [
self.first_match_me, self.decimal_integer_value_model_element,
self.fixed_data_me2
])
def get_model(user_name_model=None):
"""Return a model to parse a sshd information message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
"user", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
from_str = b" from "
port = b" port "
preauth = b" [preauth]"
type_children = [
SequenceModelElement("accepted key", [
FixedDataModelElement("s0", b"Accepted publickey for "),
user_name_model,
FixedDataModelElement("s1", from_str),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s2", port),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s3", b" ssh2: "),
DelimitedDataModelElement(
"asym-algorithm", b" ", consume_delimiter=True),
VariableByteDataModelElement(
"fingerprint",
b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/:"
),
OptionalMatchModelElement(
"opt",
SequenceModelElement("seq", [
FixedDataModelElement("s4", b" ID "),
DelimitedDataModelElement("id", b" "),
FixedDataModelElement("s5", b" (serial "),
DecimalIntegerValueModelElement("serial"),
FixedDataModelElement("s6", b") CA "),
AnyByteDataModelElement("algorithm_details")
]))
]),
SequenceModelElement("btmp-perm", [
FixedDataModelElement(
"s0",
b"Excess permission or bad ownership on file /var/log/btmp")
]),
SequenceModelElement("close-sess", [
FixedDataModelElement("s0", b"Close session: user "),
user_name_model,
FixedDataModelElement("s1", from_str),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s2", port),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s3", b" id "),
DecimalIntegerValueModelElement("userid")
]),
SequenceModelElement("closing", [
FixedDataModelElement("s0", b"Closing connection to "),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s1", port),
DecimalIntegerValueModelElement("port")
]),
SequenceModelElement("closed", [
FixedDataModelElement("s0", b"Connection closed by "),
FirstMatchModelElement("fm", [
IpAddressDataModelElement("clientip"),
SequenceModelElement("seq", [
FixedWordlistDataModelElement(
"user-type", [b"authenticating", b"invalid"]),
FixedDataModelElement("s1", b" user "),
DelimitedDataModelElement("user", b" "),
FixedDataModelElement("s2", b" "),
FirstMatchModelElement("fm", [
IpAddressDataModelElement("ip"),
IpAddressDataModelElement("ipv6", ipv6=True)
]),
FixedDataModelElement("s3", b" port "),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s4", b" [preauth]")
])
])
]),
SequenceModelElement("connect", [
FixedDataModelElement("s0", b"Connection from "),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s1", port),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s2", b" on "),
IpAddressDataModelElement("serverip"),
FixedDataModelElement("s3", port),
DecimalIntegerValueModelElement("sport")
]),
SequenceModelElement("disconnectreq", [
FixedDataModelElement("s0", b"Received disconnect from "),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s1", port),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s2", b":"),
DecimalIntegerValueModelElement("session"),
FixedDataModelElement("s3", b": "),
FixedWordlistDataModelElement("reason", [b"disconnected by user"])
]),
SequenceModelElement("disconnected", [
FixedDataModelElement("s0", b"Disconnected from "),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s1", port),
DecimalIntegerValueModelElement("port")
]),
FixedDataModelElement("error-bind",
b"error: bind: Cannot assign requested address"),
SequenceModelElement("error-max-auth", [
FixedDataModelElement(
"s0", b"error: maximum authentication attempts exceeded for "),
OptionalMatchModelElement(
"opt", FixedDataModelElement("invalid", b"invalid user ")),
DelimitedDataModelElement("user", b" "),
FixedDataModelElement("s1", b" from "),
FirstMatchModelElement("fm", [
IpAddressDataModelElement("from_ip"),
IpAddressDataModelElement("from_ip_v6", ipv6=True)
]),
FixedDataModelElement("s2", b" port "),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s3", b" ssh2 [preauth]")
]),
FixedDataModelElement("error-cert-exp",
b"error: Certificate invalid: expired"),
FixedDataModelElement("error-cert-not-yet-valid",
b"error: Certificate invalid: not yet valid"),
FixedDataModelElement(
"error-cert-not-listed-principal",
b"error: Certificate invalid: name is not a listed principal"),
FixedDataModelElement("error-refused-by-options",
b"error: Refused by certificate options"),
SequenceModelElement("error-channel-setup", [
FixedDataModelElement(
"s0",
b"error: channel_setup_fwd_listener: cannot listen to port: "),
DecimalIntegerValueModelElement("port")
]),
SequenceModelElement("error-auth-key", [
FixedDataModelElement("s0", b"error: Authentication key "),
DelimitedDataModelElement("asym-algorithm", b" "),
FixedDataModelElement("s1", b" "),
DelimitedDataModelElement("hash", b" "),
FixedDataModelElement("s2", b" revoked by file "),
AnyByteDataModelElement("file")
]),
SequenceModelElement("error-load-key", [
FixedDataModelElement("s0", b"error: Could not load host key: "),
AnyByteDataModelElement("file")
]),
SequenceModelElement("ident-missing", [
FixedDataModelElement(
"s0", b"Did not receive identification string from "),
IpAddressDataModelElement("clientip")
]),
SequenceModelElement("invalid-user", [
FixedDataModelElement("s0", b"Invalid user "),
DelimitedDataModelElement("user", from_str),
FixedDataModelElement("s1", from_str),
FirstMatchModelElement("fm", [
IpAddressDataModelElement("from_ip"),
IpAddressDataModelElement("from_ip_v6", ipv6=True)
]),
FixedDataModelElement("s2", b" port "),
DecimalIntegerValueModelElement("port")
]),
SequenceModelElement("invalid-user-auth-req", [
FixedDataModelElement("s0",
b"input_userauth_request: invalid user "),
DelimitedDataModelElement("user", preauth),
FixedDataModelElement("s1", preauth)
]),
SequenceModelElement("postppk", [
FixedDataModelElement("s0", b"Postponed publickey for "),
user_name_model,
FixedDataModelElement("s1", from_str),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s2", port),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s3", b" ssh2 [preauth]")
]),
SequenceModelElement("readerr", [
FixedDataModelElement("s0", b"Read error from remote host "),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s1", b": Connection timed out")
]),
SequenceModelElement("disconnect", [
FixedDataModelElement("s0", b"Received disconnect from "),
FirstMatchModelElement("fm", [
IpAddressDataModelElement("from_ip"),
IpAddressDataModelElement("from_ip_v6", ipv6=True)
]),
FixedDataModelElement("s1", b": 11: "),
FirstMatchModelElement("reason", [
FixedDataModelElement("disconnected", b"disconnected by user"),
SequenceModelElement("remotemsg", [
DelimitedDataModelElement("msg", preauth),
FixedDataModelElement("s0", preauth)
])
])
]),
SequenceModelElement("signal", [
FixedDataModelElement("s0", b"Received signal "),
DecimalIntegerValueModelElement("signal"),
FixedDataModelElement("s1", b"; terminating.")
]),
SequenceModelElement("server", [
FixedDataModelElement("s0", b"Server listening on "),
DelimitedDataModelElement("serverip", b" "),
FixedDataModelElement("s1", port),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s2", b".")
]),
SequenceModelElement("oom-adjust", [
FixedDataModelElement("s0", b"Set /proc/self/oom_score_adj "),
OptionalMatchModelElement(
"from", FixedDataModelElement("default", b"from 0 ")),
FixedDataModelElement("s1", b"to "),
DecimalIntegerValueModelElement(
"newval",
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL)
]),
SequenceModelElement("session-start", [
FixedDataModelElement("s0", b"Starting session: "),
FirstMatchModelElement("sess-info", [
SequenceModelElement("shell", [
FixedDataModelElement("s0", b"shell on "),
DelimitedDataModelElement("terminal", b" ")
]),
SequenceModelElement(
"subsystem",
[FixedDataModelElement("s0", b"subsystem \"sftp\"")]),
SequenceModelElement("forced-command", [
FixedDataModelElement("s0",
b"forced-command (key-option) \""),
DelimitedDataModelElement("command", b"\" for "),
FixedDataModelElement("s1", b"\"")
])
]),
FixedDataModelElement("s1", b" for "), user_name_model,
FixedDataModelElement("s2", from_str),
IpAddressDataModelElement("clientip"),
FixedDataModelElement("s3", port),
DecimalIntegerValueModelElement("port"),
OptionalMatchModelElement(
"idinfo",
SequenceModelElement("idinfo", [
FixedDataModelElement("s0", b" id "),
DecimalIntegerValueModelElement("id")
]))
]),
SequenceModelElement("transferred", [
FixedDataModelElement("s0", b"Transferred: sent "),
DecimalIntegerValueModelElement("sent"),
FixedDataModelElement("s1", b", received "),
DecimalIntegerValueModelElement("received"),
FixedDataModelElement("s1", b" bytes")
]),
SequenceModelElement("pam", [
FixedDataModelElement("s0", b"pam_unix(sshd:session): session "),
FixedWordlistDataModelElement("change", [b"opened", b"closed"]),
FixedDataModelElement("s1", b" for user "), user_name_model,
OptionalMatchModelElement(
"openby", FixedDataModelElement("default", b" by (uid=0)"))
]),
SequenceModelElement("child", [
FixedDataModelElement("s0", b"User child is on pid "),
DecimalIntegerValueModelElement("pid")
]),
SequenceModelElement("failed/accept", [
FixedWordlistDataModelElement("s0", [b"Failed ", b"Accepted "]),
FixedWordlistDataModelElement(
"type", [b"password", b"none", b"publickey"]),
FixedDataModelElement("s1", b" for "),
OptionalMatchModelElement(
"opt", FixedDataModelElement("invalid", b"invalid user ")),
DelimitedDataModelElement("user", b" "),
FixedDataModelElement("s2", b" from "),
FirstMatchModelElement("fm", [
IpAddressDataModelElement("from_ip"),
IpAddressDataModelElement("from_ip_v6", ipv6=True)
]),
FixedDataModelElement("s3", b" port "),
DecimalIntegerValueModelElement("port"),
AnyByteDataModelElement("service")
]),
SequenceModelElement("disconnecting", [
FixedDataModelElement("s0", b"Disconnecting "),
FixedWordlistDataModelElement("type",
[b"authenticating", b"invalid"]),
FixedDataModelElement("s1", b" user "),
DelimitedDataModelElement("user", b" "),
FixedDataModelElement("s1", b" "),
IpAddressDataModelElement("ip"),
FixedDataModelElement("s2", b" port "),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement(
"s3", b": Too many authentication failures [preauth]")
]),
SequenceModelElement("fatal", [
FixedDataModelElement(
"s0", b"fatal: Timeout before authentication for "),
IpAddressDataModelElement("ip"),
FixedDataModelElement("s1", b" port "),
DecimalIntegerValueModelElement("port")
]),
SequenceModelElement("cert-auth", [
FixedDataModelElement("s0", b"cert: Authentication tried for "),
DelimitedDataModelElement("user", b" "),
FixedDataModelElement(
"s1",
b" with valid certificate but not from a permitted source address ("
),
IpAddressDataModelElement("ip"),
FixedDataModelElement("s2", b")."),
]),
SequenceModelElement("change-root-dir", [
FixedDataModelElement("s0", b"Changed root directory to \""),
DelimitedDataModelElement("root-dir", b"\""),
FixedDataModelElement("s1", b"\"")
]),
FixedDataModelElement("subsystem-request",
b"subsystem request for sftp"),
SequenceModelElement("conn-write-poll", [
FixedDataModelElement("s0",
b"packet_write_poll: Connection from "),
IpAddressDataModelElement("from_ip"),
FixedDataModelElement("s1", b" port "),
DecimalIntegerValueModelElement("port"),
FixedDataModelElement("s2", b": Host is down")
]),
SequenceModelElement("debug", [
FixedDataModelElement("s0", b"debug"),
DecimalIntegerValueModelElement("debug-num"),
FixedDataModelElement("s1", b": "),
FirstMatchModelElement("fm", [
SequenceModelElement("seq1", [
FixedDataModelElement("s2", b"Got "),
DecimalIntegerValueModelElement("num1"),
FixedDataModelElement("s3", b"/"),
DecimalIntegerValueModelElement("num2"),
FixedDataModelElement("s4", b" for keepalive")
]),
SequenceModelElement("seq2", [
FixedDataModelElement("s2", b"channel "),
DecimalIntegerValueModelElement("channel-num"),
FixedDataModelElement("s3", b": request "),
DelimitedDataModelElement("mail", b" "),
FixedDataModelElement("s4", b" confirm "),
DecimalIntegerValueModelElement("num")
]),
SequenceModelElement("seq3", [
FixedDataModelElement("s2", b"send packet: type "),
DecimalIntegerValueModelElement("packet-type")
]),
SequenceModelElement("seq4", [
FixedDataModelElement("s2", b"receive packet: type "),
DecimalIntegerValueModelElement("packet-type")
]),
FixedDataModelElement("do-cleanup", b"do_cleanup"),
SequenceModelElement("seq5", [
FixedDataModelElement("s2",
b"session_pty_cleanup: session "),
DecimalIntegerValueModelElement("sess-num"),
FixedDataModelElement("s3", b" release "),
AnyByteDataModelElement("file")
])
])
]),
SequenceModelElement("pam_succeed_if", [
FixedDataModelElement(
"s0", b"pam_succeed_if(sshd:auth): requirement \"uid >= "),
DecimalIntegerValueModelElement("uid"),
FixedDataModelElement("s1", b"\" not met by user \""),
DelimitedDataModelElement("user", b"\""),
FixedDataModelElement("s2", b"\"")
]),
]
model = SequenceModelElement("sshd", [
FixedDataModelElement("sname", b"sshd["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
class JsonModelElementTest(unittest.TestCase):
"""Unittests for the JsonModelElement."""
single_line_json = b'{"menu": {"id": "file", "value": "File", "popup": {"menuitem": [{"value": "New", "onclick": "CreateNewDoc()"}, {' \
b'"value": "Open", "onclick": "OpenDoc()"}, {"value": "Close", "onclick": "CloseDoc()"}]}}}'
single_line_with_optional_key_json = b'{"menu": {"id": "file", "value": "File", "popup": {"menuitem": [{"value": "New", "onclick":' \
b' "CreateNewDoc()", "clickable": false}, {"value": "Open", "onclick": "OpenDoc()"}, {"value": ' \
b'"Close", "onclick": "CloseDoc()", "clickable": false}]}}}'
single_line_missing_key_json = b'{"menu": {"id": "file", "popup": {"menuitem": [{"value": "New", "onclick": "CreateNewDoc()"}, {' \
b'"value": "Open", "onclick": "OpenDoc()"}, {"value": "Close", "onclick": "CloseDoc()"}]}}}'
single_line_different_order_with_optional_key_json = \
b'{"menu": {"value": "File","popup": {"menuitem": [{"clickable": false, "value": "New", "onclick": "CreateNewDoc()"}, {' \
b'"onclick": "OpenDoc()", "value": "Open"}, {"value": "Close", "onclick": "CloseDoc()", "clickable": false}]}, "id": "file"}}'
multi_line_json = b"""{
"menu": {
"id": "file",
"value": "File",
"popup": {
"menuitem": [
{"value": "New", "onclick": "CreateNewDoc()"},
{"value": "Open", "onclick": "OpenDoc()"},
{"value": "Close", "onclick": "CloseDoc()"}
]
}
}
}"""
everything_new_line_json = b"""{
"menu":
{
"id": "file",
"value": "File",
"popup":
{
"menuitem":
[
{
"value": "New",
"onclick": "CreateNewDoc()"
},
{
"value": "Open",
"onclick": "OpenDoc()"
},
{
"value": "Close",
"onclick": "CloseDoc()"
}
]
}
}
}"""
alphabet = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789_-.'
key_parser_dict = {
'menu': {
'id': VariableByteDataModelElement('id', alphabet),
'value': VariableByteDataModelElement('value', alphabet),
'popup': {
'menuitem': [{
'value':
FixedWordlistDataModelElement('buttonNames',
[b'New', b'Open', b'Close']),
'onclick':
FixedWordlistDataModelElement(
'buttonOnclick',
[b'CreateNewDoc()', b'OpenDoc()', b'CloseDoc()']),
'optional_key_clickable':
FixedWordlistDataModelElement('clickable',
[b'true', b'false'])
}]
}
}
}
key_parser_dict_allow_all = {
'menu': {
'id': VariableByteDataModelElement('id', alphabet),
'value': VariableByteDataModelElement('value', alphabet),
'popup': 'ALLOW_ALL'
}
}
def test1get_valid_match_elements(self):
"""Get valid json elements with different formats."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_json))
self.assertEqual(match.match_object, json.loads(self.single_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.multi_line_json))
self.assertEqual(match.match_object, json.loads(self.multi_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.everything_new_line_json))
self.assertEqual(match.match_object,
json.loads(self.everything_new_line_json))
def test2optional_key_prefix(self):
"""Validate optional keys with the optional_key_prefix."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_with_optional_key_json))
self.assertEqual(match.match_object,
json.loads(self.single_line_with_optional_key_json))
def test3missing_key(self):
"""Check if no match is returned if a key is missing."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_missing_key_json))
self.assertEqual(match, None)
def test4allow_all_dict(self):
"""Test a simplified key_parser_dict with ALLOW_ALL."""
json_model_element = JsonModelElement('json',
self.key_parser_dict_allow_all)
match = json_model_element.get_match_element(
'match', MatchContext(self.single_line_json))
self.assertEqual(match.match_object, json.loads(self.single_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.multi_line_json))
self.assertEqual(match.match_object, json.loads(self.multi_line_json))
match = json_model_element.get_match_element(
'match', MatchContext(self.everything_new_line_json))
self.assertEqual(match.match_object,
json.loads(self.everything_new_line_json))
def test5different_order_keys(self):
"""Test if keys differently ordered than in the key_parser_dict are parsed properly."""
json_model_element = JsonModelElement('json', self.key_parser_dict)
match = json_model_element.get_match_element(
'match',
MatchContext(
self.single_line_different_order_with_optional_key_json))
self.assertEqual(
match.match_object,
json.loads(
self.single_line_different_order_with_optional_key_json))
json_model_element = JsonModelElement('json',
self.key_parser_dict_allow_all)
match = json_model_element.get_match_element(
'match',
MatchContext(
self.single_line_different_order_with_optional_key_json))
self.assertEqual(
match.match_object,
json.loads(
self.single_line_different_order_with_optional_key_json))
def test6null_value(self):
"""Test if null values are parsed to "null"."""
key_parser_dict = {
"works":
VariableByteDataModelElement("id", b"abc123"),
"problem":
FixedWordlistDataModelElement("wordlist",
[b"allowed value", b"null"])
}
data1 = b"""{
"works": "abc",
"problem": "allowed value"
}"""
data2 = b"""{
"works": "123",
"problem": null
}"""
json_model_element = JsonModelElement('json', key_parser_dict)
self.assertIsNotNone(
json_model_element.get_match_element('match', MatchContext(data1)))
self.assertIsNotNone(
json_model_element.get_match_element('match', MatchContext(data2)))
def get_model():
"""Return a model to parse a audispd message logged via syslog after any standard logging preamble, e.g. from syslog."""
class ExecArgumentDataModelElement():
"""This is a helper class for parsing the (encoded) exec argument strings found within audit logs."""
def __init__(self, element_id):
self.element_id = element_id
@staticmethod
def get_child_elements():
"""Get the children of this element (none)."""
return None
def get_match_element(self, path, match_context):
"""
Find the maximum number of bytes belonging to an exec argument.
@return a match when at least two bytes were found including the delimiters.
"""
data = match_context.match_data
match_len = 0
match_value = b''
if data[0] == ord(b'"'):
match_len = data.find(b'"', 1)
if match_len == -1:
return None
match_value = data[1:match_len]
match_len += 1
elif data.startswith(b'(null)'):
match_len = 6
match_value = None
else:
# Must be upper case hex encoded:
next_value = -1
for d_byte in data:
if 0x30 <= d_byte <= 0x39:
d_byte -= 0x30
elif 0x41 <= d_byte <= 0x46:
d_byte -= 0x37
else:
break
if next_value == -1:
next_value = (d_byte << 4)
else:
match_value += bytearray(((next_value | d_byte), ))
next_value = -1
match_len += 1
if next_value != -1:
return None
match_data = data[:match_len]
match_context.update(match_data)
return MatchElement("%s/%s" % (path, self.element_id), match_data,
match_value, None)
pam_status_word_list = FixedWordlistDataModelElement(
'status', [b'failed', b'success'])
pid = b' pid='
uid = b' uid='
auid = b' auid='
ses = b' ses='
exe = b' exe="'
hostname = b'" hostname='
addr = b' addr='
terminal = b' terminal='
success = b' res=success\''
res = b' res='
exe1 = b'" exe="'
type_branches = {
'ADD_USER':
SequenceModelElement('adduser', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=adding user id='),
DecimalIntegerValueModelElement('newuserid'),
FixedDataModelElement('s5', exe),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
]),
'BPRM_FCAPS':
SequenceModelElement('bprmfcaps', [
FixedDataModelElement('s0', b' fver=0 fp='),
HexStringModelElement('fp'),
FixedDataModelElement('s1', b' fi='),
HexStringModelElement('fi'),
FixedDataModelElement('s2', b' fe='),
HexStringModelElement('fe'),
FixedDataModelElement('s3', b' old_pp='),
HexStringModelElement('pp-old'),
FixedDataModelElement('s4', b' old_pi='),
HexStringModelElement('pi-old'),
FixedDataModelElement('s5', b' old_pe='),
HexStringModelElement('pe-old'),
FixedDataModelElement('s6', b' new_pp='),
HexStringModelElement('pp-new'),
FixedDataModelElement('s7', b' new_pi='),
HexStringModelElement('pi-new'),
FixedDataModelElement('s8', b' new_pe='),
HexStringModelElement('pe-new')
]),
'CONFIG_CHANGE':
SequenceModelElement('conf-change', [
FixedDataModelElement('s0', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s1', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s2', b' op="add rule" key=(null) list='),
DecimalIntegerValueModelElement('list'),
FixedDataModelElement('s3', res),
DecimalIntegerValueModelElement('result')
]),
'CRED_ACQ':
SequenceModelElement('credacq', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:setcred acct="'),
DelimitedDataModelElement('username', b'"'),
FixedDataModelElement('s5', exe1),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
]),
'CRED_DISP':
SequenceModelElement('creddisp', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:setcred acct="'),
DelimitedDataModelElement('username', b'"'),
FixedDataModelElement('s5', exe1),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
]),
'CRED_REFR':
SequenceModelElement('creddisp', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement(
's4',
b' msg=\'op=PAM:setcred acct="root" exe="/usr/sbin/sshd" hostname='
),
IpAddressDataModelElement('clientname'),
FixedDataModelElement('s5', addr),
IpAddressDataModelElement('clientip'),
FixedDataModelElement('s6', b' terminal=ssh res=success\'')
]),
'CWD':
SequenceModelElement('cwd', [
FixedDataModelElement('s0', b' cwd='),
ExecArgumentDataModelElement('cwd')
]),
'EOE':
OptionalMatchModelElement('eoe', FixedDataModelElement('s0', b''))
}
# We need a type branch here also, but there is no additional
# data in EOE records after Ubuntu Trusty any more.
exec_arg_model = SequenceModelElement('execarg', [
FixedDataModelElement('s0', b' a'),
DecimalIntegerValueModelElement('argn'),
FixedDataModelElement('s1', b'='),
ExecArgumentDataModelElement('argval')
])
type_branches['EXECVE'] = SequenceModelElement('execve', [
FixedDataModelElement('s0', b' argc='),
DecimalIntegerValueModelElement('argc'),
RepeatedElementDataModelElement('arg', exec_arg_model)
])
type_branches['FD_PAIR'] = SequenceModelElement('fdpair', [
FixedDataModelElement('s0', b' fd0='),
DecimalIntegerValueModelElement('fd0'),
FixedDataModelElement('s1', b' fd1='),
DecimalIntegerValueModelElement('fd1')
])
# This message differs on Ubuntu 32/64 bit variants.
type_branches['LOGIN'] = SequenceModelElement('login', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedWordlistDataModelElement('s2', [b' old auid=', b' old-auid=']),
DecimalIntegerValueModelElement('auid-old'),
FixedWordlistDataModelElement('s3', [b' new auid=', auid]),
DecimalIntegerValueModelElement('auid-new'),
FixedWordlistDataModelElement('s4', [b' old ses=', b' old-ses=']),
DecimalIntegerValueModelElement('ses-old'),
FixedWordlistDataModelElement('s5', [b' new ses=', ses]),
DecimalIntegerValueModelElement('ses-new'),
FixedDataModelElement('s6', res),
DecimalIntegerValueModelElement('result')
])
inode_info_model_element = SequenceModelElement(
'inodeinfo',
[
FixedDataModelElement('s0', b' inode='),
DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('s1', b' dev='),
# A special major/minor device element could be better here.
VariableByteDataModelElement('dev', b'0123456789abcdef:'),
FixedDataModelElement('s2', b' mode='),
# is octal
DecimalIntegerValueModelElement('mode'),
FixedDataModelElement('s3', b' ouid='),
DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('s4', b' ogid='),
DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('s5', b' rdev='),
# A special major/minor device element could be better here (see above).
VariableByteDataModelElement('rdev', b'0123456789abcdef:'),
FixedDataModelElement('s6', b' nametype=')
])
type_branches['NETFILTER_CFG'] = SequenceModelElement(
'conf-change', [
FixedDataModelElement('s0', b' table='),
FixedWordlistDataModelElement('table',
[b'filter', b'mangle', b'nat']),
FixedDataModelElement('s1', b' family='),
DecimalIntegerValueModelElement('family'),
FixedDataModelElement('s2', b' entries='),
DecimalIntegerValueModelElement('entries')
])
type_branches['OBJ_PID'] = SequenceModelElement('objpid', [
FixedDataModelElement('s0', b' opid='),
DecimalIntegerValueModelElement('opid'),
FixedDataModelElement('s1', b' oauid='),
DecimalIntegerValueModelElement(
'oauid',
value_sign_type=DecimalIntegerValueModelElement.SIGN_TYPE_OPTIONAL
),
FixedDataModelElement('s2', b' ouid='),
DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('s3', b' oses='),
DecimalIntegerValueModelElement(
'oses',
value_sign_type=DecimalIntegerValueModelElement.SIGN_TYPE_OPTIONAL
),
FixedDataModelElement('s4', b' ocomm='),
ExecArgumentDataModelElement('ocomm')
])
type_branches['PATH'] = SequenceModelElement('path', [
FixedDataModelElement('s0', b' item='),
DecimalIntegerValueModelElement('item'),
FixedDataModelElement('s1', b' name='),
ExecArgumentDataModelElement('name'),
FirstMatchModelElement('fsinfo', [
inode_info_model_element,
FixedDataModelElement('noinfo', b' nametype=')
]),
FixedWordlistDataModelElement(
'nametype',
[b'CREATE', b'DELETE', b'NORMAL', b'PARENT', b'UNKNOWN'])
])
type_branches['PROCTITLE'] = SequenceModelElement('proctitle', [
FixedDataModelElement('s1', b' proctitle='),
ExecArgumentDataModelElement('proctitle')
])
type_branches['SERVICE_START'] = SequenceModelElement(
'service', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'unit='),
DelimitedDataModelElement('unit', b' '),
FixedDataModelElement('s5', b' comm="systemd" exe="'),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', res), pam_status_word_list,
FixedDataModelElement('s10', b'\'')
])
type_branches['SERVICE_STOP'] = type_branches['SERVICE_START']
type_branches['SOCKADDR'] = SequenceModelElement('sockaddr', [
FixedDataModelElement('s0', b' saddr='),
HexStringModelElement('sockaddr', upper_case=True)
])
type_branches['SYSCALL'] = SequenceModelElement('syscall', [
FixedDataModelElement('s0', b' arch='),
HexStringModelElement('arch'),
FixedDataModelElement('s1', b' syscall='),
DecimalIntegerValueModelElement('syscall'),
OptionalMatchModelElement(
'personality',
SequenceModelElement('pseq', [
FixedDataModelElement('s0', b' per='),
DecimalIntegerValueModelElement('personality')
])),
OptionalMatchModelElement(
'result',
SequenceModelElement('rseq', [
FixedDataModelElement('s2', b' success='),
FixedWordlistDataModelElement('succes', [b'no', b'yes']),
FixedDataModelElement('s3', b' exit='),
DecimalIntegerValueModelElement(
'exit',
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL)
])),
FixedDataModelElement('s4', b' a0='),
HexStringModelElement('arg0'),
FixedDataModelElement('s5', b' a1='),
HexStringModelElement('arg1'),
FixedDataModelElement('s6', b' a2='),
HexStringModelElement('arg2'),
FixedDataModelElement('s7', b' a3='),
HexStringModelElement('arg3'),
FixedDataModelElement('s8', b' items='),
DecimalIntegerValueModelElement('items'),
FixedDataModelElement('s9', b' ppid='),
DecimalIntegerValueModelElement('ppid'),
FixedDataModelElement('s10', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s11', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s12', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s13', b' gid='),
DecimalIntegerValueModelElement('gid'),
FixedDataModelElement('s14', b' euid='),
DecimalIntegerValueModelElement('euid'),
FixedDataModelElement('s15', b' suid='),
DecimalIntegerValueModelElement('suid'),
FixedDataModelElement('s16', b' fsuid='),
DecimalIntegerValueModelElement('fsuid'),
FixedDataModelElement('s17', b' egid='),
DecimalIntegerValueModelElement('egid'),
FixedDataModelElement('s18', b' sgid='),
DecimalIntegerValueModelElement('sgid'),
FixedDataModelElement('s19', b' fsgid='),
DecimalIntegerValueModelElement('fsgid'),
FixedDataModelElement('s20', b' tty='),
DelimitedDataModelElement('tty', b' '),
FixedDataModelElement('s21', ses),
DecimalIntegerValueModelElement('sesid'),
FixedDataModelElement('s22', b' comm='),
ExecArgumentDataModelElement('command'),
FixedDataModelElement('s23', exe),
DelimitedDataModelElement('executable', b'"'),
FixedDataModelElement('s24', b'" key='),
AnyByteDataModelElement('key')
])
# The UNKNOWN type is used then audispd does not know the type of the event, usually because the kernel is more recent than
# audispd, thus emiting yet unknown event types.
# * type=1327: procitle: see https://www.redhat.com/archives/linux-audit/2014-February/msg00047.html
type_branches['UNKNOWN[1327]'] = SequenceModelElement(
'unknown-proctitle', [
FixedDataModelElement('s0', b' proctitle='),
ExecArgumentDataModelElement('proctitle')
])
type_branches['USER_ACCT'] = SequenceModelElement('useracct', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:accounting acct="'),
DelimitedDataModelElement('username', b'"'),
FixedDataModelElement('s5', exe1),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
])
type_branches['USER_AUTH'] = SequenceModelElement('userauth', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:authentication acct="'),
DelimitedDataModelElement('username', b'"'),
FixedDataModelElement('s5', exe1),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
])
type_branches['USER_START'] = SequenceModelElement('userstart', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:session_open acct="'),
DelimitedDataModelElement('username', b'"'),
FixedDataModelElement('s5', exe1),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
])
type_branches['USER_END'] = SequenceModelElement('userend', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:session_close acct="'),
DelimitedDataModelElement('username', b'"'),
FixedDataModelElement('s5', exe1),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', success)
])
type_branches['USER_ERR'] = SequenceModelElement('usererr', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=PAM:bad_ident acct="?" exe="'),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s5', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s6', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s7', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s8', b' res=failed\'')
])
type_branches['USER_LOGIN'] = SequenceModelElement('userlogin', [
FixedDataModelElement('s0', pid),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s1', uid),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s2', auid),
DecimalIntegerValueModelElement('auid'),
FixedDataModelElement('s3', ses),
DecimalIntegerValueModelElement('ses'),
FixedDataModelElement('s4', b' msg=\'op=login '),
FirstMatchModelElement('msgtype', [
FixedDataModelElement('loginok', b'id=0'),
SequenceModelElement('loginfail', [
FixedDataModelElement('s0', b'acct='),
ExecArgumentDataModelElement('account')
])
]),
FixedDataModelElement('s5', exe),
DelimitedDataModelElement('exec', b'"'),
FixedDataModelElement('s6', hostname),
DelimitedDataModelElement('clientname', b' '),
FixedDataModelElement('s7', addr),
DelimitedDataModelElement('clientip', b' '),
FixedDataModelElement('s8', terminal),
WhiteSpaceLimitedDataModelElement('terminal'),
FixedDataModelElement('s9', res), pam_status_word_list,
FixedDataModelElement('s10', b'\'')
])
model = SequenceModelElement('audispd', [
FixedDataModelElement('sname', b'audispd: '),
FirstMatchModelElement('msg', [
ElementValueBranchModelElement(
'record',
SequenceModelElement('preamble', [
FixedDataModelElement('s0', b'type='),
WhiteSpaceLimitedDataModelElement('type'),
FixedDataModelElement('s1', b' msg=audit('),
DecimalIntegerValueModelElement('time'),
FixedDataModelElement('s0', b'.'),
DecimalIntegerValueModelElement('ms'),
FixedDataModelElement('s1', b':'),
DecimalIntegerValueModelElement('seq'),
FixedDataModelElement('s2', b'):')
]),
'type',
type_branches,
default_branch=None),
FixedDataModelElement('queue-full',
b'queue is full - dropping event')
])
])
return model
def get_model():
"""Return a model to parse Syslogs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
user_info = SequenceModelElement('user_info', [
FixedDataModelElement('user_str', b'user=<'),
OptionalMatchModelElement('user',
DelimitedDataModelElement('user', b'>')),
FixedDataModelElement('method_str', b'>'),
OptionalMatchModelElement(
'method',
SequenceModelElement('method', [
FixedDataModelElement('method_str', b', method='),
DelimitedDataModelElement('method', b','),
])),
FixedDataModelElement('rip_str', b', rip='),
IpAddressDataModelElement('rip'),
FixedDataModelElement('lip_str', b', lip='),
IpAddressDataModelElement('lip'),
OptionalMatchModelElement(
'mpid',
SequenceModelElement('mpid', [
FixedDataModelElement('mpid_str', b', mpid='),
DecimalIntegerValueModelElement('mpid'),
])),
OptionalMatchModelElement(
'secured', FixedDataModelElement('secured_str', b', secured')),
FixedDataModelElement('session_str', b', session=<'),
DelimitedDataModelElement('session', b'>'),
FixedDataModelElement('bracket_str', b'>'),
])
model = SequenceModelElement(
'model',
[
DateTimeModelElement('time', b'%b %d %H:%M:%S', start_year=2020),
FixedDataModelElement('sp1', b' '),
DelimitedDataModelElement('host', b' '),
FirstMatchModelElement(
'service',
[
SequenceModelElement('dovecot', [
FixedDataModelElement('dovecot_str', b' dovecot: '),
FirstMatchModelElement('imap', [
SequenceModelElement('imap', [
FixedDataModelElement('imap_str', b'imap('),
DelimitedDataModelElement('user', b')'),
FixedDataModelElement('bracket_str', b'): '),
FirstMatchModelElement('message', [
SequenceModelElement(
'logout', [
FixedDataModelElement(
'logout_str',
b'Logged out in='),
DecimalIntegerValueModelElement(
'in'),
FixedDataModelElement(
'out_str', b' out='),
DecimalIntegerValueModelElement(
'out')
]),
SequenceModelElement(
'err_mail', [
FixedDataModelElement(
'mail_str',
b'Error: Failed to autocreate mailbox INBOX: Internal error occurred. '
b'Refer to server log for more information. ['
),
DelimitedDataModelElement(
'err_time', b']'),
FixedDataModelElement(
'brack', b']')
]),
SequenceModelElement(
'err_open', [
FixedDataModelElement(
'err_str', b'Error: '),
DelimitedDataModelElement(
'function_name', b'('),
FixedDataModelElement(
'brack_str1', b'('),
DelimitedDataModelElement(
'arg', b')'),
FixedDataModelElement(
'failed_str',
b') failed: Permission denied (euid='
),
DecimalIntegerValueModelElement(
'euid'),
FixedDataModelElement(
'brack_str2', b'('),
DelimitedDataModelElement(
'euid_user', b')'),
FixedDataModelElement(
'egid_str', b') egid='),
DecimalIntegerValueModelElement(
'egid'),
FixedDataModelElement(
'brack_str3', b'('),
DelimitedDataModelElement(
'egid_user', b')'),
FixedDataModelElement(
'perm_str',
b') missing +w perm: '),
DelimitedDataModelElement(
'mail_path', b','),
FixedDataModelElement(
'group_str',
b', we\'re not in group '),
DecimalIntegerValueModelElement(
'group_id'),
FixedDataModelElement(
'brack_str4', b'('),
DelimitedDataModelElement(
'group_name', b')'),
FixedDataModelElement(
'owned_str',
b'), dir owned by '),
DelimitedDataModelElement(
'owner', b' '),
FixedDataModelElement(
'mode_str', b' mode='),
DelimitedDataModelElement(
'mode', b')'),
FixedDataModelElement(
'brack_str5', b')'),
OptionalMatchModelElement(
'set',
SequenceModelElement(
'set', [
FixedDataModelElement(
'set_str',
b' (set'),
DelimitedDataModelElement(
'param', b'='),
FixedDataModelElement(
'equal_str', b'='),
DelimitedDataModelElement(
'val', b')'),
FixedDataModelElement(
'brack_str6', b')')
]))
]),
SequenceModelElement(
'err_mail', [
FixedDataModelElement(
'mail_str',
b'Failed to autocreate mailbox INBOX: Internal error occurred. '
b'Refer to server log for more information. ['
),
DelimitedDataModelElement(
'err_time', b']'),
FixedDataModelElement(
'brack', b']')
]),
]),
]),
SequenceModelElement('imap_login', [
FixedDataModelElement('imap_login_str',
b'imap-login: '******'login', [
SequenceModelElement(
'disconnected_str', [
FixedDataModelElement(
'disconnected_str',
b'Disconnected '),
FirstMatchModelElement(
'auth', [
SequenceModelElement(
'auth_failed', [
FixedDataModelElement(
'auth_failed_str',
b'(auth failed, '
),
DecimalIntegerValueModelElement(
'attempts'),
FixedDataModelElement(
'attempts_str',
b' attempts in '
),
]),
FixedDataModelElement(
'no_auth_str',
b'(no auth attempts in '
),
FixedDataModelElement(
'no_auth_str',
b'(disconnected before auth was ready, waited '
),
]),
DecimalIntegerValueModelElement(
'duration'),
FixedDataModelElement(
'secs_str', b' secs): '),
user_info
]),
SequenceModelElement(
'login', [
FixedDataModelElement(
'login_str', b'Login: '******'anvil', [
FixedDataModelElement(
'anvil_str', b'Error: anvil:'),
AnyByteDataModelElement(
'anvil_msg')
]),
SequenceModelElement(
'auth_responding', [
FixedDataModelElement(
'auth_responding_str',
b'Warning: Auth process not responding, '
b'delayed sending initial response (greeting): '
), user_info
]),
]),
]),
SequenceModelElement('auth', [
FixedDataModelElement('auth_worker_str',
b'auth: '),
AnyByteDataModelElement('message')
]),
SequenceModelElement('auth_worker', [
FixedDataModelElement('auth_worker_str',
b'auth-worker('),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('brack', b'):'),
AnyByteDataModelElement('message')
]),
SequenceModelElement('master', [
FixedDataModelElement('master_str',
b'master: '),
AnyByteDataModelElement('message')
])
]),
]),
SequenceModelElement('horde', [
FixedDataModelElement('horde_str', b' HORDE: '),
FirstMatchModelElement('horde', [
SequenceModelElement('imp', [
FixedDataModelElement('succ_str', b'[imp] '),
FirstMatchModelElement('imp', [
SequenceModelElement(
'login', [
FixedDataModelElement(
'succ_str',
b'Login success for '),
DelimitedDataModelElement(
'user', b' '),
FixedDataModelElement(
'brack_str1', b' ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement(
'to_str', b') to {'),
DelimitedDataModelElement(
'imap_addr', b'}'),
FixedDataModelElement(
'brack_str2', b'}'),
]),
SequenceModelElement(
'message_sent', [
FixedDataModelElement(
'message_sent_str',
b'Message sent to '),
VariableByteDataModelElement(
'user',
b'.0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'
),
OptionalMatchModelElement(
'mail',
SequenceModelElement(
'mail', [
FixedDataModelElement(
'brack_mail1',
b' <'),
DelimitedDataModelElement(
'mail', b'>'),
FixedDataModelElement(
'brack_mail2',
b'>')
])),
RepeatedElementDataModelElement(
'more_recepients_rep',
SequenceModelElement(
'more_recepients', [
FixedDataModelElement(
'comma_str',
b', '),
VariableByteDataModelElement(
'more_recepients_mail',
b'.0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'
),
OptionalMatchModelElement(
'more_recepients_mail',
SequenceModelElement(
'more_recepients',
[
FixedDataModelElement(
'brack_more_recepients1',
b' <'),
DelimitedDataModelElement(
'more_recepients',
b'>'),
FixedDataModelElement(
'brack_more_recepients2',
b'>')
])),
])),
FixedDataModelElement(
'from_str', b' from '),
DelimitedDataModelElement(
'user', b' '),
FixedDataModelElement(
'brack_str1', b' ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement(
'brack_str2', b')'),
]),
SequenceModelElement(
'login_failed', [
FixedDataModelElement(
'succ_str',
b'FAILED LOGIN for '),
DelimitedDataModelElement(
'user', b' '),
FixedDataModelElement(
'brack_str1', b' ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement(
'to_str', b') to {'),
DelimitedDataModelElement(
'imap_addr', b'}'),
FixedDataModelElement(
'brack_str2', b'}'),
]),
SequenceModelElement(
'status', [
FixedDataModelElement(
'status_str',
b'[status] Could not open mailbox "INBOX".'
),
]),
SequenceModelElement(
'sync_token', [
FixedDataModelElement(
'sync_token_str',
b'[getSyncToken] IMAP error reported by server.'
),
]),
SequenceModelElement(
'auth_failed', [
FixedDataModelElement(
'auth_failed_str',
b'[login] Authentication failed.'
),
]),
]),
]),
SequenceModelElement('horde', [
FixedDataModelElement('succ_str', b'[horde] '),
FirstMatchModelElement('horde', [
SequenceModelElement(
'success', [
FixedDataModelElement(
'success_str',
b'Login success for '),
DelimitedDataModelElement(
'user', b' '),
FixedDataModelElement(
'brack_str1', b' to horde ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement(
'brack_str2', b')'),
]),
SequenceModelElement(
'success', [
FixedDataModelElement(
'success_str', b'User '),
DelimitedDataModelElement(
'user', b' '),
FixedDataModelElement(
'brack_str1',
b' logged out of Horde ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement(
'brack_str2', b')'),
]),
SequenceModelElement(
'login_failed', [
FixedDataModelElement(
'failed_str',
b'FAILED LOGIN for '),
DelimitedDataModelElement(
'user', b' '),
FixedDataModelElement(
'to_horde_str',
b' to horde ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement(
'brack_str', b')'),
]),
])
]),
SequenceModelElement('function', [
FixedWordlistDataModelElement(
'horde_function',
[b'[nag]', b'[turba]', b'[horde]']),
FixedDataModelElement('nag_str',
b' PHP ERROR: '),
FirstMatchModelElement('php_error', [
SequenceModelElement(
'declaration', [
FixedDataModelElement(
'declaration_str',
b'Declaration of '),
DelimitedDataModelElement(
'function_name1', b'('),
FixedDataModelElement(
'brack_str1', b'('),
OptionalMatchModelElement(
'arg1',
DelimitedDataModelElement(
'arg1', b')')),
FixedDataModelElement(
'failed_str',
b') should be compatible with '
),
DelimitedDataModelElement(
'function_name2', b'('),
FixedDataModelElement(
'brack_str2', b'('),
OptionalMatchModelElement(
'arg2',
DelimitedDataModelElement(
'arg2', b')')),
FixedDataModelElement(
'brack_str3', b')'),
]),
FixedDataModelElement(
'file_str',
b'finfo_file(): Empty filename or path'
),
FixedDataModelElement(
'header_str',
b'Cannot modify header information - headers already sent'
)
])
]),
SequenceModelElement('guest', [
FixedDataModelElement(
'guest_str',
b'Guest user is not authorized for Horde (Host: '
),
IpAddressDataModelElement('ip'),
FixedDataModelElement('brack_str', b').')
]),
SequenceModelElement('php_error', [
FixedDataModelElement('php_error_str',
b'PHP ERROR: '),
DelimitedDataModelElement('msg', b' ['),
]),
SequenceModelElement('free_msg', [
DelimitedDataModelElement('msg', b' ['),
])
]),
FixedDataModelElement('to_str', b' [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('line_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('of_str', b' of "'),
DelimitedDataModelElement('path', b'"'),
FixedDataModelElement('brack_str', b'"]')
]),
SequenceModelElement(
'cron',
[
FixedDataModelElement('cron_str', b' CRON['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('brack_str1', b']: '),
FirstMatchModelElement(
'cron',
[
SequenceModelElement(
'cmd', [
FixedDataModelElement(
'brack_str', b'('),
DelimitedDataModelElement(
'user', b')'),
FixedDataModelElement(
'cmd_str', b') CMD '),
AnyByteDataModelElement('cmd_msg')
]),
SequenceModelElement(
'session',
[ # This only occurs in auth.log
DelimitedDataModelElement(
'pam', b'('),
FixedDataModelElement(
'brack_str', b'('),
DelimitedDataModelElement(
'name', b')'),
FixedDataModelElement(
'session_str', b'): session '),
FixedWordlistDataModelElement(
'status',
[b'opened', b'closed']),
FixedDataModelElement(
'user_str', b' for user '),
VariableByteDataModelElement(
'user', alphabet),
OptionalMatchModelElement(
'uid',
SequenceModelElement(
'uid', [
FixedDataModelElement(
'uid_str',
b' by (uid='),
DecimalIntegerValueModelElement(
'uid'),
FixedDataModelElement(
'brack_str', b')')
]))
])
])
]),
SequenceModelElement(
'auth',
[ # This only occurs in auth.log
FixedDataModelElement('auth_str', b' auth: '),
DelimitedDataModelElement('pam', b'('),
FixedDataModelElement('brack_str', b'('),
DelimitedDataModelElement('name', b')'),
FixedDataModelElement(
'session_str',
b'): authentication failure; logname='),
OptionalMatchModelElement(
'logname',
DelimitedDataModelElement('logname', b' ')),
FixedDataModelElement('uid_str', b' uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('euid_str', b' euid='),
DecimalIntegerValueModelElement('euid'),
FixedDataModelElement('tty_str', b' tty='),
DelimitedDataModelElement('tty', b' '),
FixedDataModelElement('ruser_str', b' ruser='******'ruser', b' '),
FixedDataModelElement('rhost_str', b' rhost='),
IpAddressDataModelElement('rhost'),
OptionalMatchModelElement(
'user',
SequenceModelElement('user', [
FixedDataModelElement(
'user_str', b' user='******'user', alphabet)
]))
]),
SequenceModelElement('systemd', [
FixedDataModelElement('systemd_str', b' systemd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('brack_str1', b']: '),
FirstMatchModelElement('systemd', [
FixedDataModelElement(
'php_starting_str',
b'Starting Clean php session files...'),
FixedDataModelElement(
'php_started_str',
b'Started Clean php session files.'),
FixedDataModelElement(
'php_starting_str',
b'Starting Cleanup of Temporary Directories...'
),
FixedDataModelElement(
'php_started_str',
b'Started Cleanup of Temporary Directories.'),
FixedDataModelElement(
'php_started_str',
b'Starting Daily apt upgrade and clean activities...'
),
FixedDataModelElement(
'php_started_str',
b'Started Daily apt upgrade and clean activities.'
),
FixedDataModelElement(
'php_started_str',
b'Starting Daily apt download activities...'),
FixedDataModelElement(
'php_started_str',
b'Started Daily apt download activities.'),
FixedDataModelElement(
'php_started_str',
b'Starting Security Auditing Service...'),
FixedDataModelElement(
'php_started_str',
b'Started Security Auditing Service.'),
FixedDataModelElement(
'php_started_str',
b'Stopping Security Auditing Service...'),
FixedDataModelElement(
'php_started_str',
b'Stopped Security Auditing Service.'),
FixedDataModelElement(
'php_started_str',
b'Reloading The Apache HTTP Server.'),
FixedDataModelElement(
'php_started_str',
b'Reloaded The Apache HTTP Server.'),
FixedDataModelElement(
'php_started_str',
b'Mounting Arbitrary Executable File Formats File System...'
),
FixedDataModelElement(
'php_started_str',
b'Mounted Arbitrary Executable File Formats File System.'
),
SequenceModelElement('apt', [
FixedDataModelElement('apt_str', b'apt'),
AnyByteDataModelElement('apt_msg')
]),
SequenceModelElement('service', [
DelimitedDataModelElement('service', b':'),
FixedDataModelElement('col_str', b': '),
AnyByteDataModelElement('_msg')
])
]),
]),
SequenceModelElement('kernel', [
FixedDataModelElement('kernel_str', b' kernel'),
OptionalMatchModelElement(
'id',
SequenceModelElement('id', [
FixedDataModelElement('brack_str', b'['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str2', b']')
])),
FixedDataModelElement('col_str', b': '),
AnyByteDataModelElement('kernel_msg')
]),
SequenceModelElement('augenrules', [
FixedDataModelElement('augenrules_str',
b' augenrules['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
AnyByteDataModelElement('augenrules_msg')
]),
SequenceModelElement('auditd', [
FixedDataModelElement('auditd_str', b' auditd['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
AnyByteDataModelElement('auditd_msg')
]),
SequenceModelElement('auditd2', [
FixedDataModelElement('auditd2_str', b' auditd: '),
AnyByteDataModelElement('auditd_msg')
]),
SequenceModelElement('audispd', [
FixedDataModelElement('audispd_str', b' audispd: '),
AnyByteDataModelElement('audispd_msg')
]),
SequenceModelElement('liblogging', [
FixedDataModelElement('liblogging_str',
b' liblogging-stdlog: '),
AnyByteDataModelElement('liblogging_msg')
]),
SequenceModelElement('freshclam', [
FixedDataModelElement('freshclam_str', b' freshclam['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
AnyByteDataModelElement('freshclam_msg')
]),
SequenceModelElement('dhclient', [
FixedDataModelElement('dhclient_str', b' dhclient['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
FirstMatchModelElement('dhclient', [
SequenceModelElement('dhcprequest', [
FixedDataModelElement('dhcprequest_str',
b'DHCPREQUEST of '),
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('on_str', b' on '),
DelimitedDataModelElement(
'network_interface', b' '),
FixedDataModelElement('to_str', b' to '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('port_str', b' port '),
DecimalIntegerValueModelElement('port')
]),
SequenceModelElement('dhcpack', [
FixedDataModelElement('dhcpack_str',
b'DHCPACK of '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('on_str', b' from '),
IpAddressDataModelElement('src_ip')
]),
SequenceModelElement('bound', [
FixedDataModelElement('bound_str',
b'bound to '),
IpAddressDataModelElement('ip'),
FixedDataModelElement('renewal_str',
b' -- renewal in '),
DecimalIntegerValueModelElement('seconds'),
FixedDataModelElement('seconds_str',
b' seconds.')
]),
]),
]),
])
])
return model
def test4get_match_element_no_match(self):
"""Parse not matching substring from MatchContext and check if the MatchContext was not changed."""
data = b"string wordlist"
match_context = DummyMatchContext(data)
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
match_element = fixed_wordlist_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"wor wordlist"
match_context = DummyMatchContext(data)
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
match_element = fixed_wordlist_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"0 wordlist"
match_context = DummyMatchContext(data)
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
match_element = fixed_wordlist_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
data = b"1 word"
match_context = DummyMatchContext(data)
fixed_wordlist_dme = FixedWordlistDataModelElement(
self.id_, self.wordlist)
match_element = fixed_wordlist_dme.get_match_element(
self.path, match_context)
self.compare_no_match_results(data, match_element, match_context)
def get_model():
"""Return a model for su session information messages after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement("build-stack", [
FixedDataModelElement("s0",
b"building new pluginstance stack: \""),
DelimitedDataModelElement("stack", b"\""),
FixedDataModelElement("s1", b"\"")
]),
SequenceModelElement(
"nfct-event",
[
FixedDataModelElement("s0", b"[DESTROY] ORIG: SRC="),
IpAddressDataModelElement("osrcip"),
FixedDataModelElement("s1", b" DST="),
IpAddressDataModelElement("odstip"),
FixedDataModelElement("s2", b" PROTO="),
FixedWordlistDataModelElement("proto", [b"TCP", b"UDP"]),
FixedDataModelElement("s3", b" SPT="),
DecimalIntegerValueModelElement("ospt"),
FixedDataModelElement("s4", b" DPT="),
DecimalIntegerValueModelElement("odpt"),
FixedDataModelElement("s5", b" PKTS="),
DecimalIntegerValueModelElement("opkts"),
FixedDataModelElement("s6", b" BYTES="),
DecimalIntegerValueModelElement("obytes"),
FixedDataModelElement("s7", b" , REPLY: SRC="),
IpAddressDataModelElement("rsrcip"),
FixedDataModelElement("s8", b" DST="),
IpAddressDataModelElement("rdstip"),
FixedDataModelElement("s9", b" PROTO="),
FixedWordlistDataModelElement("rproto", [b"TCP", b"UDP"]),
FixedDataModelElement("s10", b" SPT="),
DecimalIntegerValueModelElement("rspt"),
FixedDataModelElement("s11", b" DPT="),
DecimalIntegerValueModelElement("rdpt"),
FixedDataModelElement("s12", b" PKTS="),
DecimalIntegerValueModelElement("rpkts"),
FixedDataModelElement("s13", b" BYTES="),
DecimalIntegerValueModelElement("rbytes"),
# No additional whitespace from Ubuntu Trusty 14.04 on.
OptionalMatchModelElement("tail",
FixedDataModelElement("s0", b" "))
]),
FixedDataModelElement("nfct-plugin",
b"NFCT plugin working in event mode"),
FixedDataModelElement("reopen", b"reopening capture file"),
FixedDataModelElement("signal",
b"signal received, calling pluginstances"),
FixedDataModelElement("uidchange", b"Changing UID / GID"),
SequenceModelElement("seq", [
FixedDataModelElement("s0", b"id=\""),
DecimalIntegerValueModelElement("id"),
FixedDataModelElement("s1", b"\" severity=\""),
DelimitedDataModelElement("severity", b"\""),
FixedDataModelElement("s2", b"\" sys=\""),
DelimitedDataModelElement("sys", b"\""),
FixedDataModelElement("s3", b"\" sub=\""),
DelimitedDataModelElement("sub", b"\""),
FixedDataModelElement("s4", b"\" name=\""),
DelimitedDataModelElement("name", b"\""),
FixedDataModelElement("s5", b"\" action=\""),
DelimitedDataModelElement("action", b"\""),
FixedDataModelElement("s6", b"\" fwrule=\""),
DelimitedDataModelElement("fwrule", b"\""),
FixedDataModelElement("s7", b"\" initf=\""),
DelimitedDataModelElement("initf", b"\""),
FixedDataModelElement("s8", b"\" srcmac=\""),
DelimitedDataModelElement("srcmac", b"\""),
FixedDataModelElement("s9", b"\" dstmac=\""),
DelimitedDataModelElement("dstmac", b"\""),
FixedDataModelElement("s10", b"\" srcip=\""),
DelimitedDataModelElement("srcip", b"\""),
FixedDataModelElement("s11", b"\" dstip=\""),
DelimitedDataModelElement("dstip", b"\""),
FixedDataModelElement("s12", b"\" proto=\""),
DelimitedDataModelElement("proto", b"\""),
FixedDataModelElement("s13", b"\" length=\""),
DelimitedDataModelElement("length", b"\""),
FixedDataModelElement("s14", b"\" tos=\""),
DelimitedDataModelElement("tos", b"\""),
FixedDataModelElement("s15", b"\" prec=\""),
DelimitedDataModelElement("prec", b"\""),
FixedDataModelElement("s16", b"\" ttl=\""),
DelimitedDataModelElement("ttl", b"\""),
FixedDataModelElement("s17", b"\" srcport=\""),
DelimitedDataModelElement("srcport", b"\""),
FixedDataModelElement("s18", b"\" dstport=\""),
DelimitedDataModelElement("dstport", b"\""),
FixedDataModelElement("s19", b"\" tcpflags=\""),
DelimitedDataModelElement("tcpflags", b"\""),
FixedDataModelElement("s20", b"\"")
])
]
# Netflow entry
model = SequenceModelElement("ulogd", [
FixedDataModelElement("sname", b"ulogd["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return a model to parse Suricata Event logs from the AIT-LDS."""
conn = SequenceModelElement('conn', [
FixedDataModelElement('src_ip_str', b'"src_ip":"'),
FirstMatchModelElement('ip', [
SequenceModelElement('ipv4', [
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('src_port_str', b'","src_port":'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('dest_ip_str', b',"dest_ip":"'),
IpAddressDataModelElement('dest_ip'),
FixedDataModelElement('dest_port_str', b'","dest_port":'),
DecimalIntegerValueModelElement('dest_port'),
FixedDataModelElement('proto_str', b',"proto":"'),
DelimitedDataModelElement('proto', b'"'),
FixedDataModelElement('quote', b'"')
]),
SequenceModelElement('ipv6', [
DelimitedDataModelElement('src_ip', b'"'),
FixedDataModelElement('dest_ip_str', b'","dest_ip":"'),
DelimitedDataModelElement('dest_ip', b'"'),
FixedDataModelElement('proto_str', b'","proto":"'),
DelimitedDataModelElement('proto', b'"'),
FixedDataModelElement('icmp_type_str', b'","icmp_type":'),
DecimalIntegerValueModelElement('icmp_type'),
FixedDataModelElement('icmp_code_str', b',"icmp_code":'),
DecimalIntegerValueModelElement('icmp_code'),
]),
])
])
http = SequenceModelElement('http', [
FixedDataModelElement('hostname_str', b',"http":{"hostname":"'),
DelimitedDataModelElement('hostname', b'"'),
FixedDataModelElement('url_str', b'","url":"'),
DelimitedDataModelElement('url', b'"', escape=b'\\'),
FixedDataModelElement('http_user_agent_str', b'","http_user_agent":"'),
DelimitedDataModelElement('http_user_agent', b'"'),
OptionalMatchModelElement(
'content_type',
SequenceModelElement('content_type', [
FixedDataModelElement('http_content_type_str',
b'","http_content_type":"'),
DelimitedDataModelElement('http_content_type', b'"'),
])),
OptionalMatchModelElement(
'http_refer',
SequenceModelElement('http_refer', [
FixedDataModelElement('http_refer_str', b'","http_refer":"'),
DelimitedDataModelElement('http_refer', b'"'),
])),
FixedDataModelElement('http_method_str', b'","http_method":"'),
DelimitedDataModelElement('http_method', b'"'),
FixedDataModelElement('protocol_str', b'","protocol":"'),
DelimitedDataModelElement('protocol', b'"'),
FixedDataModelElement('quote_str', b'"'),
OptionalMatchModelElement(
'status',
SequenceModelElement('status', [
FixedDataModelElement('status_str', b',"status":'),
DecimalIntegerValueModelElement('status'),
])),
OptionalMatchModelElement(
'redirect',
SequenceModelElement('redirect', [
FixedDataModelElement('redirect_str', b',"redirect":"'),
DelimitedDataModelElement('redirect', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
FixedDataModelElement('length_str', b',"length":'),
DecimalIntegerValueModelElement('length'),
FixedDataModelElement('brack_str', b'}')
])
model = SequenceModelElement('model', [
FixedDataModelElement('time_str', b'{"timestamp":"'),
DateTimeModelElement('time', b'%Y-%m-%dT%H:%M:%S.%f'),
FixedDataModelElement('plus_sign', b'+'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('comma_str', b'",'),
OptionalMatchModelElement(
'flow_id',
SequenceModelElement('flow_id', [
FixedDataModelElement('flow_id_str', b'"flow_id":'),
DecimalIntegerValueModelElement('flow_id'),
FixedDataModelElement('comma_str', b',')
])),
OptionalMatchModelElement(
'in_iface',
SequenceModelElement('in_iface', [
FixedDataModelElement('in_iface_str', b'"in_iface":"'),
DelimitedDataModelElement('in_iface', b'"'),
FixedDataModelElement('comma_str', b'",')
])),
FixedDataModelElement('event_type_str', b'"event_type":"'),
FirstMatchModelElement('event_type', [
SequenceModelElement('dns', [
FixedDataModelElement('dns_str', b'dns",'),
conn,
SequenceModelElement('dns', [
FixedDataModelElement('type_str', b',"dns":{"type":"'),
DelimitedDataModelElement('type', b'"'),
FixedDataModelElement('id_str', b'","id":'),
DecimalIntegerValueModelElement('id'),
OptionalMatchModelElement(
'rcode',
SequenceModelElement('rcode', [
FixedDataModelElement('rcode_str', b',"rcode":"'),
DelimitedDataModelElement('rcode', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
FixedDataModelElement('rrname_str', b',"rrname":"'),
DelimitedDataModelElement('rrname', b'"'),
OptionalMatchModelElement(
'rrtype',
SequenceModelElement('rrtype', [
FixedDataModelElement('rrtype_str',
b'","rrtype":"'),
DelimitedDataModelElement('rrtype', b'"')
])),
FixedDataModelElement('quote', b'"'),
OptionalMatchModelElement(
'tx_id',
SequenceModelElement('tx_id', [
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id')
])),
OptionalMatchModelElement(
'ttl',
SequenceModelElement('ttl', [
FixedDataModelElement('ttl_str', b',"ttl":'),
DecimalIntegerValueModelElement('ttl')
])),
OptionalMatchModelElement(
'rdata',
SequenceModelElement('rdata', [
FixedDataModelElement('rdata_str', b',"rdata":"'),
DelimitedDataModelElement('rdata', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
FixedDataModelElement('brack_str', b'}}')
]),
]),
SequenceModelElement('flow', [
FixedDataModelElement('flow_str', b'flow",'),
conn,
OptionalMatchModelElement(
'app_proto',
SequenceModelElement('app_proto', [
FixedDataModelElement('app_proto_str',
b',"app_proto":"'),
DelimitedDataModelElement('app_proto', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
OptionalMatchModelElement(
'app_proto_tc',
SequenceModelElement('app_proto_tc', [
FixedDataModelElement('app_proto_tc_str',
b',"app_proto_tc":"'),
DelimitedDataModelElement('app_proto_tc', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
SequenceModelElement('flow', [
FixedDataModelElement('pkts_toserver_str',
b',"flow":{"pkts_toserver":'),
DecimalIntegerValueModelElement('pkts_toserver'),
FixedDataModelElement('pkts_toclient_str',
b',"pkts_toclient":'),
DecimalIntegerValueModelElement('pkts_toclient'),
FixedDataModelElement('bytes_toserver_str',
b',"bytes_toserver":'),
DecimalIntegerValueModelElement('bytes_toserver'),
FixedDataModelElement('bytes_toclient_str',
b',"bytes_toclient":'),
DecimalIntegerValueModelElement('bytes_toclient'),
FixedDataModelElement('start_str', b',"start":"'),
DelimitedDataModelElement('start', b'"'),
FixedDataModelElement('end_str', b'","end":"'),
DelimitedDataModelElement('end', b'"'),
FixedDataModelElement('age_str', b'","age":'),
DecimalIntegerValueModelElement('age'),
FixedDataModelElement('state_str', b',"state":"'),
DelimitedDataModelElement('state', b'"'),
FixedDataModelElement('reason_str', b'","reason":"'),
DelimitedDataModelElement('reason', b'"'),
FixedDataModelElement('alerted_str', b'","alerted":'),
FixedWordlistDataModelElement('alerted',
[b'true', b'false']),
FixedDataModelElement('brack_str1', b'}'),
OptionalMatchModelElement(
'tcp',
SequenceModelElement('tcp', [
FixedDataModelElement('tcp_flags_str',
b',"tcp":{"tcp_flags":"'),
HexStringModelElement('tcp_flags'),
FixedDataModelElement('tcp_flags_ts_str',
b'","tcp_flags_ts":"'),
HexStringModelElement('tcp_flags_ts'),
FixedDataModelElement('tcp_flags_tc_str',
b'","tcp_flags_tc":"'),
HexStringModelElement('tcp_flags_tc'),
OptionalMatchModelElement(
'flags',
SequenceModelElement('flags', [
FixedDataModelElement(
'syn_str', b'","syn":'),
FixedWordlistDataModelElement(
'syn', [b'true', b'false']),
OptionalMatchModelElement(
'fin',
SequenceModelElement(
'fin', [
FixedDataModelElement(
'fin_str', b',"fin":'),
FixedWordlistDataModelElement(
'fin',
[b'true', b'false']),
])),
OptionalMatchModelElement(
'rst',
SequenceModelElement(
'rst', [
FixedDataModelElement(
'rst_str', b',"rst":'),
FixedWordlistDataModelElement(
'rst',
[b'true', b'false']),
])),
OptionalMatchModelElement(
'psh',
SequenceModelElement(
'psh', [
FixedDataModelElement(
'psh_str', b',"psh":'),
FixedWordlistDataModelElement(
'psh',
[b'true', b'false']),
])),
FixedDataModelElement(
'ack_str', b',"ack":'),
FixedWordlistDataModelElement(
'ack', [b'true', b'false']),
FixedDataModelElement(
'tcp_state_str', b',"state":"'),
DelimitedDataModelElement(
'tcp_state', b'"'),
])),
FixedDataModelElement('tcp_brack_str', b'"}'),
])),
FixedDataModelElement('brack_str2', b'}')
]),
]),
SequenceModelElement('http', [
FixedDataModelElement('http_str', b'http",'), conn,
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'), http,
FixedDataModelElement('brack_str', b'}')
]),
SequenceModelElement('fileinfo', [
FixedDataModelElement('fileinfo_str', b'fileinfo",'),
conn,
http,
FixedDataModelElement('app_proto_str', b',"app_proto":"'),
DelimitedDataModelElement('app_proto', b'"'),
SequenceModelElement('fileinfo', [
FixedDataModelElement('fileinfo_str', b'","fileinfo":{'),
OptionalMatchModelElement(
'filename',
SequenceModelElement('filename', [
FixedDataModelElement('filename_str',
b'"filename":"'),
DelimitedDataModelElement('filename', b'"'),
FixedDataModelElement('quote_str', b'",')
])),
FixedDataModelElement('state_str', b'"state":"'),
DelimitedDataModelElement('state', b'"'),
FixedDataModelElement('stored_str', b'","stored":'),
FixedWordlistDataModelElement('stored',
[b'true', b'false']),
FixedDataModelElement('size_str', b',"size":'),
DecimalIntegerValueModelElement('size'),
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
FixedDataModelElement('brack_str', b'}}')
]),
]),
SequenceModelElement('stats', [
FixedDataModelElement('stats_str', b'stats",'),
FixedDataModelElement('uptime_str', b'"stats":{"uptime":'),
DecimalIntegerValueModelElement('uptime'),
SequenceModelElement('capture', [
FixedDataModelElement('capture_str', b',"capture":{'),
FixedDataModelElement('kernel_packets_str',
b'"kernel_packets":'),
DecimalIntegerValueModelElement('kernel_packets'),
FixedDataModelElement('kernel_drops_str',
b',"kernel_drops":'),
DecimalIntegerValueModelElement('kernel_drops'),
FixedDataModelElement('brack_str', b'}')
]),
SequenceModelElement('decoder', [
FixedDataModelElement('pkts_str', b',"decoder":{"pkts":'),
DecimalIntegerValueModelElement('pkts'),
FixedDataModelElement('bytes_str', b',"bytes":'),
DecimalIntegerValueModelElement('bytes'),
FixedDataModelElement('invalid_str', b',"invalid":'),
DecimalIntegerValueModelElement('invalid'),
FixedDataModelElement('ipv4_str', b',"ipv4":'),
DecimalIntegerValueModelElement('ipv4'),
FixedDataModelElement('ipv6_str', b',"ipv6":'),
DecimalIntegerValueModelElement('ipv6'),
FixedDataModelElement('ethernet_str', b',"ethernet":'),
DecimalIntegerValueModelElement('ethernet'),
FixedDataModelElement('raw_str', b',"raw":'),
DecimalIntegerValueModelElement('raw'),
FixedDataModelElement('null_str', b',"null":'),
DecimalIntegerValueModelElement('null'),
FixedDataModelElement('sll_str', b',"sll":'),
DecimalIntegerValueModelElement('sll'),
FixedDataModelElement('tcp_str', b',"tcp":'),
DecimalIntegerValueModelElement('tcp'),
FixedDataModelElement('udp_str', b',"udp":'),
DecimalIntegerValueModelElement('udp'),
FixedDataModelElement('sctp_str', b',"sctp":'),
DecimalIntegerValueModelElement('sctp'),
FixedDataModelElement('icmpv4_str', b',"icmpv4":'),
DecimalIntegerValueModelElement('icmpv4'),
FixedDataModelElement('icmpv6_str', b',"icmpv6":'),
DecimalIntegerValueModelElement('icmpv6'),
FixedDataModelElement('ppp_str', b',"ppp":'),
DecimalIntegerValueModelElement('ppp'),
FixedDataModelElement('pppoe_str', b',"pppoe":'),
DecimalIntegerValueModelElement('pppoe'),
FixedDataModelElement('gre_str', b',"gre":'),
DecimalIntegerValueModelElement('gre'),
FixedDataModelElement('vlan_str', b',"vlan":'),
DecimalIntegerValueModelElement('vlan'),
FixedDataModelElement('vlan_qinq_str', b',"vlan_qinq":'),
DecimalIntegerValueModelElement('vlan_qinq'),
FixedDataModelElement('teredo_str', b',"teredo":'),
DecimalIntegerValueModelElement('teredo'),
FixedDataModelElement('ipv4_in_ipv6_str',
b',"ipv4_in_ipv6":'),
DecimalIntegerValueModelElement('ipv4_in_ipv6'),
FixedDataModelElement('ipv6_in_ipv6_str',
b',"ipv6_in_ipv6":'),
DecimalIntegerValueModelElement('ipv6_in_ipv6'),
FixedDataModelElement('mpls_str', b',"mpls":'),
DecimalIntegerValueModelElement('mpls'),
FixedDataModelElement('avg_pkt_size_str',
b',"avg_pkt_size":'),
DecimalIntegerValueModelElement('avg_pkt_size'),
FixedDataModelElement('max_pkt_size_str',
b',"max_pkt_size":'),
DecimalIntegerValueModelElement('max_pkt_size'),
FixedDataModelElement('erspan_str', b',"erspan":'),
DecimalIntegerValueModelElement('erspan'),
SequenceModelElement('ipraw', [
FixedDataModelElement(
'invalid_ip_version_str',
b',"ipraw":{"invalid_ip_version":'),
DecimalIntegerValueModelElement('invalid_ip_version'),
]),
SequenceModelElement('ltnull', [
FixedDataModelElement('ipraw_pkt_too_small_str',
b'},"ltnull":{"pkt_too_small":'),
DecimalIntegerValueModelElement('ipraw_pkt_too_small'),
FixedDataModelElement('unsupported_type',
b',"unsupported_type":'),
DecimalIntegerValueModelElement('unsupported_type'),
]),
SequenceModelElement('dce', [
FixedDataModelElement('dce_pkt_too_small_str',
b'},"dce":{"pkt_too_small":'),
DecimalIntegerValueModelElement('dce_pkt_too_small'),
FixedDataModelElement('brack_str', b'}')
])
]),
SequenceModelElement('flow', [
FixedDataModelElement('memcap_str',
b'},"flow":{"memcap":'),
DecimalIntegerValueModelElement('memcap'),
FixedDataModelElement('spare_str', b',"spare":'),
DecimalIntegerValueModelElement('spare'),
FixedDataModelElement('emerg_mode_entered_str',
b',"emerg_mode_entered":'),
DecimalIntegerValueModelElement('emerg_mode_entered'),
FixedDataModelElement('emerg_mode_over_str',
b',"emerg_mode_over":'),
DecimalIntegerValueModelElement('emerg_mode_over'),
FixedDataModelElement('tcp_reuse_str', b',"tcp_reuse":'),
DecimalIntegerValueModelElement('tcp_reuse'),
FixedDataModelElement('memuse_str', b',"memuse":'),
DecimalIntegerValueModelElement('memuse'),
]),
SequenceModelElement('defrag', [
SequenceModelElement('ipv4', [
FixedDataModelElement(
'fragments_str',
b'},"defrag":{"ipv4":{"fragments":'),
DecimalIntegerValueModelElement('fragments'),
FixedDataModelElement('reassembled_str',
b',"reassembled":'),
DecimalIntegerValueModelElement('reassembled_str'),
FixedDataModelElement('timeouts_str', b',"timeouts":'),
DecimalIntegerValueModelElement('timeouts'),
]),
SequenceModelElement('ipv6', [
FixedDataModelElement('fragments_str',
b'},"ipv6":{"fragments":'),
DecimalIntegerValueModelElement('fragments'),
FixedDataModelElement('reassembled_str',
b',"reassembled":'),
DecimalIntegerValueModelElement('reassembled_str'),
FixedDataModelElement('timeouts_str', b',"timeouts":'),
DecimalIntegerValueModelElement('timeouts'),
]),
FixedDataModelElement('max_frag_hits_str',
b'},"max_frag_hits":'),
DecimalIntegerValueModelElement('max_frag_hits'),
]),
SequenceModelElement('tcp', [
FixedDataModelElement('sessions_str',
b'},"tcp":{"sessions":'),
DecimalIntegerValueModelElement('sessions'),
FixedDataModelElement('ssn_memcap_drop_str',
b',"ssn_memcap_drop":'),
DecimalIntegerValueModelElement('ssn_memcap_drop'),
FixedDataModelElement('pseudo_str', b',"pseudo":'),
DecimalIntegerValueModelElement('pseudo'),
FixedDataModelElement('pseudo_failed_str',
b',"pseudo_failed":'),
DecimalIntegerValueModelElement('pseudo_failed'),
FixedDataModelElement('invalid_checksum_str',
b',"invalid_checksum":'),
DecimalIntegerValueModelElement('invalid_checksum'),
FixedDataModelElement('no_flow_str', b',"no_flow":'),
DecimalIntegerValueModelElement('no_flow'),
FixedDataModelElement('syn_str', b',"syn":'),
DecimalIntegerValueModelElement('syn'),
FixedDataModelElement('synack_str', b',"synack":'),
DecimalIntegerValueModelElement('synack'),
FixedDataModelElement('rst_str', b',"rst":'),
DecimalIntegerValueModelElement('rst'),
FixedDataModelElement('segment_memcap_drop_str',
b',"segment_memcap_drop":'),
DecimalIntegerValueModelElement('segment_memcap_drop'),
FixedDataModelElement('stream_depth_reached_str',
b',"stream_depth_reached":'),
DecimalIntegerValueModelElement('stream_depth_reached'),
FixedDataModelElement('reassembly_gap_str',
b',"reassembly_gap":'),
DecimalIntegerValueModelElement('reassembly_gap'),
FixedDataModelElement('memuse_str', b',"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('reassembly_memuse_str',
b',"reassembly_memuse":'),
DecimalIntegerValueModelElement('reassembly_memuse'),
]),
SequenceModelElement('detect', [
FixedDataModelElement('alert_str',
b'},"detect":{"alert":'),
DecimalIntegerValueModelElement('alert')
]),
SequenceModelElement('app_layer', [
SequenceModelElement('flow', [
FixedDataModelElement(
'http_str', b'},"app_layer":{"flow":{"http":'),
DecimalIntegerValueModelElement('http'),
FixedDataModelElement('ftp_str', b',"ftp":'),
DecimalIntegerValueModelElement('ftp'),
FixedDataModelElement('smtp_str', b',"smtp":'),
DecimalIntegerValueModelElement('smtp'),
FixedDataModelElement('tls_str', b',"tls":'),
DecimalIntegerValueModelElement('tls'),
FixedDataModelElement('ssh_str', b',"ssh":'),
DecimalIntegerValueModelElement('ssh'),
FixedDataModelElement('imap_str', b',"imap":'),
DecimalIntegerValueModelElement('imap'),
FixedDataModelElement('msn_str', b',"msn":'),
DecimalIntegerValueModelElement('msn'),
FixedDataModelElement('smb_str', b',"smb":'),
DecimalIntegerValueModelElement('smb'),
FixedDataModelElement('dcerpc_tcp_str',
b',"dcerpc_tcp":'),
DecimalIntegerValueModelElement('dcerpc_tcp'),
FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'),
DecimalIntegerValueModelElement('dns_tcp'),
FixedDataModelElement('failed_tcp_str',
b',"failed_tcp":'),
DecimalIntegerValueModelElement('failed_tcp'),
FixedDataModelElement('dcerpc_udp_str',
b',"dcerpc_udp":'),
DecimalIntegerValueModelElement('dcerpc_udp'),
FixedDataModelElement('dns_udp_str', b',"dns_udp":'),
DecimalIntegerValueModelElement('dns_udp'),
FixedDataModelElement('failed_udp_str',
b',"failed_udp":'),
DecimalIntegerValueModelElement('failed_udp'),
]),
SequenceModelElement('tx', [
FixedDataModelElement('http_str', b'},"tx":{"http":'),
DecimalIntegerValueModelElement('http'),
FixedDataModelElement('smtp_str', b',"smtp":'),
DecimalIntegerValueModelElement('smtp'),
FixedDataModelElement('tls_str', b',"tls":'),
DecimalIntegerValueModelElement('tls'),
FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'),
DecimalIntegerValueModelElement('dns_tcp'),
FixedDataModelElement('dns_udp_str', b',"dns_udp":'),
DecimalIntegerValueModelElement('dns_udp'),
])
]),
SequenceModelElement('flow_mgr', [
FixedDataModelElement('closed_pruned_str',
b'}},"flow_mgr":{"closed_pruned":'),
DecimalIntegerValueModelElement('closed_pruned'),
FixedDataModelElement('new_pruned_str', b',"new_pruned":'),
DecimalIntegerValueModelElement('new_pruned'),
FixedDataModelElement('est_pruned_str', b',"est_pruned":'),
DecimalIntegerValueModelElement('est_pruned'),
FixedDataModelElement('bypassed_pruned_str',
b',"bypassed_pruned":'),
DecimalIntegerValueModelElement('bypassed_pruned'),
FixedDataModelElement('flows_checked_str',
b',"flows_checked":'),
DecimalIntegerValueModelElement('flows_checked'),
FixedDataModelElement('flows_notimeout_str',
b',"flows_notimeout":'),
DecimalIntegerValueModelElement('flows_notimeout'),
FixedDataModelElement('flows_timeout_str',
b',"flows_timeout":'),
DecimalIntegerValueModelElement('flows_timeout'),
FixedDataModelElement('flows_timeout_inuse_str',
b',"flows_timeout_inuse":'),
DecimalIntegerValueModelElement('flows_timeout_inuse'),
FixedDataModelElement('flows_removed_str',
b',"flows_removed":'),
DecimalIntegerValueModelElement('flows_removed'),
FixedDataModelElement('rows_checked_str',
b',"rows_checked":'),
DecimalIntegerValueModelElement('rows_checked'),
FixedDataModelElement('rows_skipped_str',
b',"rows_skipped":'),
DecimalIntegerValueModelElement('rows_skipped'),
FixedDataModelElement('rows_empty_str', b',"rows_empty":'),
DecimalIntegerValueModelElement('rows_empty'),
FixedDataModelElement('rows_busy_str', b',"rows_busy":'),
DecimalIntegerValueModelElement('rows_busy'),
FixedDataModelElement('rows_maxlen_str',
b',"rows_maxlen":'),
DecimalIntegerValueModelElement('rows_maxlen'),
]),
SequenceModelElement('dns', [
FixedDataModelElement('memuse_str', b'},"dns":{"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('memcap_state_str',
b',"memcap_state":'),
DecimalIntegerValueModelElement('memcap_state'),
FixedDataModelElement('memcap_global_str',
b',"memcap_global":'),
DecimalIntegerValueModelElement('memcap_global'),
]),
SequenceModelElement('http', [
FixedDataModelElement('memuse_str',
b'},"http":{"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('memcap_str', b',"memcap":'),
DecimalIntegerValueModelElement('memcap'),
]),
FixedDataModelElement('quote_str', b'}}}')
]),
SequenceModelElement('tls', [
FixedDataModelElement('tls_str', b'tls",'), conn,
SequenceModelElement('tls', [
FixedDataModelElement('subject_str',
b',"tls":{"subject":"'),
DelimitedDataModelElement('subject', b'"'),
FixedDataModelElement('issuerdn_str', b'","issuerdn":"'),
DelimitedDataModelElement('issuerdn', b'"'),
FixedDataModelElement('fingerprint_str',
b'","fingerprint":"'),
DelimitedDataModelElement('fingerprint', b'"'),
OptionalMatchModelElement(
'sni',
SequenceModelElement('sni', [
FixedDataModelElement('sni_str', b'","sni":"'),
DelimitedDataModelElement('sni', b'"'),
])),
FixedDataModelElement('version_str', b'","version":"'),
DelimitedDataModelElement('version', b'"'),
FixedDataModelElement('notbefore_str', b'","notbefore":"'),
DelimitedDataModelElement('notbefore', b'"'),
FixedDataModelElement('notafter_str', b'","notafter":"'),
DelimitedDataModelElement('notafter', b'"'),
]),
FixedDataModelElement('brack_str', b'"}}')
]),
SequenceModelElement('alert', [
FixedDataModelElement('alert_str', b'alert",'), conn,
OptionalMatchModelElement(
'tx_id',
SequenceModelElement('tx_id', [
FixedDataModelElement('tx_id', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
])),
SequenceModelElement('alert', [
FixedDataModelElement('action_str',
b',"alert":{"action":"'),
DelimitedDataModelElement('action', b'"'),
FixedDataModelElement('gid_str', b'","gid":'),
DecimalIntegerValueModelElement('gid'),
FixedDataModelElement('signature_id_str',
b',"signature_id":'),
DecimalIntegerValueModelElement('signature_id'),
FixedDataModelElement('rev_str', b',"rev":'),
DecimalIntegerValueModelElement('rev'),
FixedDataModelElement('signature_str', b',"signature":"'),
DelimitedDataModelElement('signature', b'"'),
FixedDataModelElement('category_str', b'","category":"'),
DelimitedDataModelElement('category', b'"'),
FixedDataModelElement('severity_str', b'","severity":'),
DecimalIntegerValueModelElement('severity'),
FixedDataModelElement('brack_str', b'}')
]), http,
FixedDataModelElement('brack_str', b'}')
]),
])
])
return model
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement('Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%'),
DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')]
service_children_login_details = [
FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']),
OptionalMatchModelElement('PastTime', SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')]))]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')]
service_children_random_time = [FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random')]
service_children_sensors = [SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')]), FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)]
service_children_user_ip_address = [
FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')]
service_children_audit = [SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO),
FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')])]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'),
SequenceModelElement('se', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';')])]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement("wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement("decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement('ElementValueBranchModelElement', FirstMatchModelElement("first", [
SequenceModelElement("seq1", [fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])]), "wordlist",
{0: decimal_integer_value_model_element, 1: fixed_data_me2}))
service_children_parsing_model_element.append(HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(SequenceModelElement('se2', [
FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement')]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1))
service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(SequenceModelElement('se', [
WhiteSpaceLimitedDataModelElement('WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ')]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement('/', FirstMatchModelElement('FirstMatchModelElement//optional', [
FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [
FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])])))
alphabet = b'ghijkl'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement),
SequenceModelElement('CronExecution', service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element)])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust([atom_filter])
analysis_context.register_component(simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
from aminer.events.JsonConverterHandler import JsonConverterHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
json_converter_handler = JsonConverterHandler([stream_printer_event_handler], analysis_context)
anomaly_event_handlers = [json_converter_handler]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True)
analysis_context.register_component(verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.AndMatchRule([
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails'),
Rules.DebugMatchRule(debug_match_result=True)]),
Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10)
analysis_context.register_component(parser_count, component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers)
analysis_context.register_component(etd, component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False,
output_log_line=False, ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5,
ignore_list=["/model/RandomTime"])
analysis_context.register_component(vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True,
hypothesis_max_delta_time=1.0)
analysis_context.register_component(ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector
efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1)
analysis_context.register_component(efd, component_name="EventFrequencyDetector")
atom_filter.add_handler(efd)
from aminer.analysis.EventSequenceDetector import EventSequenceDetector
esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random',
'/model/RandomTime', '/model/DailyCron'])
analysis_context.register_component(esd, component_name="EventSequenceDetector")
atom_filter.add_handler(esd)
from aminer.analysis.MatchFilter import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[
1, 10, 100], output_log_line=True)
analysis_context.register_component(match_filter, component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(analysis_context.aminer_config, [
'/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function, output_log_line=True)
analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
import re
ip_match_action = Rules.EventGenerationMatchAction(
"Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers)
vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3])
mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None)
time_allowlist_rules = [
Rules.AndMatchRule([
Rules.ParallelMatchRule([
Rules.ValueDependentDelegatedMatchRule([
'/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], {
(b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt),
Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action),
Rules.DebugHistoryMatchRule(debug_match_result=True)
]),
# IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed
Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])),
Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)),
Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$")))
])
]
time_allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist")
atom_filter.add_handler(time_allowlist_violation_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(analysis_context.aminer_config, [
('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(analysis_context.aminer_config, anomaly_event_handlers, None, [
'/model/Random'], 100, 10, output_log_line=True)
analysis_context.register_component(match_value_average_change_detector, component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(
sys.stdout, ['/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM'], b';', b'')
analysis_context.register_component(match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers, output_log_line=True)
analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(analysis_context.aminer_config, [
'/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=[
'/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(analysis_context.aminer_config, [
'/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True)
analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2,
realert_interval=5, output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5,
record_count_before_event=10000, output_log_line=True)
analysis_context.register_component(time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule('CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[
('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement])
rules = [Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)]
time_correlation_violation_detector = TimeCorrelationViolationDetector(analysis_context.aminer_config, rules, anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def get_model():
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('queue', [
FixedWordlistDataModelElement('type', [b'Start', b'End']),
FixedDataModelElement('s0', b' queue run: pid='),
DecimalIntegerValueModelElement('pid')
]),
SequenceModelElement('rec-log', [
WhiteSpaceLimitedDataModelElement('id'),
FixedDataModelElement('s0', b' <= '),
WhiteSpaceLimitedDataModelElement('env-from'),
FirstMatchModelElement('source', [
SequenceModelElement('network', [
FixedDataModelElement('s0', b' H=('),
DelimitedDataModelElement('hostname', b') '),
FixedDataModelElement('s1', b') ['),
IpAddressDataModelElement('hostip'),
FixedDataModelElement('s2', b']')
]),
SequenceModelElement('user', [
FixedDataModelElement('s0', b' U='),
WhiteSpaceLimitedDataModelElement('user')
])
]),
FixedDataModelElement('s2', b' P='),
WhiteSpaceLimitedDataModelElement('proto'),
FixedDataModelElement('s3', b' S='),
DecimalIntegerValueModelElement('size'),
OptionalMatchModelElement(
'idopt',
SequenceModelElement('iddata', [
FixedDataModelElement('s0', b' id='),
AnyByteDataModelElement('id')
]))
]),
SequenceModelElement(
'send-log',
[
WhiteSpaceLimitedDataModelElement('id'),
# Strange: first address seems to use different separator than second one.
FixedWordlistDataModelElement('s0', [b' => b', b' ->']),
DelimitedDataModelElement('env-to', b' R='),
FixedDataModelElement('s1', b' R='),
WhiteSpaceLimitedDataModelElement('route'),
FixedDataModelElement('s2', b' T='),
WhiteSpaceLimitedDataModelElement('transport'),
AnyByteDataModelElement('unparsed')
]),
SequenceModelElement('sent', [
WhiteSpaceLimitedDataModelElement('id'),
FixedDataModelElement('s0', b' Completed')
]),
SequenceModelElement('started', [
FixedDataModelElement('s0', b' exim '),
WhiteSpaceLimitedDataModelElement('version'),
FixedDataModelElement('s1', b' daemon started: pid='),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement(
's2', b', -q30m, listening for SMTP on [127.0.0.1]:25')
])
]
model = SequenceModelElement('exim', [
FixedDataModelElement('sname', b'exim['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPUWorkload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S')
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filters = AtomFilters.SubhandlerFilter(None)
analysis_context.register_component(atom_filters,
component_name="AtomFilter")
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filters])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(
analysis_context) # skipcq: BAN-B108
from aminer.events.Utils import VolatileLogarithmicBackoffEventHistory
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(
100)
anomaly_event_handlers = [
stream_printer_event_handler,
volatile_logarithmic_backoff_event_history
]
analysis_context.register_component(
volatile_logarithmic_backoff_event_history,
component_name="VolatileLogarithmicBackoffEventHistory")
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filters.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filters.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filters.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None,
anomaly_event_handlers, 10)
analysis_context.register_component(parser_count,
component_name="ParserCount")
atom_filters.add_handler(parser_count)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
check_rules_flag=True,
hypothesis_max_delta_time=1.0,
auto_include_flag=True)
analysis_context.register_component(
ecd, component_name="EventCorrelationDetector")
atom_filters.add_handler(ecd)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filters.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=False,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filters.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filters.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filters.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filters.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filters.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=False)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filters.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
analysis_context.aminer_config,
['/model/type/path/name', '/model/type/syscall/syscall'],
anomaly_event_handlers,
id_path_list=['/model/type/path/id', '/model/type/syscall/id'],
min_allowed_time_diff=5,
auto_include_flag=True,
allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(
new_match_id_value_combo_detector,
component_name="NewMatchIdValueComboDetector")
atom_filters.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/Job Number', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=False)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filters.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'],
anomaly_event_handlers,
auto_include_flag=False,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filters.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filters.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filters.add_handler(time_correlation_violation_detector)
from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers.append(mail_notification_handler)
class EventTypeDetectorTest(TestBase):
"""Unittests for the EventTypeDetector."""
log_lines = [
b'type=SYSCALL msg=audit(1580367384.000:1): arch=c000003e syscall=1 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367385.000:1): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367386.000:2): arch=c000003e syscall=2 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367387.000:2): item=0 name="two" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367388.000:3): arch=c000003e syscall=3 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367389.000:3): item=0 name="three" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367388.500:100): arch=c000003e syscall=1 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1'
b' ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=SYSCALL msg=audit(1580367390.000:4): arch=c000003e syscall=1 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367391.000:4): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=PATH msg=audit(1580367392.000:5): item=0 name="two" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367393.000:5): arch=c000003e syscall=2 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=SYSCALL msg=audit(1580367394.000:6): arch=c000003e syscall=4 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367395.000:7): item=0 name="five" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367396.000:8): arch=c000003e syscall=6 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367397.000:6): item=0 name="four" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367398.000:7): arch=c000003e syscall=5 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367399.000:8): item=0 name="six" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367400.000:9): arch=c000003e syscall=2 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)',
b'type=PATH msg=audit(1580367401.000:9): item=0 name="three" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=PATH msg=audit(1580367402.000:10): item=0 name="one" inode=790106 dev=fe:01 mode=0100666 ouid=1000 ogid=1000 rdev=00:00 '
b'nametype=NORMAL',
b'type=SYSCALL msg=audit(1580367403.000:10): arch=c000003e syscall=3 success=yes exit=21 a0=7ffda5863060 a1=0 a2=1b6 a3=4f items=1 '
b'ppid=22913 pid=13187 auid=4294967295 uid=33 gid=33 euid=33 suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 '
b'comm="apache2" exe="/usr/sbin/apache2" key=(null)']
parsing_model = FirstMatchModelElement('type', [SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='), DecimalIntegerValueModelElement('mode'),
FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')])])
def test1receive_atoms_with_default_values(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is empty and all paths are learned dynamically in variable_key_list.
"""
event_type_detector = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(line, ParserMatch(self.parsing_model.get_match_element('parser', MatchContext(line))), t, self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.total_records, i + 1)
def test2receive_atoms_with_defined_path_list(self):
"""
In this test case multiple log_atoms are received with default values of the EventTypeDetector.
path_list is set to a static list of paths and variable_key_list should not be used.
"""
event_type_detector = EventTypeDetector(
self.aminer_config, [self.stream_printer_event_handler], path_list=['parser/type/path/nametype'])
results = [True, False, True, False, True, False, True, True, False, False, True, True, False, True, False, True, False, True,
False, False, True]
log_atoms = []
for line in self.log_lines:
t = time.time()
log_atoms.append(
LogAtom(line, ParserMatch(self.parsing_model.get_match_element('parser', MatchContext(line))), t, self.__class__.__name__))
for i, log_atom in enumerate(log_atoms):
old_vals = (event_type_detector.num_events, event_type_detector.num_eventlines,
event_type_detector.total_records, event_type_detector.longest_path)
self.assertEqual(event_type_detector.receive_atom(log_atom), not results[i], i)
if results[i]:
self.assertEqual(old_vals, (
event_type_detector.num_events, event_type_detector.num_eventlines,
event_type_detector.total_records, event_type_detector.longest_path))
def test3append_values_float(self):
"""This unittest checks the append_values method with raw_match_object being a float value."""
event_type_detector = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
# initialize all values.
t = time.time()
log_atom = LogAtom(b'22.2', ParserMatch(MatchElement('path', '22.2', 22.2, None)), t, self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22.2]]])
log_atom = LogAtom(b'22', ParserMatch(MatchElement('path', '22', 22, None)), t, self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22]]])
log_atom = LogAtom(b'22.2', ParserMatch(MatchElement('path', '22', b'22', None)), t, self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[[22]]])
def test4append_values_bytestring(self):
"""
This unittest checks the append_values method with raw_match_object being a bytestring.
This should trigger a ValueError and append the match_string.
"""
event_type_detector = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
# initialize all values.
t = time.time()
log_atom = LogAtom(b'This is a string', ParserMatch(
MatchElement('path', 'This is a string', b'This is a string', None)), t, self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[['This is a string']]])
log_atom = LogAtom(b'24.05.', ParserMatch(MatchElement('path', '24.05.', b'24.05.', None)), t, self.__class__.__name__)
event_type_detector.values = [[[]]]
event_type_detector.append_values(log_atom, 0)
self.assertEqual(event_type_detector.values, [[['24.05.']]])
def test5check_value_reduction(self):
"""This unittest checks the functionality of reducing the values when the maxNumVals threshold is reached."""
event_type_detector = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
t = time.time()
val_list = [[[]]]
for i in range(1, event_type_detector.max_num_vals + 1, 1):
log_atom = LogAtom(str(i).encode(), ParserMatch(MatchElement('path', str(i), i, None)), t, self.__class__.__name__)
val_list[0][0].append(float(i))
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.values, val_list)
i += 1
log_atom = LogAtom(str(i).encode(), ParserMatch(MatchElement('path', str(i), i, None)), t, self.__class__.__name__)
val_list[0][0].append(float(i))
self.assertTrue(event_type_detector.receive_atom(log_atom))
self.assertEqual(event_type_detector.values, [[val_list[0][0][-event_type_detector.min_num_vals:]]])
def test6persist_and_load_data(self):
"""This unittest checks the functionality of the persistence by persisting and reloading values."""
event_type_detector = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
t = time.time()
log_atom = LogAtom(b'22.2', ParserMatch(MatchElement('path', '22.2', 22.2, None)), t, self.__class__.__name__)
event_type_detector.receive_atom(log_atom)
event_type_detector.do_persist()
event_type_detector_loaded = EventTypeDetector(self.aminer_config, [self.stream_printer_event_handler])
self.assertEqual(event_type_detector.variable_key_list, event_type_detector_loaded.variable_key_list)
self.assertEqual(event_type_detector.values, event_type_detector_loaded.values)
self.assertEqual(event_type_detector.longest_path, event_type_detector_loaded.longest_path)
self.assertEqual(event_type_detector.check_variables, event_type_detector_loaded.check_variables)
self.assertEqual(event_type_detector.num_eventlines, event_type_detector_loaded.num_eventlines)