def get_model():
"""Return a model to parse messages from kernel logging."""
type_children = [
SequenceModelElement("ipv4-martian", [
FixedDataModelElement("s0", b"IPv4: martian "),
FixedWordlistDataModelElement("direction", [b"source", b"destination"]),
FixedDataModelElement("s1", b" "),
IpAddressDataModelElement("destination"),
FixedDataModelElement("s2", b" from "),
IpAddressDataModelElement("source"),
FixedDataModelElement("s3", b", on dev "),
AnyByteDataModelElement("interface")]),
SequenceModelElement("net-llheader", [
FixedDataModelElement("s0", b"ll header: "),
AnyByteDataModelElement("data")
]),
AnyByteDataModelElement("unparsed")
]
model = SequenceModelElement("kernel", [
FixedDataModelElement("sname", b"kernel: "),
OptionalMatchModelElement("opt", SequenceModelElement("seq", [
FixedDataModelElement("opt_s0", b"]"),
DelimitedDataModelElement("timestamp", b"]"),
FixedDataModelElement("opt_s1", b"] "),
])),
FirstMatchModelElement("msg", type_children)
])
return model
def get_logind_model(user_name_model=None):
"""Return a model to parse a systemd logind daemon message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz-')
type_children = [
SequenceModelElement('new session', [
FixedDataModelElement('s0', b'New session '),
DecimalIntegerValueModelElement('session'),
FixedDataModelElement('s1', b' of user '), user_name_model,
FixedDataModelElement('s2', b'.')
]),
SequenceModelElement('removed session', [
FixedDataModelElement('s0', b'Removed session '),
DecimalIntegerValueModelElement('session'),
FixedDataModelElement('s1', b'.')
])
]
# Will fail on username models including the dot at the end.
model = SequenceModelElement('systemd-logind', [
FixedDataModelElement('sname', b'systemd-logind['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_logind_model(user_name_model=None):
"""Return a model to parse a systemd logind daemon message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement("user", b"0123456789abcdefghijklmnopqrstuvwxyz-_")
type_children = [
SequenceModelElement("new session", [
FixedDataModelElement("s0", b"New session "),
DecimalIntegerValueModelElement("session"),
FixedDataModelElement("s1", b" of user "),
user_name_model,
FixedDataModelElement("s2", b".")
]),
SequenceModelElement("removed session", [
FixedDataModelElement("s0", b"Removed session "),
DecimalIntegerValueModelElement("session"),
FixedDataModelElement("s1", b".")
]),
SequenceModelElement("logged out", [
FixedDataModelElement("s0", b"Session "),
DecimalIntegerValueModelElement("session"),
FixedDataModelElement("s1", b" logged out. Waiting for processes to exit.")
]),
FixedDataModelElement("failed abandon", b"Failed to abandon session scope: Transport endpoint is not connected")
]
# Will fail on username models including the dot at the end.
model = SequenceModelElement("systemd-logind", [
FixedDataModelElement("sname", b"systemd-logind["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('gidchange', [
FixedDataModelElement('s0', b'rsyslogd\'s groupid changed to '),
DecimalIntegerValueModelElement('gid')
]),
SequenceModelElement('statechange', [
FixedDataModelElement('s0',
b'[origin software="rsyslogd" swVersion="'),
DelimitedDataModelElement('version', b'"'),
FixedDataModelElement('s1', b'" x-pid="'),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s2',
b'" x-info="http://www.rsyslog.com"] '),
FirstMatchModelElement('type', [
FixedDataModelElement('HUPed', b'rsyslogd was HUPed'),
FixedDataModelElement('start', b'start')
])
]),
SequenceModelElement('uidchange', [
FixedDataModelElement('s0', b'rsyslogd\'s userid changed to '),
DecimalIntegerValueModelElement('uid')
])
]
model = SequenceModelElement('rsyslog', [
FixedDataModelElement('sname', b'rsyslogd: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return the model."""
type_children = [
SequenceModelElement("sent", [
FixedDataModelElement("s0", b"Sent mail for "),
DelimitedDataModelElement("to-addr", b" ("),
FixedDataModelElement("s1", b" ("),
DelimitedDataModelElement("status", b") uid="),
FixedDataModelElement("s2", b") uid="),
DecimalIntegerValueModelElement("uid"),
FixedDataModelElement("s3", b" username="******"username", b" outbytes="),
FixedDataModelElement("s4", b" outbytes="),
DecimalIntegerValueModelElement("bytes")
]),
SequenceModelElement("sent", [
DelimitedDataModelElement("program", b" "),
FixedDataModelElement("s0", b" sent mail for "),
AnyByteDataModelElement("user")
])
]
model = SequenceModelElement("ssmtp", [
FixedDataModelElement("sname", b"sSMTP["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return a model to parse messages from kernel logging."""
type_children = [
SequenceModelElement('ipv4-martian', [
FixedDataModelElement('s0', b'IPv4: martian '),
FixedWordlistDataModelElement('direction',
[b'source', b'destination']),
FixedDataModelElement('s1', b' '),
IpAddressDataModelElement('destination'),
FixedDataModelElement('s2', b' from '),
IpAddressDataModelElement('source'),
FixedDataModelElement('s3', b', on dev '),
AnyByteDataModelElement('interface')
]),
SequenceModelElement('net-llheader', [
FixedDataModelElement('s0', b'll header: '),
AnyByteDataModelElement('data')
]),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('kernel', [
FixedDataModelElement('sname', b'kernel: ['),
DelimitedDataModelElement('timestamp', b']'),
FixedDataModelElement('s0', b'] '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_systemd_model():
"""Return the parsing model for messages directly from systemd."""
type_children = [
FixedDataModelElement('apt-daily-start',
b'Starting Daily apt activities...'),
FixedDataModelElement('apt-daily-started',
b'Started Daily apt activities.'),
SequenceModelElement('apt-daily-timer', [
FixedDataModelElement('s0', b'apt-daily.timer: Adding '),
OptionalMatchModelElement(
'hopt',
SequenceModelElement('hblock', [
DecimalIntegerValueModelElement('hours'),
FixedDataModelElement('s1', b'h ')
])),
DecimalIntegerValueModelElement('minutes'),
FixedDataModelElement('s2', b'min '),
DecimalFloatValueModelElement('seconds'),
FixedDataModelElement('s3', b's random time.')
]),
FixedDataModelElement('tmp-file-cleanup',
b'Starting Cleanup of Temporary Directories...'),
FixedDataModelElement('tmp-file-cleanup-started',
b'Started Cleanup of Temporary Directories.')
]
model = SequenceModelElement('systemd', [
FixedDataModelElement('sname', b'systemd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return a parser for apache2 access.log."""
new_time_model = SequenceModelElement('time_model', [
DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S '),
FixedWordlistDataModelElement('sign', [b'+', b'-']),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('bracket', b']')
])
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identity_model = VariableByteDataModelElement(
'ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
request_method_model = FixedWordlistDataModelElement(
'method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT',
b'OPTIONS', b'TRACE', b'PATCH'
])
request_model = VariableByteDataModelElement(
'request',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+'
)
version_model = VariableByteDataModelElement('version', b'0123456789.')
status_code_model = DecimalIntegerValueModelElement('status')
size_model = DecimalIntegerValueModelElement('size')
user_agent_model = VariableByteDataModelElement(
'useragent',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+;:_ '
)
whitespace_str = b' '
model = SequenceModelElement('accesslog', [
host_name_model,
FixedDataModelElement('sp0', whitespace_str),
identity_model,
FixedDataModelElement('sp1', whitespace_str),
user_name_model,
FixedDataModelElement('sp2', whitespace_str),
new_time_model,
FixedDataModelElement('sp3', b' "'),
request_method_model,
FixedDataModelElement('sp4', whitespace_str),
request_model,
FixedDataModelElement('sp5', b' HTTP/'),
version_model,
FixedDataModelElement('sp6', b'" '),
status_code_model,
FixedDataModelElement('sp7', whitespace_str),
size_model,
FixedDataModelElement('sp8', b' "-" "'),
user_agent_model,
FixedDataModelElement('sp9', b'"'),
])
return model
def get_model():
"""Return a model for su session information messages after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('build-stack', [
FixedDataModelElement('s0', b'building new pluginstance stack: \''),
DelimitedDataModelElement('stack', b'\''),
FixedDataModelElement('s1', b'\'')
]),
SequenceModelElement('nfct-event', [
FixedDataModelElement('s0', b'[DESTROY] ORIG: SRC='),
IpAddressDataModelElement('osrcip'),
FixedDataModelElement('s1', b' DST='),
IpAddressDataModelElement('odstip'),
FixedDataModelElement('s2', b' PROTO='),
FixedWordlistDataModelElement('proto', [b'TCP', b'UDP']),
FixedDataModelElement('s3', b' SPT='),
DecimalIntegerValueModelElement('ospt'),
FixedDataModelElement('s4', b' DPT='),
DecimalIntegerValueModelElement('odpt'),
FixedDataModelElement('s5', b' PKTS='),
DecimalIntegerValueModelElement('opkts'),
FixedDataModelElement('s6', b' BYTES='),
DecimalIntegerValueModelElement('obytes'),
FixedDataModelElement('s7', b' , REPLY: SRC='),
IpAddressDataModelElement('rsrcip'),
FixedDataModelElement('s8', b' DST='),
IpAddressDataModelElement('rdstip'),
FixedDataModelElement('s9', b' PROTO='),
FixedWordlistDataModelElement('rproto', [b'TCP', b'UDP']),
FixedDataModelElement('s10', b' SPT='),
DecimalIntegerValueModelElement('rspt'),
FixedDataModelElement('s11', b' DPT='),
DecimalIntegerValueModelElement('rdpt'),
FixedDataModelElement('s12', b' PKTS='),
DecimalIntegerValueModelElement('rpkts'),
FixedDataModelElement('s13', b' BYTES='),
DecimalIntegerValueModelElement('rbytes'),
# No additional whitespace from Ubuntu Trusty 14.04 on.
OptionalMatchModelElement('tail', FixedDataModelElement('s0', b' '))
]),
FixedDataModelElement('nfct-plugin', b'NFCT plugin working in event mode'),
FixedDataModelElement('reopen', b'reopening capture file'),
FixedDataModelElement('signal', b'signal received, calling pluginstances'),
FixedDataModelElement('uidchange', b'Changing UID / GID')
]
# Netflow entry
model = SequenceModelElement('ulogd', [
FixedDataModelElement('sname', b'ulogd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def test4atom_no_match_missing_value_string_set(self):
"""
This test case sets up a set of values, which are all expected to be matched.
The missing value string is set to a value, so when a string does not match this value is used instead.
"""
description = "Test4MatchValueStreamWriter"
output_stream = BytesIO()
match_context = MatchContext(
b'25537Euro 25538Euro 25539Euro 25540Pfund ')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
fixed_dme = FixedDataModelElement('s1', self.euro)
sequence_model_element = SequenceModelElement(
'sequence', [decimal_integer_value_me, fixed_dme])
match_value_stream_writer = MatchValueStreamWriter(
output_stream, [self.match_sequence_d1, self.match_sequence_s1],
b';', b'-')
self.analysis_context.register_component(match_value_stream_writer,
description)
match_element = sequence_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
match_element = sequence_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
match_element = sequence_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
match_element = decimal_integer_value_me.get_match_element(
'match', match_context)
match_element.path = self.match_sequence_d1
log_atom = LogAtom(match_context.match_data,
ParserMatch(match_element), 1,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
self.assertEqual(output_stream.getvalue().decode(),
'25537;Euro \n25538;Euro \n25539;Euro \n25540;-\n')
def test11multiple_paths(self):
"""Test the functionality of the MissingMatchPathValueDetector with multiple paths."""
description = "Test11MissingMatchPathValueDetector"
match_context = MatchContext(self.pid + b"22")
fixed_dme = FixedDataModelElement('s1', self.pid)
decimal_integer_value_me = DecimalIntegerValueModelElement('d1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
seq = SequenceModelElement('model', [fixed_dme, decimal_integer_value_me])
match_element = seq.get_match_element("match", match_context)
missing_match_path_value_detector = MissingMatchPathValueDetector(self.aminer_config, [
"match/model", "match/model/s1", "match/model/d1"], [self.stream_printer_event_handler], 'Default', False,
self.__default_interval, self.__realert_interval)
self.analysis_context.register_component(missing_match_path_value_detector, description)
log_atom = LogAtom(fixed_dme.fixed_data + b"22", ParserMatch(match_element), 1, missing_match_path_value_detector)
self.assertTrue(missing_match_path_value_detector.receive_atom(log_atom))
def get_model():
"""Return a model to parse Suricata Fast logs from the AIT-LDS."""
model = SequenceModelElement('model', [
DateTimeModelElement('time', b'%m/%d/%Y-%H:%M:%S.%f'),
FixedDataModelElement('brack_str1', b' [**] ['),
DecimalIntegerValueModelElement('id1'),
FixedDataModelElement('sep1', b':'),
DecimalIntegerValueModelElement('id2'),
FixedDataModelElement('sep2', b':'),
DecimalIntegerValueModelElement('id3'),
FixedDataModelElement('sep3', b'] '),
DelimitedDataModelElement('message', b' [**] '),
FixedDataModelElement('classification_str',
b' [**] [Classification: '),
DelimitedDataModelElement('classification', b']'),
FixedDataModelElement('priority_str', b'] [Priority: '),
DecimalIntegerValueModelElement('priority'),
FixedDataModelElement('brack_str1', b'] {'),
DelimitedDataModelElement('conn', b'}'),
FixedDataModelElement('brack_str2', b'} '),
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('arrow_str', b' -> '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('dst_port'),
])
return model
def get_model():
"""Return the model."""
type_children = [
FixedDataModelElement(
'warn-no-openat',
b'WARNING: SECURITY: No secure open yet due to missing openat in python!'
),
FixedDataModelElement(
'warn-no-OPATH',
b'WARNING: SECURITY: Open should use O_PATH, but not yet available in python'
),
FixedDataModelElement(
'warn-POSIX-acls',
b'WARNING: SECURITY: No checking for backdoor access via \
POSIX ACLs, use "getfacl" from "acl" package to check manually.'),
FixedDataModelElement(
'warn-no-linkat',
b'WARNING: SECURITY: unsafe unlink (unavailable unlinkat/linkat \
should be used, but not available in python)'),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('aminer', [
FixedDataModelElement('sname', b'aminer['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return a model to parse Suricata Fast logs from the AIT-LDS."""
model = SequenceModelElement("model", [
DateTimeModelElement("time", b"%m/%d/%Y-%H:%M:%S.%f"),
FixedDataModelElement("brack_str1", b" [**] ["),
DecimalIntegerValueModelElement("id1"),
FixedDataModelElement("sep1", b":"),
DecimalIntegerValueModelElement("id2"),
FixedDataModelElement("sep2", b":"),
DecimalIntegerValueModelElement("id3"),
FixedDataModelElement("sep3", b"] "),
DelimitedDataModelElement("message", b" [**] "),
FixedDataModelElement("classification_str",
b" [**] [Classification: "),
DelimitedDataModelElement("classification", b"]"),
FixedDataModelElement("priority_str", b"] [Priority: "),
DecimalIntegerValueModelElement("priority"),
FixedDataModelElement("brack_str1", b"] {"),
DelimitedDataModelElement("conn", b"}"),
FixedDataModelElement("brack_str2", b"} "),
IpAddressDataModelElement("src_ip"),
FixedDataModelElement("colon", b":"),
DecimalIntegerValueModelElement("src_port"),
FixedDataModelElement("arrow_str", b" -> "),
IpAddressDataModelElement("dst_ip"),
FixedDataModelElement("colon", b":"),
DecimalIntegerValueModelElement("dst_port"),
])
return model
class SequenceModelElementTest(unittest.TestCase):
sequence_start = b'The sequence starts with a number: '
fixed_data_model_element = FixedDataModelElement('fixed', sequence_start)
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
'decimal', DecimalIntegerValueModelElement.SIGN_TYPE_NONE, DecimalIntegerValueModelElement.PAD_TYPE_NONE)
fixed_wordlist_data_model_element = FixedWordlistDataModelElement('wordlist', [b' Euro', b' Dollar', b' Pfund'])
sequence_model_element = SequenceModelElement(
'sequence', [fixed_data_model_element, decimal_integer_value_model_element, fixed_wordlist_data_model_element])
def test1sequence_of_matching_elements(self):
"""A normal sequence of matching elements is tested in this example test case"""
match_context = MatchContext(b'The sequence starts with a number: 25538 Euro')
self.assertEqual(self.sequence_model_element.get_match_element(
'match', match_context).get_match_string(), b'The sequence starts with a number: 25538 Euro')
self.assertEqual(match_context.match_data, b'')
def test2sequence_not_matching(self):
"""A normal sequence of elements, which do not match with the expected sequence_model is tested."""
match_context = MatchContext(b'The sequence starts with a number: 25538 US-Dollar')
self.assertEqual(self.sequence_model_element.get_match_element('match', match_context), None)
self.assertEqual(match_context.match_data, b'The sequence starts with a number: 25538 US-Dollar')
def test3match_context_shorter_than_sequence(self):
"""This test case unit if the sequence_model returns None, when the match_context is too short for a match."""
match_context = MatchContext(self.sequence_start)
self.assertEqual(self.sequence_model_element.get_match_element('match', match_context), None)
self.assertEqual(match_context.match_data, self.sequence_start)
def get_model():
"""Return a parser for apache2 access.log."""
new_time_model = DateTimeModelElement("time", b"[%d/%b/%Y:%H:%M:%S%z")
host_name_model = VariableByteDataModelElement(
"host", b"-.01234567890abcdefghijklmnopqrstuvwxyz:")
identity_model = VariableByteDataModelElement(
"ident", b"-.01234567890abcdefghijklmnopqrstuvwxyz:")
user_name_model = VariableByteDataModelElement(
"user", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
request_method_model = FirstMatchModelElement("fm", [
FixedDataModelElement("dash", b"-"),
SequenceModelElement("request", [
FixedWordlistDataModelElement("method", [
b"GET", b"POST", b"PUT", b"HEAD", b"DELETE", b"CONNECT",
b"OPTIONS", b"TRACE", b"PATCH"
]),
FixedDataModelElement("sp5", b" "),
DelimitedDataModelElement("request", b" ", b"\\"),
FixedDataModelElement("sp6", b" "),
DelimitedDataModelElement("version", b'"'),
])
])
status_code_model = DecimalIntegerValueModelElement("status")
size_model = DecimalIntegerValueModelElement("size")
whitespace_str = b" "
model = SequenceModelElement("accesslog", [
host_name_model,
FixedDataModelElement("sp0", whitespace_str), identity_model,
FixedDataModelElement("sp1", whitespace_str), user_name_model,
FixedDataModelElement("sp2", whitespace_str), new_time_model,
FixedDataModelElement("sp3", b'] "'), request_method_model,
FixedDataModelElement("sp6", b'" '), status_code_model,
FixedDataModelElement("sp7", whitespace_str), size_model,
OptionalMatchModelElement(
"combined",
SequenceModelElement("combined", [
FixedDataModelElement("sp9", b' "'),
DelimitedDataModelElement("referer", b'"', b"\\"),
FixedDataModelElement("sp10", b'" "'),
DelimitedDataModelElement("user_agent", b'"', b"\\"),
FixedDataModelElement("sp11", b'"')
]))
])
return model
def get_model():
"""Return a model to parse Apache Access logs from the AIT-LDS."""
alphabet = b"!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]"
model = SequenceModelElement("model", [
FirstMatchModelElement("client_ip", [
IpAddressDataModelElement("client_ip"),
FixedDataModelElement("localhost", b"::1")
]),
FixedDataModelElement("sp1", b" "),
VariableByteDataModelElement("client_id", alphabet),
FixedDataModelElement("sp2", b" "),
VariableByteDataModelElement("user_id", alphabet),
FixedDataModelElement("sp3", b" ["),
DateTimeModelElement("time", b"%d/%b/%Y:%H:%M:%S%z"),
FixedDataModelElement("sp4", b'] "'),
FirstMatchModelElement("fm", [
FixedDataModelElement("dash", b"-"),
SequenceModelElement("request", [
FixedWordlistDataModelElement("method", [
b"GET", b"POST", b"PUT", b"HEAD", b"DELETE", b"CONNECT",
b"OPTIONS", b"TRACE", b"PATCH"
]),
FixedDataModelElement("sp5", b" "),
DelimitedDataModelElement("request", b" ", b"\\"),
FixedDataModelElement("sp6", b" "),
DelimitedDataModelElement("version", b'"'),
])
]),
FixedDataModelElement("sp7", b'" '),
DecimalIntegerValueModelElement("status_code"),
FixedDataModelElement("sp8", b" "),
DecimalIntegerValueModelElement("content_size"),
OptionalMatchModelElement(
"combined",
SequenceModelElement("combined", [
FixedDataModelElement("sp9", b' "'),
DelimitedDataModelElement("referer", b'"', b"\\"),
FixedDataModelElement("sp10", b'" "'),
DelimitedDataModelElement("user_agent", b'"', b"\\"),
FixedDataModelElement("sp11", b'"'),
]))
])
return model
def get_tmp_files_model():
"""Return a model to parse a systemd tmpfiles daemon message after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement("duplicate", [
FixedDataModelElement("s0", b'[/usr/lib/tmpfiles.d/var.conf:14] Duplicate line for path "'),
DelimitedDataModelElement("path", b'", ignoring.'),
FixedDataModelElement("s2", b'", ignoring.')
])
]
# Will fail on username models including the dot at the end.
model = SequenceModelElement("systemd-tmpfiles", [
FixedDataModelElement("sname", b"systemd-tmpfiles["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return a model to parse Apache Access logs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
model = SequenceModelElement('model', [
FirstMatchModelElement('client_ip', [
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('localhost', b'::1')
]),
FixedDataModelElement('sp1', b' '),
VariableByteDataModelElement('client_id', alphabet),
FixedDataModelElement('sp2', b' '),
VariableByteDataModelElement('user_id', alphabet),
FixedDataModelElement('sp3', b' ['),
DateTimeModelElement('time', b'%d/%b/%Y:%H:%M:%S'),
FixedDataModelElement('sp4', b' +'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('sp5', b'] "'),
FirstMatchModelElement('fm', [
FixedDataModelElement('dash', b'-'),
SequenceModelElement('request', [
FixedWordlistDataModelElement('method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH']),
FixedDataModelElement('sp6', b' '),
DelimitedDataModelElement('request', b' ', b'\\'),
FixedDataModelElement('sp7', b' '),
DelimitedDataModelElement('version', b'"'),
])
]),
FixedDataModelElement('sp8', b'" '),
DecimalIntegerValueModelElement('status_code'),
FixedDataModelElement('sp9', b' '),
DecimalIntegerValueModelElement('content_size'),
OptionalMatchModelElement(
'combined', SequenceModelElement('combined', [
FixedDataModelElement('sp10', b' "'),
DelimitedDataModelElement('referer', b'"', b'\\'),
FixedDataModelElement('sp11', b'" "'),
DelimitedDataModelElement('user_agent', b'"', b'\\'),
FixedDataModelElement('sp12', b'"'),
])),
])
return model
def get_model(user_name_model=None):
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
srcuser_name_model = VariableByteDataModelElement(
'srcuser', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
type_children = [
SequenceModelElement('su-good', [
FixedDataModelElement('s0', b'Successful su for '),
user_name_model,
FixedDataModelElement('s1', b' by '), srcuser_name_model
]),
SequenceModelElement('su-good', [
FixedDataModelElement('s0', b'+ '),
DelimitedDataModelElement('terminal', b' '),
FixedDataModelElement('s1', b' '), srcuser_name_model,
FixedDataModelElement('s2', b':'), user_name_model
]),
SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(su:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '), user_name_model,
OptionalMatchModelElement(
'openby',
SequenceModelElement('userinfo', [
FixedDataModelElement('s0', b' by (uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s1', b')')
]))
])
]
model = SequenceModelElement('su', [
FixedDataModelElement('sname', b'su['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model(user_name_model=None):
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
"user", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
srcuser_name_model = VariableByteDataModelElement(
"srcuser", b"0123456789abcdefghijklmnopqrstuvwxyz.-")
type_children = [
SequenceModelElement("su-good", [
FixedDataModelElement("s0", b"Successful su for "),
user_name_model,
FixedDataModelElement("s1", b" by "), srcuser_name_model
]),
SequenceModelElement("su-good", [
FixedDataModelElement("s0", b"+ "),
DelimitedDataModelElement("terminal", b" "),
FixedDataModelElement("s1", b" "), srcuser_name_model,
FixedDataModelElement("s2", b":"), user_name_model
]),
SequenceModelElement("pam", [
FixedDataModelElement("s0", b"pam_unix(su:session): session "),
FixedWordlistDataModelElement("change", [b"opened", b"closed"]),
FixedDataModelElement("s1", b" for user "), user_name_model,
OptionalMatchModelElement(
"openby",
SequenceModelElement("userinfo", [
FixedDataModelElement("s0", b" by (uid="),
DecimalIntegerValueModelElement("uid"),
FixedDataModelElement("s1", b")")
]))
])
]
model = SequenceModelElement("su", [
FixedDataModelElement("sname", b"su["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def test6get_match_element_match_context_input_validation(self):
"""Check if an exception is raised, when other classes than MatchContext are used in get_match_element."""
model_element = SequenceModelElement(self.id_, self.children)
data = b"string0 string1 string2"
model_element.get_match_element(self.path, DummyMatchContext(data))
model_element.get_match_element(self.path, MatchContext(data))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, MatchElement(None, data, None, None))
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, data.decode())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, 123.22)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, True)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, None)
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, [])
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, {"key": MatchContext(data)})
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, set())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, ())
self.assertRaises(AttributeError, model_element.get_match_element,
self.path, model_element)
def get_model(user_name_model=None):
"""Return a model to parse a cron message logged via syslog after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
type_children = [
SequenceModelElement('exec', [
FixedDataModelElement('s0', b'('), user_name_model,
FixedDataModelElement('s1', b') CMD '),
AnyByteDataModelElement('command')
]),
SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(cron:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '), user_name_model,
OptionalMatchModelElement(
'openby', FixedDataModelElement('default', b' by (uid=0)'))
])
]
model = FirstMatchModelElement('cron', [
SequenceModelElement('std', [
FixedDataModelElement('sname', b'CRON['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msgtype', type_children)
]),
SequenceModelElement('low', [
FixedDataModelElement('sname', b'cron['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: (*system*'),
DelimitedDataModelElement('rname', b') RELOAD ('),
FixedDataModelElement('s1', b') RELOAD ('),
DelimitedDataModelElement('fname', b')'),
FixedDataModelElement('s2', b')'),
])
])
return model
def get_systemd_model():
"""Return the parsing model for messages directly from systemd."""
type_children = [
FixedDataModelElement("apt-daily-start", b"Starting Daily apt upgrade and clean activities..."),
FixedDataModelElement("apt-daily-started", b"Started Daily apt upgrade and clean activities."),
FixedDataModelElement("apt-daily-finished", b"Finished Daily apt upgrade and clean activities."),
SequenceModelElement("service-succeeded", [
DelimitedDataModelElement("service", b" "),
FixedDataModelElement("s0", b" Succeeded.")
]),
FixedDataModelElement("clean-php", b"Finished Clean php session files."),
FixedDataModelElement("finished-logrotate", b"Finished Rotate log files."),
FixedDataModelElement("finished-man-db-daily", b"Finished Daily man-db regeneration."),
FixedDataModelElement("finished-ubuntu-advantages", b"Finished Ubuntu Advantage APT and MOTD Messages."),
FixedDataModelElement("finished-refresh", b"Finished Refresh fwupd metadata and update motd."),
FixedDataModelElement("finished-daily-apt", b"Finished Daily apt download activities."),
SequenceModelElement("apt-daily-timer", [
FixedDataModelElement("s0", b"apt-daily.timer: Adding "),
OptionalMatchModelElement("hopt", SequenceModelElement("hblock", [
DecimalIntegerValueModelElement("hours"),
FixedDataModelElement("s1", b"h ")
])),
DecimalIntegerValueModelElement("minutes"),
FixedDataModelElement("s2", b"min "),
DecimalFloatValueModelElement("seconds"),
FixedDataModelElement("s3", b"s random time.")
]),
FixedDataModelElement("tmp-file-cleanup", b"Starting Cleanup of Temporary Directories..."),
FixedDataModelElement("tmp-file-cleanup-started", b"Started Cleanup of Temporary Directories."),
SequenceModelElement("killing-process", [
DelimitedDataModelElement("service", b":"),
FixedDataModelElement("s0", b": Killing process "),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s1", b" (update-notifier) with signal SIGKILL.")
]),
SequenceModelElement("starting", [
FixedDataModelElement("s0", b"Starting "),
DelimitedDataModelElement("service", b"."),
FixedDataModelElement("s1", b"...")
]),
SequenceModelElement("started", [
FixedDataModelElement("s0", b"Started "),
DelimitedDataModelElement("service", b".", consume_delimiter=True)
]),
FixedDataModelElement("reloading", b"Reloading.")
]
model = SequenceModelElement("systemd", [
FixedDataModelElement("sname", b"systemd["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return the model."""
type_children = [
SequenceModelElement('sent', [
FixedDataModelElement('s0', b'Sent mail for '),
DelimitedDataModelElement('to-addr', b' ('),
FixedDataModelElement('s1', b' ('),
DelimitedDataModelElement('status', b') uid='),
FixedDataModelElement('s2', b') uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s3', b' username='******'username', b' outbytes='),
FixedDataModelElement('s4', b' outbytes='),
DecimalIntegerValueModelElement('bytes')
])
]
model = SequenceModelElement('ssmtp', [
FixedDataModelElement('sname', b'sSMTP['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return a model to parse Apache Error logs from the AIT-LDS."""
model = SequenceModelElement('model', [
FixedDataModelElement('sp1', b'['),
FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']),
FixedDataModelElement('sp2', b' '),
DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'),
FixedDataModelElement('error_str', b'] [:error] [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('sp3', b'] [client '),
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('client_port'),
FixedDataModelElement('php', b'] PHP '),
FirstMatchModelElement('fphp', [
SequenceModelElement('warning', [
FixedDataModelElement('warning_str', b'Warning: '),
FirstMatchModelElement('warning', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function', b')'),
FixedDataModelElement('compatible_str', b') should be compatible with '),
DelimitedDataModelElement('function2', b')'),
FixedDataModelElement('compatible_str', b') in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer')]),
SequenceModelElement('system', [
FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')])])]),
SequenceModelElement('notice', [
FixedDataModelElement('notice_str', b'Notice: Undefined index: '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('sp', b' in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')]),
SequenceModelElement('deprecated', [
FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class '
b'will not be constructors in a future version of PHP; '),
DelimitedDataModelElement('class', b' '),
FixedDataModelElement('constructor_str', b' has a deprecated constructor in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer'),
])])])
return model
def get_model(time_model=None):
"""
Return the model for parsing a standard syslog preamble including timestamp and hostname.
@param time_model when not none, the given model element is used for parsing timestamps. Otherwise a standard DateTimeModelElement
with format b'%b %d %H:%M:%S' is created. CAVEAT: the standard model may not work when log data timestamp locale does not match
host or shell environment locale. See MultiLocaleDatetime_modelElement instead.
"""
if time_model is None:
time_model = DateTimeModelElement('time', b'%b %d %H:%M:%S')
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz')
model = SequenceModelElement('syslog', [
time_model,
FixedDataModelElement('sp0', b' '), host_name_model,
FixedDataModelElement('sp1', b' ')
])
return model
def get_model():
"""Get the model."""
interface_name_model = VariableByteDataModelElement('interface', b'0123456789abcdefghijklmnopqrstuvwxyz.')
type_children = [
SequenceModelElement('exit', [
FixedDataModelElement('s0', b'ntpd exiting on signal '),
DecimalIntegerValueModelElement('signal')
]),
SequenceModelElement('listen-drop', [
FixedDataModelElement('s0', b'Listen and drop on '),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' '),
interface_name_model,
FixedDataModelElement('s2', b' '),
FirstMatchModelElement('address', [
IpAddressDataModelElement('ipv4'),
DelimitedDataModelElement('ipv6', b' ')
]),
FixedDataModelElement('s3', b' UDP 123')
]),
SequenceModelElement('listen-normal', [
FixedDataModelElement('s0', b'Listen normally on '),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' '),
interface_name_model,
FixedDataModelElement('s2', b' '),
IpAddressDataModelElement('ip'),
FirstMatchModelElement('msg', [
FixedDataModelElement('port-new', b':123'),
FixedDataModelElement('port-old', b' UDP 123')
])
]),
SequenceModelElement('listen-routing', [
FixedDataModelElement('s0', b'Listening on routing socket on fd #'),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' for interface updates')
]),
FixedDataModelElement('new-interfaces', b'new interface(s) found: waking up resolver'),
FixedDataModelElement('ntp-io', b'ntp_io: estimated max descriptors: 1024, initial socket boundary: 16'),
FixedDataModelElement('peers-refreshed', b'peers refreshed'),
SequenceModelElement('precision', [
FixedDataModelElement('s0', b'proto: precision = '),
DecimalFloatValueModelElement('precision'),
FixedDataModelElement('s1', b' usec')])]
model = SequenceModelElement('ntpd', [
FixedDataModelElement('sname', b'ntpd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return the model."""
type_children = [
FixedDataModelElement("start",
b" * Starting Tomcat servlet engine tomcat7"),
FixedDataModelElement("stop",
b" * Stopping Tomcat servlet engine tomcat7"),
FixedDataModelElement("done", b" ...done."),
AnyByteDataModelElement("unparsed")
]
model = SequenceModelElement("tomcat7", [
FixedDataModelElement("sname", b"tomcat7["),
DecimalIntegerValueModelElement("pid"),
FixedDataModelElement("s0", b"]: "),
FirstMatchModelElement("msg", type_children)
])
return model
def get_model():
"""Return the model."""
type_children = [
FixedDataModelElement('start',
b' * Starting Tomcat servlet engine tomcat7'),
FixedDataModelElement('stop',
b' * Stopping Tomcat servlet engine tomcat7'),
FixedDataModelElement('done', b' ...done.'),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('tomcat7', [
FixedDataModelElement('sname', b'tomcat7['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model