def test8special_characters_escape_no_match(self): """In this test case special character escaping is tested without matching.""" data = b'error: the command \\"python run.py\\" was not found\\" ' match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b'"', b"\\") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) data = rb"^This is a simple regex string. It costs 10\$.\$" match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"$", b"\\") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) data = b"the searched file is .gitignore ." match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b".", b" ") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def test4special_characters_escape(self): """In this test case special character escaping is tested.""" match_context = MatchContext( b'error: the command \\"python run.py\\" was not found" ') delimited_data_model_element = DelimitedDataModelElement( 'id', b'"', b'\\') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual( match_element.get_match_string(), b'error: the command \\"python run.py\\" was not found') match_context = MatchContext( rb'^This is a simple regex string. It costs 10\$.$') delimited_data_model_element = DelimitedDataModelElement( 'id', b'$', b'\\') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), rb'^This is a simple regex string. It costs 10\$.') match_context = MatchContext(b'the searched file is .gitignore.') delimited_data_model_element = DelimitedDataModelElement( 'id', b'.', b' ') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'the searched file is .gitignore')
def test6delimiter_string_no_match(self): """In this test case a whole string is searched for in the match_data with no match.""" data = b"this is a match context.\n" match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"other data") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"isa") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"context\n") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"this is a match context.\n") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def test7special_characters_escape(self): """In this test case special character escaping is tested. The delimiter is not consumed (consume_delimiter=False).""" data = b'error: the command \\"python run.py\\" was not found" ' value = b'error: the command \\"python run.py\\" was not found' match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b'"', b"\\") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) data = rb"^This is a simple regex string. It costs 10\$.$" value = rb"^This is a simple regex string. It costs 10\$." match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"$", b"\\") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) data = b"the searched file is .gitignore." value = b"the searched file is .gitignore" match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b".", b" ") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None)
def get_model(): """Return a model to parse Suricata Fast logs from the AIT-LDS.""" model = SequenceModelElement("model", [ DateTimeModelElement("time", b"%m/%d/%Y-%H:%M:%S.%f"), FixedDataModelElement("brack_str1", b" [**] ["), DecimalIntegerValueModelElement("id1"), FixedDataModelElement("sep1", b":"), DecimalIntegerValueModelElement("id2"), FixedDataModelElement("sep2", b":"), DecimalIntegerValueModelElement("id3"), FixedDataModelElement("sep3", b"] "), DelimitedDataModelElement("message", b" [**] "), FixedDataModelElement("classification_str", b" [**] [Classification: "), DelimitedDataModelElement("classification", b"]"), FixedDataModelElement("priority_str", b"] [Priority: "), DecimalIntegerValueModelElement("priority"), FixedDataModelElement("brack_str1", b"] {"), DelimitedDataModelElement("conn", b"}"), FixedDataModelElement("brack_str2", b"} "), IpAddressDataModelElement("src_ip"), FixedDataModelElement("colon", b":"), DecimalIntegerValueModelElement("src_port"), FixedDataModelElement("arrow_str", b" -> "), IpAddressDataModelElement("dst_ip"), FixedDataModelElement("colon", b":"), DecimalIntegerValueModelElement("dst_port"), ]) return model
def get_model(): """Return the model.""" type_children = [ SequenceModelElement("sent", [ FixedDataModelElement("s0", b"Sent mail for "), DelimitedDataModelElement("to-addr", b" ("), FixedDataModelElement("s1", b" ("), DelimitedDataModelElement("status", b") uid="), FixedDataModelElement("s2", b") uid="), DecimalIntegerValueModelElement("uid"), FixedDataModelElement("s3", b" username="******"username", b" outbytes="), FixedDataModelElement("s4", b" outbytes="), DecimalIntegerValueModelElement("bytes") ]), SequenceModelElement("sent", [ DelimitedDataModelElement("program", b" "), FixedDataModelElement("s0", b" sent mail for "), AnyByteDataModelElement("user") ]) ] model = SequenceModelElement("ssmtp", [ FixedDataModelElement("sname", b"sSMTP["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s0", b"]: "), FirstMatchModelElement("msg", type_children) ]) return model
def get_model(): """Return a model to parse Suricata Fast logs from the AIT-LDS.""" model = SequenceModelElement('model', [ DateTimeModelElement('time', b'%m/%d/%Y-%H:%M:%S.%f'), FixedDataModelElement('brack_str1', b' [**] ['), DecimalIntegerValueModelElement('id1'), FixedDataModelElement('sep1', b':'), DecimalIntegerValueModelElement('id2'), FixedDataModelElement('sep2', b':'), DecimalIntegerValueModelElement('id3'), FixedDataModelElement('sep3', b'] '), DelimitedDataModelElement('message', b' [**] '), FixedDataModelElement('classification_str', b' [**] [Classification: '), DelimitedDataModelElement('classification', b']'), FixedDataModelElement('priority_str', b'] [Priority: '), DecimalIntegerValueModelElement('priority'), FixedDataModelElement('brack_str1', b'] {'), DelimitedDataModelElement('conn', b'}'), FixedDataModelElement('brack_str2', b'} '), IpAddressDataModelElement('src_ip'), FixedDataModelElement('colon', b':'), DecimalIntegerValueModelElement('src_port'), FixedDataModelElement('arrow_str', b' -> '), IpAddressDataModelElement('dst_ip'), FixedDataModelElement('colon', b':'), DecimalIntegerValueModelElement('dst_port'), ]) return model
def test5consume_delimeter(self): """In this test case check if the consume_delimeter parameter is working properly.""" match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement( 'id', b'c', consume_delimiter=False) match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'this is a mat') match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement( 'id', b'c', consume_delimiter=True) match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'this is a matc') match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement( 'id', b' is', consume_delimiter=False) match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'this') match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement( 'id', b' is', consume_delimiter=True) match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'this is')
def get_systemd_model(): """Return the parsing model for messages directly from systemd.""" type_children = [ FixedDataModelElement("apt-daily-start", b"Starting Daily apt upgrade and clean activities..."), FixedDataModelElement("apt-daily-started", b"Started Daily apt upgrade and clean activities."), FixedDataModelElement("apt-daily-finished", b"Finished Daily apt upgrade and clean activities."), SequenceModelElement("service-succeeded", [ DelimitedDataModelElement("service", b" "), FixedDataModelElement("s0", b" Succeeded.") ]), FixedDataModelElement("clean-php", b"Finished Clean php session files."), FixedDataModelElement("finished-logrotate", b"Finished Rotate log files."), FixedDataModelElement("finished-man-db-daily", b"Finished Daily man-db regeneration."), FixedDataModelElement("finished-ubuntu-advantages", b"Finished Ubuntu Advantage APT and MOTD Messages."), FixedDataModelElement("finished-refresh", b"Finished Refresh fwupd metadata and update motd."), FixedDataModelElement("finished-daily-apt", b"Finished Daily apt download activities."), SequenceModelElement("apt-daily-timer", [ FixedDataModelElement("s0", b"apt-daily.timer: Adding "), OptionalMatchModelElement("hopt", SequenceModelElement("hblock", [ DecimalIntegerValueModelElement("hours"), FixedDataModelElement("s1", b"h ") ])), DecimalIntegerValueModelElement("minutes"), FixedDataModelElement("s2", b"min "), DecimalFloatValueModelElement("seconds"), FixedDataModelElement("s3", b"s random time.") ]), FixedDataModelElement("tmp-file-cleanup", b"Starting Cleanup of Temporary Directories..."), FixedDataModelElement("tmp-file-cleanup-started", b"Started Cleanup of Temporary Directories."), SequenceModelElement("killing-process", [ DelimitedDataModelElement("service", b":"), FixedDataModelElement("s0", b": Killing process "), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s1", b" (update-notifier) with signal SIGKILL.") ]), SequenceModelElement("starting", [ FixedDataModelElement("s0", b"Starting "), DelimitedDataModelElement("service", b"."), FixedDataModelElement("s1", b"...") ]), SequenceModelElement("started", [ FixedDataModelElement("s0", b"Started "), DelimitedDataModelElement("service", b".", consume_delimiter=True) ]), FixedDataModelElement("reloading", b"Reloading.") ] model = SequenceModelElement("systemd", [ FixedDataModelElement("sname", b"systemd["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s0", b"]: "), FirstMatchModelElement("msg", type_children) ]) return model
def test1delimeter_single_char(self): """A single character is used as delimeter.""" match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement('id', b'c') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'this is a mat') match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement('id', b'f') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element, None)
def get_model(): """Return a model to parse messages from kernel logging.""" type_children = [ SequenceModelElement("ipv4-martian", [ FixedDataModelElement("s0", b"IPv4: martian "), FixedWordlistDataModelElement("direction", [b"source", b"destination"]), FixedDataModelElement("s1", b" "), IpAddressDataModelElement("destination"), FixedDataModelElement("s2", b" from "), IpAddressDataModelElement("source"), FixedDataModelElement("s3", b", on dev "), AnyByteDataModelElement("interface")]), SequenceModelElement("net-llheader", [ FixedDataModelElement("s0", b"ll header: "), AnyByteDataModelElement("data") ]), AnyByteDataModelElement("unparsed") ] model = SequenceModelElement("kernel", [ FixedDataModelElement("sname", b"kernel: "), OptionalMatchModelElement("opt", SequenceModelElement("seq", [ FixedDataModelElement("opt_s0", b"]"), DelimitedDataModelElement("timestamp", b"]"), FixedDataModelElement("opt_s1", b"] "), ])), FirstMatchModelElement("msg", type_children) ]) return model
def get_model(): """Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog.""" type_children = [ SequenceModelElement('gidchange', [ FixedDataModelElement('s0', b'rsyslogd\'s groupid changed to '), DecimalIntegerValueModelElement('gid') ]), SequenceModelElement('statechange', [ FixedDataModelElement('s0', b'[origin software="rsyslogd" swVersion="'), DelimitedDataModelElement('version', b'"'), FixedDataModelElement('s1', b'" x-pid="'), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s2', b'" x-info="http://www.rsyslog.com"] '), FirstMatchModelElement('type', [ FixedDataModelElement('HUPed', b'rsyslogd was HUPed'), FixedDataModelElement('start', b'start') ]) ]), SequenceModelElement('uidchange', [ FixedDataModelElement('s0', b'rsyslogd\'s userid changed to '), DecimalIntegerValueModelElement('uid') ]) ] model = SequenceModelElement('rsyslog', [ FixedDataModelElement('sname', b'rsyslogd: '), FirstMatchModelElement('msg', type_children) ]) return model
def get_model(): """Return a model to parse messages from kernel logging.""" type_children = [ SequenceModelElement('ipv4-martian', [ FixedDataModelElement('s0', b'IPv4: martian '), FixedWordlistDataModelElement('direction', [b'source', b'destination']), FixedDataModelElement('s1', b' '), IpAddressDataModelElement('destination'), FixedDataModelElement('s2', b' from '), IpAddressDataModelElement('source'), FixedDataModelElement('s3', b', on dev '), AnyByteDataModelElement('interface') ]), SequenceModelElement('net-llheader', [ FixedDataModelElement('s0', b'll header: '), AnyByteDataModelElement('data') ]), AnyByteDataModelElement('unparsed') ] model = SequenceModelElement('kernel', [ FixedDataModelElement('sname', b'kernel: ['), DelimitedDataModelElement('timestamp', b']'), FixedDataModelElement('s0', b'] '), FirstMatchModelElement('msg', type_children) ]) return model
def test14get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = DelimitedDataModelElement(self.id_, self.delimiter) data = b"one, two, three" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(None, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def test2delimeter_string(self): """In this test case a whole string is searched for in the match_data.""" match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement('id', b' is') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element.get_match_string(), b'this')
def get_model(): """Return a parser for apache2 access.log.""" new_time_model = DateTimeModelElement("time", b"[%d/%b/%Y:%H:%M:%S%z") host_name_model = VariableByteDataModelElement( "host", b"-.01234567890abcdefghijklmnopqrstuvwxyz:") identity_model = VariableByteDataModelElement( "ident", b"-.01234567890abcdefghijklmnopqrstuvwxyz:") user_name_model = VariableByteDataModelElement( "user", b"0123456789abcdefghijklmnopqrstuvwxyz.-") request_method_model = FirstMatchModelElement("fm", [ FixedDataModelElement("dash", b"-"), SequenceModelElement("request", [ FixedWordlistDataModelElement("method", [ b"GET", b"POST", b"PUT", b"HEAD", b"DELETE", b"CONNECT", b"OPTIONS", b"TRACE", b"PATCH" ]), FixedDataModelElement("sp5", b" "), DelimitedDataModelElement("request", b" ", b"\\"), FixedDataModelElement("sp6", b" "), DelimitedDataModelElement("version", b'"'), ]) ]) status_code_model = DecimalIntegerValueModelElement("status") size_model = DecimalIntegerValueModelElement("size") whitespace_str = b" " model = SequenceModelElement("accesslog", [ host_name_model, FixedDataModelElement("sp0", whitespace_str), identity_model, FixedDataModelElement("sp1", whitespace_str), user_name_model, FixedDataModelElement("sp2", whitespace_str), new_time_model, FixedDataModelElement("sp3", b'] "'), request_method_model, FixedDataModelElement("sp6", b'" '), status_code_model, FixedDataModelElement("sp7", whitespace_str), size_model, OptionalMatchModelElement( "combined", SequenceModelElement("combined", [ FixedDataModelElement("sp9", b' "'), DelimitedDataModelElement("referer", b'"', b"\\"), FixedDataModelElement("sp10", b'" "'), DelimitedDataModelElement("user_agent", b'"', b"\\"), FixedDataModelElement("sp11", b'"') ])) ]) return model
def get_model(): """Return a model to parse Apache Access logs from the AIT-LDS.""" alphabet = b"!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]" model = SequenceModelElement("model", [ FirstMatchModelElement("client_ip", [ IpAddressDataModelElement("client_ip"), FixedDataModelElement("localhost", b"::1") ]), FixedDataModelElement("sp1", b" "), VariableByteDataModelElement("client_id", alphabet), FixedDataModelElement("sp2", b" "), VariableByteDataModelElement("user_id", alphabet), FixedDataModelElement("sp3", b" ["), DateTimeModelElement("time", b"%d/%b/%Y:%H:%M:%S%z"), FixedDataModelElement("sp4", b'] "'), FirstMatchModelElement("fm", [ FixedDataModelElement("dash", b"-"), SequenceModelElement("request", [ FixedWordlistDataModelElement("method", [ b"GET", b"POST", b"PUT", b"HEAD", b"DELETE", b"CONNECT", b"OPTIONS", b"TRACE", b"PATCH" ]), FixedDataModelElement("sp5", b" "), DelimitedDataModelElement("request", b" ", b"\\"), FixedDataModelElement("sp6", b" "), DelimitedDataModelElement("version", b'"'), ]) ]), FixedDataModelElement("sp7", b'" '), DecimalIntegerValueModelElement("status_code"), FixedDataModelElement("sp8", b" "), DecimalIntegerValueModelElement("content_size"), OptionalMatchModelElement( "combined", SequenceModelElement("combined", [ FixedDataModelElement("sp9", b' "'), DelimitedDataModelElement("referer", b'"', b"\\"), FixedDataModelElement("sp10", b'" "'), DelimitedDataModelElement("user_agent", b'"', b"\\"), FixedDataModelElement("sp11", b'"'), ])) ]) return model
def get_model(): """Return a model to parse Apache Error logs from the AIT-LDS.""" model = SequenceModelElement('model', [ FixedDataModelElement('sp1', b'['), FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']), FixedDataModelElement('sp2', b' '), DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'), FixedDataModelElement('error_str', b'] [:error] [pid '), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('sp3', b'] [client '), IpAddressDataModelElement('client_ip'), FixedDataModelElement('colon', b':'), DecimalIntegerValueModelElement('client_port'), FixedDataModelElement('php', b'] PHP '), FirstMatchModelElement('fphp', [ SequenceModelElement('warning', [ FixedDataModelElement('warning_str', b'Warning: '), FirstMatchModelElement('warning', [ SequenceModelElement('declaration', [ FixedDataModelElement('declaration_str', b'Declaration of '), DelimitedDataModelElement('function', b')'), FixedDataModelElement('compatible_str', b') should be compatible with '), DelimitedDataModelElement('function2', b')'), FixedDataModelElement('compatible_str', b') in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line'), FixedDataModelElement('referer_str', b', referer: '), AnyByteDataModelElement('referer')]), SequenceModelElement('system', [ FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line')])])]), SequenceModelElement('notice', [ FixedDataModelElement('notice_str', b'Notice: Undefined index: '), DelimitedDataModelElement('command', b' '), FixedDataModelElement('sp', b' in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line')]), SequenceModelElement('deprecated', [ FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class ' b'will not be constructors in a future version of PHP; '), DelimitedDataModelElement('class', b' '), FixedDataModelElement('constructor_str', b' has a deprecated constructor in '), DelimitedDataModelElement('path', b' '), FixedDataModelElement('compatible_str', b' on line '), DecimalIntegerValueModelElement('line'), FixedDataModelElement('referer_str', b', referer: '), AnyByteDataModelElement('referer'), ])])]) return model
def get_model(): """Return a model to parse Apache Access logs from the AIT-LDS.""" alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]' model = SequenceModelElement('model', [ FirstMatchModelElement('client_ip', [ IpAddressDataModelElement('client_ip'), FixedDataModelElement('localhost', b'::1') ]), FixedDataModelElement('sp1', b' '), VariableByteDataModelElement('client_id', alphabet), FixedDataModelElement('sp2', b' '), VariableByteDataModelElement('user_id', alphabet), FixedDataModelElement('sp3', b' ['), DateTimeModelElement('time', b'%d/%b/%Y:%H:%M:%S'), FixedDataModelElement('sp4', b' +'), DecimalIntegerValueModelElement('tz'), FixedDataModelElement('sp5', b'] "'), FirstMatchModelElement('fm', [ FixedDataModelElement('dash', b'-'), SequenceModelElement('request', [ FixedWordlistDataModelElement('method', [ b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH']), FixedDataModelElement('sp6', b' '), DelimitedDataModelElement('request', b' ', b'\\'), FixedDataModelElement('sp7', b' '), DelimitedDataModelElement('version', b'"'), ]) ]), FixedDataModelElement('sp8', b'" '), DecimalIntegerValueModelElement('status_code'), FixedDataModelElement('sp9', b' '), DecimalIntegerValueModelElement('content_size'), OptionalMatchModelElement( 'combined', SequenceModelElement('combined', [ FixedDataModelElement('sp10', b' "'), DelimitedDataModelElement('referer', b'"', b'\\'), FixedDataModelElement('sp11', b'" "'), DelimitedDataModelElement('user_agent', b'"', b'\\'), FixedDataModelElement('sp12', b'"'), ])), ]) return model
def test3delimeter_none_empty_or_not_printable(self): """In this test case all not allowed values are tested.""" match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement('id', b'') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element, None) match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement('id', None) self.assertRaises(TypeError, delimited_data_model_element.get_match_element, 'match', match_context) match_context = MatchContext(self.match_context_string) delimited_data_model_element = DelimitedDataModelElement('id', b'\x01') match_element = delimited_data_model_element.get_match_element( 'match', match_context) self.assertEqual(match_element, None)
def test4get_match_element_single_char_no_match(self): """A single character is used as delimiter and not matched.""" data = b"this is a match context.\n" for char in "bdfgjklpqruvwyz": delimited_data_model_element = DelimitedDataModelElement( self.id_, char.encode()) match_context = DummyMatchContext(data) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def get_model(): """Return a model for su session information messages after any standard logging preamble, e.g. from syslog.""" type_children = [ SequenceModelElement('build-stack', [ FixedDataModelElement('s0', b'building new pluginstance stack: \''), DelimitedDataModelElement('stack', b'\''), FixedDataModelElement('s1', b'\'') ]), SequenceModelElement('nfct-event', [ FixedDataModelElement('s0', b'[DESTROY] ORIG: SRC='), IpAddressDataModelElement('osrcip'), FixedDataModelElement('s1', b' DST='), IpAddressDataModelElement('odstip'), FixedDataModelElement('s2', b' PROTO='), FixedWordlistDataModelElement('proto', [b'TCP', b'UDP']), FixedDataModelElement('s3', b' SPT='), DecimalIntegerValueModelElement('ospt'), FixedDataModelElement('s4', b' DPT='), DecimalIntegerValueModelElement('odpt'), FixedDataModelElement('s5', b' PKTS='), DecimalIntegerValueModelElement('opkts'), FixedDataModelElement('s6', b' BYTES='), DecimalIntegerValueModelElement('obytes'), FixedDataModelElement('s7', b' , REPLY: SRC='), IpAddressDataModelElement('rsrcip'), FixedDataModelElement('s8', b' DST='), IpAddressDataModelElement('rdstip'), FixedDataModelElement('s9', b' PROTO='), FixedWordlistDataModelElement('rproto', [b'TCP', b'UDP']), FixedDataModelElement('s10', b' SPT='), DecimalIntegerValueModelElement('rspt'), FixedDataModelElement('s11', b' DPT='), DecimalIntegerValueModelElement('rdpt'), FixedDataModelElement('s12', b' PKTS='), DecimalIntegerValueModelElement('rpkts'), FixedDataModelElement('s13', b' BYTES='), DecimalIntegerValueModelElement('rbytes'), # No additional whitespace from Ubuntu Trusty 14.04 on. OptionalMatchModelElement('tail', FixedDataModelElement('s0', b' ')) ]), FixedDataModelElement('nfct-plugin', b'NFCT plugin working in event mode'), FixedDataModelElement('reopen', b'reopening capture file'), FixedDataModelElement('signal', b'signal received, calling pluginstances'), FixedDataModelElement('uidchange', b'Changing UID / GID') ] # Netflow entry model = SequenceModelElement('ulogd', [ FixedDataModelElement('sname', b'ulogd['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s0', b']: '), FirstMatchModelElement('msg', type_children) ]) return model
def get_model(): """Get the model.""" interface_name_model = VariableByteDataModelElement('interface', b'0123456789abcdefghijklmnopqrstuvwxyz.') type_children = [ SequenceModelElement('exit', [ FixedDataModelElement('s0', b'ntpd exiting on signal '), DecimalIntegerValueModelElement('signal') ]), SequenceModelElement('listen-drop', [ FixedDataModelElement('s0', b'Listen and drop on '), DecimalIntegerValueModelElement('fd'), FixedDataModelElement('s1', b' '), interface_name_model, FixedDataModelElement('s2', b' '), FirstMatchModelElement('address', [ IpAddressDataModelElement('ipv4'), DelimitedDataModelElement('ipv6', b' ') ]), FixedDataModelElement('s3', b' UDP 123') ]), SequenceModelElement('listen-normal', [ FixedDataModelElement('s0', b'Listen normally on '), DecimalIntegerValueModelElement('fd'), FixedDataModelElement('s1', b' '), interface_name_model, FixedDataModelElement('s2', b' '), IpAddressDataModelElement('ip'), FirstMatchModelElement('msg', [ FixedDataModelElement('port-new', b':123'), FixedDataModelElement('port-old', b' UDP 123') ]) ]), SequenceModelElement('listen-routing', [ FixedDataModelElement('s0', b'Listening on routing socket on fd #'), DecimalIntegerValueModelElement('fd'), FixedDataModelElement('s1', b' for interface updates') ]), FixedDataModelElement('new-interfaces', b'new interface(s) found: waking up resolver'), FixedDataModelElement('ntp-io', b'ntp_io: estimated max descriptors: 1024, initial socket boundary: 16'), FixedDataModelElement('peers-refreshed', b'peers refreshed'), SequenceModelElement('precision', [ FixedDataModelElement('s0', b'proto: precision = '), DecimalFloatValueModelElement('precision'), FixedDataModelElement('s1', b' usec')])] model = SequenceModelElement('ntpd', [ FixedDataModelElement('sname', b'ntpd['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s0', b']: '), FirstMatchModelElement('msg', type_children) ]) return model
def test3get_match_element_single_char(self): """A single character is used as delimiter and not consumed (consume_delimiter=False).""" data = b"this is a match context.\n" delimited_data_model_element = DelimitedDataModelElement( self.id_, b"a") value = b"this is " match_context = DummyMatchContext(data) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"c") value = b"this is a mat" match_context = DummyMatchContext(data) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"e") value = b"this is a match cont" match_context = DummyMatchContext(data) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"\n") value = b"this is a match context." match_context = DummyMatchContext(data) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None)
def get_model(user_name_model=None): """Return a model to parse a cron message logged via syslog after any standard logging preamble, e.g. from syslog.""" if user_name_model is None: user_name_model = VariableByteDataModelElement( 'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-') type_children = [ SequenceModelElement('exec', [ FixedDataModelElement('s0', b'('), user_name_model, FixedDataModelElement('s1', b') CMD '), AnyByteDataModelElement('command') ]), SequenceModelElement('pam', [ FixedDataModelElement('s0', b'pam_unix(cron:session): session '), FixedWordlistDataModelElement('change', [b'opened', b'closed']), FixedDataModelElement('s1', b' for user '), user_name_model, OptionalMatchModelElement( 'openby', FixedDataModelElement('default', b' by (uid=0)')) ]) ] model = FirstMatchModelElement('cron', [ SequenceModelElement('std', [ FixedDataModelElement('sname', b'CRON['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s0', b']: '), FirstMatchModelElement('msgtype', type_children) ]), SequenceModelElement('low', [ FixedDataModelElement('sname', b'cron['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s0', b']: (*system*'), DelimitedDataModelElement('rname', b') RELOAD ('), FixedDataModelElement('s1', b') RELOAD ('), DelimitedDataModelElement('fname', b')'), FixedDataModelElement('s2', b')'), ]) ]) return model
def test5delimiter_string(self): """In this test case a whole string is searched for in the match_data and it is not consumed (consume_delimiter=False).""" data = b"this is a match context.\n" value = b"this" match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b" is") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) value = b"th" match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"is") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) value = b"this is a match " match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"context.\n") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None) value = b"t" match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"his is a match context.\n") match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, value, None)
def test10consume_delimiter_no_match(self): """In this test case check if the consume_delimiter parameter is working properly and does not match data.""" data = b"this is a match context.\n" for char in "bdfgjklpqruvwyz": delimited_data_model_element = DelimitedDataModelElement( self.id_, char.encode(), consume_delimiter=True) match_context = DummyMatchContext(data) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"other data", consume_delimiter=True) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"isa", consume_delimiter=True) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"context\n", consume_delimiter=True) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) match_context = DummyMatchContext(data) delimited_data_model_element = DelimitedDataModelElement( self.id_, b"this is a match context.\n", consume_delimiter=True) match_element = delimited_data_model_element.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def get_model(): """Return the model.""" type_children = [ SequenceModelElement('sent', [ FixedDataModelElement('s0', b'Sent mail for '), DelimitedDataModelElement('to-addr', b' ('), FixedDataModelElement('s1', b' ('), DelimitedDataModelElement('status', b') uid='), FixedDataModelElement('s2', b') uid='), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s3', b' username='******'username', b' outbytes='), FixedDataModelElement('s4', b' outbytes='), DecimalIntegerValueModelElement('bytes') ]) ] model = SequenceModelElement('ssmtp', [ FixedDataModelElement('sname', b'sSMTP['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s0', b']: '), FirstMatchModelElement('msg', type_children) ]) return model
def get_tmp_files_model(): """Return a model to parse a systemd tmpfiles daemon message after any standard logging preamble, e.g. from syslog.""" type_children = [ SequenceModelElement("duplicate", [ FixedDataModelElement("s0", b'[/usr/lib/tmpfiles.d/var.conf:14] Duplicate line for path "'), DelimitedDataModelElement("path", b'", ignoring.'), FixedDataModelElement("s2", b'", ignoring.') ]) ] # Will fail on username models including the dot at the end. model = SequenceModelElement("systemd-tmpfiles", [ FixedDataModelElement("sname", b"systemd-tmpfiles["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s0", b"]: "), FirstMatchModelElement("msg", type_children) ]) return model
def get_model(user_name_model=None): """Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog.""" if user_name_model is None: user_name_model = VariableByteDataModelElement( "user", b"0123456789abcdefghijklmnopqrstuvwxyz.-") srcuser_name_model = VariableByteDataModelElement( "srcuser", b"0123456789abcdefghijklmnopqrstuvwxyz.-") type_children = [ SequenceModelElement("su-good", [ FixedDataModelElement("s0", b"Successful su for "), user_name_model, FixedDataModelElement("s1", b" by "), srcuser_name_model ]), SequenceModelElement("su-good", [ FixedDataModelElement("s0", b"+ "), DelimitedDataModelElement("terminal", b" "), FixedDataModelElement("s1", b" "), srcuser_name_model, FixedDataModelElement("s2", b":"), user_name_model ]), SequenceModelElement("pam", [ FixedDataModelElement("s0", b"pam_unix(su:session): session "), FixedWordlistDataModelElement("change", [b"opened", b"closed"]), FixedDataModelElement("s1", b" for user "), user_name_model, OptionalMatchModelElement( "openby", SequenceModelElement("userinfo", [ FixedDataModelElement("s0", b" by (uid="), DecimalIntegerValueModelElement("uid"), FixedDataModelElement("s1", b")") ])) ]) ] model = SequenceModelElement("su", [ FixedDataModelElement("sname", b"su["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s0", b"]: "), FirstMatchModelElement("msg", type_children) ]) return model