def get_model():
"""Return a model to parse Suricata Fast logs from the AIT-LDS."""
model = SequenceModelElement('model', [
DateTimeModelElement('time', b'%m/%d/%Y-%H:%M:%S.%f'),
FixedDataModelElement('brack_str1', b' [**] ['),
DecimalIntegerValueModelElement('id1'),
FixedDataModelElement('sep1', b':'),
DecimalIntegerValueModelElement('id2'),
FixedDataModelElement('sep2', b':'),
DecimalIntegerValueModelElement('id3'),
FixedDataModelElement('sep3', b'] '),
DelimitedDataModelElement('message', b' [**] '),
FixedDataModelElement('classification_str',
b' [**] [Classification: '),
DelimitedDataModelElement('classification', b']'),
FixedDataModelElement('priority_str', b'] [Priority: '),
DecimalIntegerValueModelElement('priority'),
FixedDataModelElement('brack_str1', b'] {'),
DelimitedDataModelElement('conn', b'}'),
FixedDataModelElement('brack_str2', b'} '),
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('arrow_str', b' -> '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('dst_port'),
])
return model
def get_model():
new_time_model = SequenceModelElement('time_model', [
DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S '),
FixedWordlistDataModelElement('sign', [b'+', b'-']),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('bracket', b']')
])
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identity_model = VariableByteDataModelElement(
'ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
request_method_model = FixedWordlistDataModelElement(
'method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT',
b'OPTIONS', b'TRACE', b'PATCH'
])
request_model = VariableByteDataModelElement(
'request',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+'
)
version_model = VariableByteDataModelElement('version', b'0123456789.')
status_code_model = DecimalIntegerValueModelElement('status')
size_model = DecimalIntegerValueModelElement('size')
user_agent_model = VariableByteDataModelElement(
'useragent',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+;:_ '
)
whitespace_str = b' '
model = SequenceModelElement('accesslog', [
host_name_model,
FixedDataModelElement('sp0', whitespace_str),
identity_model,
FixedDataModelElement('sp1', whitespace_str),
user_name_model,
FixedDataModelElement('sp2', whitespace_str),
new_time_model,
FixedDataModelElement('sp3', b' "'),
request_method_model,
FixedDataModelElement('sp4', whitespace_str),
request_model,
FixedDataModelElement('sp5', b' HTTP/'),
version_model,
FixedDataModelElement('sp6', b'" '),
status_code_model,
FixedDataModelElement('sp7', whitespace_str),
size_model,
FixedDataModelElement('sp8', b' "-" "'),
user_agent_model,
FixedDataModelElement('sp9', b'"'),
])
return model
def get_model():
"""Return a model to parse Apache Error logs from the AIT-LDS."""
model = SequenceModelElement('model', [
FixedDataModelElement('sp1', b'['),
FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']),
FixedDataModelElement('sp2', b' '),
DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'),
FixedDataModelElement('error_str', b'] [:error] [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('sp3', b'] [client '),
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('client_port'),
FixedDataModelElement('php', b'] PHP '),
FirstMatchModelElement('fphp', [
SequenceModelElement('warning', [
FixedDataModelElement('warning_str', b'Warning: '),
FirstMatchModelElement('warning', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function', b')'),
FixedDataModelElement('compatible_str', b') should be compatible with '),
DelimitedDataModelElement('function2', b')'),
FixedDataModelElement('compatible_str', b') in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer')]),
SequenceModelElement('system', [
FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')])])]),
SequenceModelElement('notice', [
FixedDataModelElement('notice_str', b'Notice: Undefined index: '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('sp', b' in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')]),
SequenceModelElement('deprecated', [
FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class '
b'will not be constructors in a future version of PHP; '),
DelimitedDataModelElement('class', b' '),
FixedDataModelElement('constructor_str', b' has a deprecated constructor in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer'),
])])])
return model
def get_model(time_model=None):
"""This function defines the model for parsing a standard syslog preamble including timestamp and hostname.
@param time_model when not none, the given model element is used for parsing timestamps. Otherwise a standard DateTimeModelElement
with format b'%b %d %H:%M:%S' is created. CAVEAT: the standard model may not work when log data timestamp locale does not match
host or shell environment locale. See MultiLocaleDatetime_modelElement instead."""
if time_model is None:
time_model = DateTimeModelElement('time', b'%b %d %H:%M:%S')
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz')
model = SequenceModelElement('syslog', [
time_model,
FixedDataModelElement('sp0', b' '), host_name_model,
FixedDataModelElement('sp1', b' ')
])
return model
def get_model():
"""Return a model to parse Apache Access logs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
model = SequenceModelElement('model', [
FirstMatchModelElement('client_ip', [
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('localhost', b'::1')
]),
FixedDataModelElement('sp1', b' '),
VariableByteDataModelElement('client_id', alphabet),
FixedDataModelElement('sp2', b' '),
VariableByteDataModelElement('user_id', alphabet),
FixedDataModelElement('sp3', b' ['),
DateTimeModelElement('time', b'%d/%b/%Y:%H:%M:%S'),
FixedDataModelElement('sp4', b' +'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('sp5', b'] "'),
FirstMatchModelElement('fm', [
FixedDataModelElement('dash', b'-'),
SequenceModelElement('request', [
FixedWordlistDataModelElement('method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH']),
FixedDataModelElement('sp6', b' '),
DelimitedDataModelElement('request', b' ', b'\\'),
FixedDataModelElement('sp7', b' '),
DelimitedDataModelElement('version', b'"'),
])
]),
FixedDataModelElement('sp8', b'" '),
DecimalIntegerValueModelElement('status_code'),
FixedDataModelElement('sp9', b' '),
DecimalIntegerValueModelElement('content_size'),
OptionalMatchModelElement(
'combined', SequenceModelElement('combined', [
FixedDataModelElement('sp10', b' "'),
DelimitedDataModelElement('referer', b'"', b'\\'),
FixedDataModelElement('sp11', b'" "'),
DelimitedDataModelElement('user_agent', b'"', b'\\'),
FixedDataModelElement('sp12', b'"'),
])),
])
return model
def getModel(timeModel=None):
timeModel = DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S +0000]')
hostNameModel = VariableByteDataModelElement('host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identityModel = VariableByteDataModelElement('ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
userNameModel = VariableByteDataModelElement('user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
requestMethodModel = FixedWordlistDataModelElement('method', [b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH'])
requestModel = VariableByteDataModelElement('request', b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+')
versionModel = VariableByteDataModelElement('version', b'0123456789.')
statuscodeModel = DecimalIntegerValueModelElement('status')
sizeModel = DecimalIntegerValueModelElement('size')
useragentModel = VariableByteDataModelElement('useragent', b'0123456789abcdefghijklmnopqrstuvwxyz.-/()[]{}!$%&=<?*+')
model = SequenceModelElement('accesslog', [
hostNameModel,
FixedDataModelElement('sp0', b' '),
identityModel,
FixedDataModelElement('sp1', b' '),
userNameModel,
FixedDataModelElement('sp2', b' '),
timeModel,
FixedDataModelElement('sp3', b' "'),
requestMethodModel,
FixedDataModelElement('sp4', b' '),
requestModel,
FixedDataModelElement('sp5', b' HTTP/'),
versionModel,
# AnyByteDataModelElement('any')
FixedDataModelElement('sp6', b'" '),
statuscodeModel,
FixedDataModelElement('sp7', b' '),
sizeModel,
FixedDataModelElement('sp8', b' "-" "'),
useragentModel,
FixedDataModelElement('sp9', b'"'),
])
return model
def get_model():
"""Return a model to parse Suricata Event logs from the AIT-LDS."""
conn = SequenceModelElement('conn', [
FixedDataModelElement('src_ip_str', b'"src_ip":"'),
FirstMatchModelElement('ip', [
SequenceModelElement('ipv4', [
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('src_port_str', b'","src_port":'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('dest_ip_str', b',"dest_ip":"'),
IpAddressDataModelElement('dest_ip'),
FixedDataModelElement('dest_port_str', b'","dest_port":'),
DecimalIntegerValueModelElement('dest_port'),
FixedDataModelElement('proto_str', b',"proto":"'),
DelimitedDataModelElement('proto', b'"'),
FixedDataModelElement('quote', b'"')
]),
SequenceModelElement('ipv6', [
DelimitedDataModelElement('src_ip', b'"'),
FixedDataModelElement('dest_ip_str', b'","dest_ip":"'),
DelimitedDataModelElement('dest_ip', b'"'),
FixedDataModelElement('proto_str', b'","proto":"'),
DelimitedDataModelElement('proto', b'"'),
FixedDataModelElement('icmp_type_str', b'","icmp_type":'),
DecimalIntegerValueModelElement('icmp_type'),
FixedDataModelElement('icmp_code_str', b',"icmp_code":'),
DecimalIntegerValueModelElement('icmp_code'),
]),
])
])
http = SequenceModelElement('http', [
FixedDataModelElement('hostname_str', b',"http":{"hostname":"'),
DelimitedDataModelElement('hostname', b'"'),
FixedDataModelElement('url_str', b'","url":"'),
DelimitedDataModelElement('url', b'"', escape=b'\\'),
FixedDataModelElement('http_user_agent_str', b'","http_user_agent":"'),
DelimitedDataModelElement('http_user_agent', b'"'),
OptionalMatchModelElement(
'content_type', SequenceModelElement('content_type', [
FixedDataModelElement('http_content_type_str', b'","http_content_type":"'),
DelimitedDataModelElement('http_content_type', b'"'),
])),
OptionalMatchModelElement(
'http_refer', SequenceModelElement('http_refer', [
FixedDataModelElement('http_refer_str', b'","http_refer":"'),
DelimitedDataModelElement('http_refer', b'"'),
])),
FixedDataModelElement('http_method_str', b'","http_method":"'),
DelimitedDataModelElement('http_method', b'"'),
FixedDataModelElement('protocol_str', b'","protocol":"'),
DelimitedDataModelElement('protocol', b'"'),
FixedDataModelElement('quote_str', b'"'),
OptionalMatchModelElement(
'status', SequenceModelElement('status', [
FixedDataModelElement('status_str', b',"status":'),
DecimalIntegerValueModelElement('status'),
])),
OptionalMatchModelElement(
'redirect', SequenceModelElement('redirect', [
FixedDataModelElement('redirect_str', b',"redirect":"'),
DelimitedDataModelElement('redirect', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
FixedDataModelElement('length_str', b',"length":'),
DecimalIntegerValueModelElement('length'),
FixedDataModelElement('brack_str', b'}')
])
model = SequenceModelElement('model', [
FixedDataModelElement('time_str', b'{"timestamp":"'),
DateTimeModelElement('time', b'%Y-%m-%dT%H:%M:%S.%f'),
FixedDataModelElement('plus_sign', b'+'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('comma_str', b'",'),
OptionalMatchModelElement(
'flow_id', SequenceModelElement('flow_id', [
FixedDataModelElement('flow_id_str', b'"flow_id":'),
DecimalIntegerValueModelElement('flow_id'),
FixedDataModelElement('comma_str', b',')])),
OptionalMatchModelElement(
'in_iface', SequenceModelElement('in_iface', [
FixedDataModelElement('in_iface_str', b'"in_iface":"'),
DelimitedDataModelElement('in_iface', b'"'),
FixedDataModelElement('comma_str', b'",')])),
FixedDataModelElement('event_type_str', b'"event_type":"'),
FirstMatchModelElement('event_type', [
SequenceModelElement('dns', [
FixedDataModelElement('dns_str', b'dns",'),
conn,
SequenceModelElement('dns', [
FixedDataModelElement('type_str', b',"dns":{"type":"'),
DelimitedDataModelElement('type', b'"'),
FixedDataModelElement('id_str', b'","id":'),
DecimalIntegerValueModelElement('id'),
OptionalMatchModelElement(
'rcode', SequenceModelElement('rcode', [
FixedDataModelElement('rcode_str', b',"rcode":"'),
DelimitedDataModelElement('rcode', b'"'),
FixedDataModelElement('quote_str', b'"')])),
FixedDataModelElement('rrname_str', b',"rrname":"'),
DelimitedDataModelElement('rrname', b'"'),
OptionalMatchModelElement('rrtype', SequenceModelElement('rrtype', [
FixedDataModelElement('rrtype_str', b'","rrtype":"'),
DelimitedDataModelElement('rrtype', b'"')])),
FixedDataModelElement('quote', b'"'),
OptionalMatchModelElement(
'tx_id', SequenceModelElement('tx_id', [
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id')])),
OptionalMatchModelElement('ttl', SequenceModelElement('ttl', [
FixedDataModelElement('ttl_str', b',"ttl":'),
DecimalIntegerValueModelElement('ttl')])),
OptionalMatchModelElement(
'rdata', SequenceModelElement('rdata', [
FixedDataModelElement('rdata_str', b',"rdata":"'),
DelimitedDataModelElement('rdata', b'"'),
FixedDataModelElement('quote_str', b'"')])),
FixedDataModelElement('brack_str', b'}}')
]),
]),
SequenceModelElement('flow', [
FixedDataModelElement('flow_str', b'flow",'),
conn,
OptionalMatchModelElement(
'app_proto', SequenceModelElement('app_proto', [
FixedDataModelElement('app_proto_str', b',"app_proto":"'),
DelimitedDataModelElement('app_proto', b'"'),
FixedDataModelElement('quote_str', b'"')
])
),
OptionalMatchModelElement(
'app_proto_tc', SequenceModelElement('app_proto_tc', [
FixedDataModelElement('app_proto_tc_str', b',"app_proto_tc":"'),
DelimitedDataModelElement('app_proto_tc', b'"'),
FixedDataModelElement('quote_str', b'"')
])
),
SequenceModelElement('flow', [
FixedDataModelElement('pkts_toserver_str', b',"flow":{"pkts_toserver":'),
DecimalIntegerValueModelElement('pkts_toserver'),
FixedDataModelElement('pkts_toclient_str', b',"pkts_toclient":'),
DecimalIntegerValueModelElement('pkts_toclient'),
FixedDataModelElement('bytes_toserver_str', b',"bytes_toserver":'),
DecimalIntegerValueModelElement('bytes_toserver'),
FixedDataModelElement('bytes_toclient_str', b',"bytes_toclient":'),
DecimalIntegerValueModelElement('bytes_toclient'),
FixedDataModelElement('start_str', b',"start":"'),
DelimitedDataModelElement('start', b'"'),
FixedDataModelElement('end_str', b'","end":"'),
DelimitedDataModelElement('end', b'"'),
FixedDataModelElement('age_str', b'","age":'),
DecimalIntegerValueModelElement('age'),
FixedDataModelElement('state_str', b',"state":"'),
DelimitedDataModelElement('state', b'"'),
FixedDataModelElement('reason_str', b'","reason":"'),
DelimitedDataModelElement('reason', b'"'),
FixedDataModelElement('alerted_str', b'","alerted":'),
FixedWordlistDataModelElement('alerted', [b'true', b'false']),
FixedDataModelElement('brack_str1', b'}'),
OptionalMatchModelElement(
'tcp', SequenceModelElement('tcp', [
FixedDataModelElement('tcp_flags_str', b',"tcp":{"tcp_flags":"'),
HexStringModelElement('tcp_flags'),
FixedDataModelElement('tcp_flags_ts_str', b'","tcp_flags_ts":"'),
HexStringModelElement('tcp_flags_ts'),
FixedDataModelElement('tcp_flags_tc_str', b'","tcp_flags_tc":"'),
HexStringModelElement('tcp_flags_tc'),
OptionalMatchModelElement(
'flags', SequenceModelElement('flags', [
FixedDataModelElement('syn_str', b'","syn":'),
FixedWordlistDataModelElement('syn', [b'true', b'false']),
OptionalMatchModelElement(
'fin', SequenceModelElement('fin', [
FixedDataModelElement('fin_str', b',"fin":'),
FixedWordlistDataModelElement('fin', [b'true', b'false']),
])
),
OptionalMatchModelElement(
'rst', SequenceModelElement('rst', [
FixedDataModelElement('rst_str', b',"rst":'),
FixedWordlistDataModelElement('rst', [b'true', b'false']),
])
),
OptionalMatchModelElement(
'psh', SequenceModelElement('psh', [
FixedDataModelElement('psh_str', b',"psh":'),
FixedWordlistDataModelElement('psh', [b'true', b'false']),
])
),
FixedDataModelElement('ack_str', b',"ack":'),
FixedWordlistDataModelElement('ack', [b'true', b'false']),
FixedDataModelElement('tcp_state_str', b',"state":"'),
DelimitedDataModelElement('tcp_state', b'"'),
])
),
FixedDataModelElement('tcp_brack_str', b'"}'),
])
),
FixedDataModelElement('brack_str2', b'}')
]),
]),
SequenceModelElement('http', [
FixedDataModelElement('http_str', b'http",'),
conn,
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
http,
FixedDataModelElement('brack_str', b'}')
]),
SequenceModelElement('fileinfo', [
FixedDataModelElement('fileinfo_str', b'fileinfo",'),
conn,
http,
FixedDataModelElement('app_proto_str', b',"app_proto":"'),
DelimitedDataModelElement('app_proto', b'"'),
SequenceModelElement('fileinfo', [
FixedDataModelElement('fileinfo_str', b'","fileinfo":{'),
OptionalMatchModelElement(
'filename', SequenceModelElement('filename', [
FixedDataModelElement('filename_str', b'"filename":"'),
DelimitedDataModelElement('filename', b'"'),
FixedDataModelElement('quote_str', b'",')
])
),
FixedDataModelElement('state_str', b'"state":"'),
DelimitedDataModelElement('state', b'"'),
FixedDataModelElement('stored_str', b'","stored":'),
FixedWordlistDataModelElement('stored', [b'true', b'false']),
FixedDataModelElement('size_str', b',"size":'),
DecimalIntegerValueModelElement('size'),
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
FixedDataModelElement('brack_str', b'}}')
]),
]),
SequenceModelElement('stats', [
FixedDataModelElement('stats_str', b'stats",'),
FixedDataModelElement('uptime_str', b'"stats":{"uptime":'),
DecimalIntegerValueModelElement('uptime'),
SequenceModelElement('capture', [
FixedDataModelElement('capture_str', b',"capture":{'),
FixedDataModelElement('kernel_packets_str', b'"kernel_packets":'),
DecimalIntegerValueModelElement('kernel_packets'),
FixedDataModelElement('kernel_drops_str', b',"kernel_drops":'),
DecimalIntegerValueModelElement('kernel_drops'),
FixedDataModelElement('brack_str', b'}')
]),
SequenceModelElement('decoder', [
FixedDataModelElement('pkts_str', b',"decoder":{"pkts":'),
DecimalIntegerValueModelElement('pkts'),
FixedDataModelElement('bytes_str', b',"bytes":'),
DecimalIntegerValueModelElement('bytes'),
FixedDataModelElement('invalid_str', b',"invalid":'),
DecimalIntegerValueModelElement('invalid'),
FixedDataModelElement('ipv4_str', b',"ipv4":'),
DecimalIntegerValueModelElement('ipv4'),
FixedDataModelElement('ipv6_str', b',"ipv6":'),
DecimalIntegerValueModelElement('ipv6'),
FixedDataModelElement('ethernet_str', b',"ethernet":'),
DecimalIntegerValueModelElement('ethernet'),
FixedDataModelElement('raw_str', b',"raw":'),
DecimalIntegerValueModelElement('raw'),
FixedDataModelElement('null_str', b',"null":'),
DecimalIntegerValueModelElement('null'),
FixedDataModelElement('sll_str', b',"sll":'),
DecimalIntegerValueModelElement('sll'),
FixedDataModelElement('tcp_str', b',"tcp":'),
DecimalIntegerValueModelElement('tcp'),
FixedDataModelElement('udp_str', b',"udp":'),
DecimalIntegerValueModelElement('udp'),
FixedDataModelElement('sctp_str', b',"sctp":'),
DecimalIntegerValueModelElement('sctp'),
FixedDataModelElement('icmpv4_str', b',"icmpv4":'),
DecimalIntegerValueModelElement('icmpv4'),
FixedDataModelElement('icmpv6_str', b',"icmpv6":'),
DecimalIntegerValueModelElement('icmpv6'),
FixedDataModelElement('ppp_str', b',"ppp":'),
DecimalIntegerValueModelElement('ppp'),
FixedDataModelElement('pppoe_str', b',"pppoe":'),
DecimalIntegerValueModelElement('pppoe'),
FixedDataModelElement('gre_str', b',"gre":'),
DecimalIntegerValueModelElement('gre'),
FixedDataModelElement('vlan_str', b',"vlan":'),
DecimalIntegerValueModelElement('vlan'),
FixedDataModelElement('vlan_qinq_str', b',"vlan_qinq":'),
DecimalIntegerValueModelElement('vlan_qinq'),
FixedDataModelElement('teredo_str', b',"teredo":'),
DecimalIntegerValueModelElement('teredo'),
FixedDataModelElement('ipv4_in_ipv6_str', b',"ipv4_in_ipv6":'),
DecimalIntegerValueModelElement('ipv4_in_ipv6'),
FixedDataModelElement('ipv6_in_ipv6_str', b',"ipv6_in_ipv6":'),
DecimalIntegerValueModelElement('ipv6_in_ipv6'),
FixedDataModelElement('mpls_str', b',"mpls":'),
DecimalIntegerValueModelElement('mpls'),
FixedDataModelElement('avg_pkt_size_str', b',"avg_pkt_size":'),
DecimalIntegerValueModelElement('avg_pkt_size'),
FixedDataModelElement('max_pkt_size_str', b',"max_pkt_size":'),
DecimalIntegerValueModelElement('max_pkt_size'),
FixedDataModelElement('erspan_str', b',"erspan":'),
DecimalIntegerValueModelElement('erspan'),
SequenceModelElement('ipraw', [
FixedDataModelElement('invalid_ip_version_str', b',"ipraw":{"invalid_ip_version":'),
DecimalIntegerValueModelElement('invalid_ip_version'),
]),
SequenceModelElement('ltnull', [
FixedDataModelElement('ipraw_pkt_too_small_str', b'},"ltnull":{"pkt_too_small":'),
DecimalIntegerValueModelElement('ipraw_pkt_too_small'),
FixedDataModelElement('unsupported_type', b',"unsupported_type":'),
DecimalIntegerValueModelElement('unsupported_type'),
]),
SequenceModelElement('dce', [
FixedDataModelElement('dce_pkt_too_small_str', b'},"dce":{"pkt_too_small":'),
DecimalIntegerValueModelElement('dce_pkt_too_small'),
FixedDataModelElement('brack_str', b'}')
])
]),
SequenceModelElement('flow', [
FixedDataModelElement('memcap_str', b'},"flow":{"memcap":'),
DecimalIntegerValueModelElement('memcap'),
FixedDataModelElement('spare_str', b',"spare":'),
DecimalIntegerValueModelElement('spare'),
FixedDataModelElement('emerg_mode_entered_str', b',"emerg_mode_entered":'),
DecimalIntegerValueModelElement('emerg_mode_entered'),
FixedDataModelElement('emerg_mode_over_str', b',"emerg_mode_over":'),
DecimalIntegerValueModelElement('emerg_mode_over'),
FixedDataModelElement('tcp_reuse_str', b',"tcp_reuse":'),
DecimalIntegerValueModelElement('tcp_reuse'),
FixedDataModelElement('memuse_str', b',"memuse":'),
DecimalIntegerValueModelElement('memuse'),
]),
SequenceModelElement('defrag', [
SequenceModelElement('ipv4', [
FixedDataModelElement('fragments_str', b'},"defrag":{"ipv4":{"fragments":'),
DecimalIntegerValueModelElement('fragments'),
FixedDataModelElement('reassembled_str', b',"reassembled":'),
DecimalIntegerValueModelElement('reassembled_str'),
FixedDataModelElement('timeouts_str', b',"timeouts":'),
DecimalIntegerValueModelElement('timeouts'),
]),
SequenceModelElement('ipv6', [
FixedDataModelElement('fragments_str', b'},"ipv6":{"fragments":'),
DecimalIntegerValueModelElement('fragments'),
FixedDataModelElement('reassembled_str', b',"reassembled":'),
DecimalIntegerValueModelElement('reassembled_str'),
FixedDataModelElement('timeouts_str', b',"timeouts":'),
DecimalIntegerValueModelElement('timeouts'),
]),
FixedDataModelElement('max_frag_hits_str', b'},"max_frag_hits":'),
DecimalIntegerValueModelElement('max_frag_hits'),
]),
SequenceModelElement('tcp', [
FixedDataModelElement('sessions_str', b'},"tcp":{"sessions":'),
DecimalIntegerValueModelElement('sessions'),
FixedDataModelElement('ssn_memcap_drop_str', b',"ssn_memcap_drop":'),
DecimalIntegerValueModelElement('ssn_memcap_drop'),
FixedDataModelElement('pseudo_str', b',"pseudo":'),
DecimalIntegerValueModelElement('pseudo'),
FixedDataModelElement('pseudo_failed_str', b',"pseudo_failed":'),
DecimalIntegerValueModelElement('pseudo_failed'),
FixedDataModelElement('invalid_checksum_str', b',"invalid_checksum":'),
DecimalIntegerValueModelElement('invalid_checksum'),
FixedDataModelElement('no_flow_str', b',"no_flow":'),
DecimalIntegerValueModelElement('no_flow'),
FixedDataModelElement('syn_str', b',"syn":'),
DecimalIntegerValueModelElement('syn'),
FixedDataModelElement('synack_str', b',"synack":'),
DecimalIntegerValueModelElement('synack'),
FixedDataModelElement('rst_str', b',"rst":'),
DecimalIntegerValueModelElement('rst'),
FixedDataModelElement('segment_memcap_drop_str', b',"segment_memcap_drop":'),
DecimalIntegerValueModelElement('segment_memcap_drop'),
FixedDataModelElement('stream_depth_reached_str', b',"stream_depth_reached":'),
DecimalIntegerValueModelElement('stream_depth_reached'),
FixedDataModelElement('reassembly_gap_str', b',"reassembly_gap":'),
DecimalIntegerValueModelElement('reassembly_gap'),
FixedDataModelElement('memuse_str', b',"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('reassembly_memuse_str', b',"reassembly_memuse":'),
DecimalIntegerValueModelElement('reassembly_memuse'),
]),
SequenceModelElement('detect', [
FixedDataModelElement('alert_str', b'},"detect":{"alert":'),
DecimalIntegerValueModelElement('alert')
]),
SequenceModelElement('app_layer', [
SequenceModelElement('flow', [
FixedDataModelElement('http_str', b'},"app_layer":{"flow":{"http":'),
DecimalIntegerValueModelElement('http'),
FixedDataModelElement('ftp_str', b',"ftp":'),
DecimalIntegerValueModelElement('ftp'),
FixedDataModelElement('smtp_str', b',"smtp":'),
DecimalIntegerValueModelElement('smtp'),
FixedDataModelElement('tls_str', b',"tls":'),
DecimalIntegerValueModelElement('tls'),
FixedDataModelElement('ssh_str', b',"ssh":'),
DecimalIntegerValueModelElement('ssh'),
FixedDataModelElement('imap_str', b',"imap":'),
DecimalIntegerValueModelElement('imap'),
FixedDataModelElement('msn_str', b',"msn":'),
DecimalIntegerValueModelElement('msn'),
FixedDataModelElement('smb_str', b',"smb":'),
DecimalIntegerValueModelElement('smb'),
FixedDataModelElement('dcerpc_tcp_str', b',"dcerpc_tcp":'),
DecimalIntegerValueModelElement('dcerpc_tcp'),
FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'),
DecimalIntegerValueModelElement('dns_tcp'),
FixedDataModelElement('failed_tcp_str', b',"failed_tcp":'),
DecimalIntegerValueModelElement('failed_tcp'),
FixedDataModelElement('dcerpc_udp_str', b',"dcerpc_udp":'),
DecimalIntegerValueModelElement('dcerpc_udp'),
FixedDataModelElement('dns_udp_str', b',"dns_udp":'),
DecimalIntegerValueModelElement('dns_udp'),
FixedDataModelElement('failed_udp_str', b',"failed_udp":'),
DecimalIntegerValueModelElement('failed_udp'),
]),
SequenceModelElement('tx', [
FixedDataModelElement('http_str', b'},"tx":{"http":'),
DecimalIntegerValueModelElement('http'),
FixedDataModelElement('smtp_str', b',"smtp":'),
DecimalIntegerValueModelElement('smtp'),
FixedDataModelElement('tls_str', b',"tls":'),
DecimalIntegerValueModelElement('tls'),
FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'),
DecimalIntegerValueModelElement('dns_tcp'),
FixedDataModelElement('dns_udp_str', b',"dns_udp":'),
DecimalIntegerValueModelElement('dns_udp'),
])
]),
SequenceModelElement('flow_mgr', [
FixedDataModelElement('closed_pruned_str', b'}},"flow_mgr":{"closed_pruned":'),
DecimalIntegerValueModelElement('closed_pruned'),
FixedDataModelElement('new_pruned_str', b',"new_pruned":'),
DecimalIntegerValueModelElement('new_pruned'),
FixedDataModelElement('est_pruned_str', b',"est_pruned":'),
DecimalIntegerValueModelElement('est_pruned'),
FixedDataModelElement('bypassed_pruned_str', b',"bypassed_pruned":'),
DecimalIntegerValueModelElement('bypassed_pruned'),
FixedDataModelElement('flows_checked_str', b',"flows_checked":'),
DecimalIntegerValueModelElement('flows_checked'),
FixedDataModelElement('flows_notimeout_str', b',"flows_notimeout":'),
DecimalIntegerValueModelElement('flows_notimeout'),
FixedDataModelElement('flows_timeout_str', b',"flows_timeout":'),
DecimalIntegerValueModelElement('flows_timeout'),
FixedDataModelElement('flows_timeout_inuse_str', b',"flows_timeout_inuse":'),
DecimalIntegerValueModelElement('flows_timeout_inuse'),
FixedDataModelElement('flows_removed_str', b',"flows_removed":'),
DecimalIntegerValueModelElement('flows_removed'),
FixedDataModelElement('rows_checked_str', b',"rows_checked":'),
DecimalIntegerValueModelElement('rows_checked'),
FixedDataModelElement('rows_skipped_str', b',"rows_skipped":'),
DecimalIntegerValueModelElement('rows_skipped'),
FixedDataModelElement('rows_empty_str', b',"rows_empty":'),
DecimalIntegerValueModelElement('rows_empty'),
FixedDataModelElement('rows_busy_str', b',"rows_busy":'),
DecimalIntegerValueModelElement('rows_busy'),
FixedDataModelElement('rows_maxlen_str', b',"rows_maxlen":'),
DecimalIntegerValueModelElement('rows_maxlen'),
]),
SequenceModelElement('dns', [
FixedDataModelElement('memuse_str', b'},"dns":{"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('memcap_state_str', b',"memcap_state":'),
DecimalIntegerValueModelElement('memcap_state'),
FixedDataModelElement('memcap_global_str', b',"memcap_global":'),
DecimalIntegerValueModelElement('memcap_global'),
]),
SequenceModelElement('http', [
FixedDataModelElement('memuse_str', b'},"http":{"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('memcap_str', b',"memcap":'),
DecimalIntegerValueModelElement('memcap'),
]),
FixedDataModelElement('quote_str', b'}}}')
]),
SequenceModelElement('tls', [
FixedDataModelElement('tls_str', b'tls",'),
conn,
SequenceModelElement('tls', [
FixedDataModelElement('subject_str', b',"tls":{"subject":"'),
DelimitedDataModelElement('subject', b'"'),
FixedDataModelElement('issuerdn_str', b'","issuerdn":"'),
DelimitedDataModelElement('issuerdn', b'"'),
FixedDataModelElement('fingerprint_str', b'","fingerprint":"'),
DelimitedDataModelElement('fingerprint', b'"'),
OptionalMatchModelElement(
'sni', SequenceModelElement('sni', [
FixedDataModelElement('sni_str', b'","sni":"'),
DelimitedDataModelElement('sni', b'"'),
])
),
FixedDataModelElement('version_str', b'","version":"'),
DelimitedDataModelElement('version', b'"'),
FixedDataModelElement('notbefore_str', b'","notbefore":"'),
DelimitedDataModelElement('notbefore', b'"'),
FixedDataModelElement('notafter_str', b'","notafter":"'),
DelimitedDataModelElement('notafter', b'"'),
]),
FixedDataModelElement('brack_str', b'"}}')
]),
SequenceModelElement('alert', [
FixedDataModelElement('alert_str', b'alert",'),
conn,
OptionalMatchModelElement(
'tx_id', SequenceModelElement('tx_id', [
FixedDataModelElement('tx_id', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
])),
SequenceModelElement('alert', [
FixedDataModelElement('action_str', b',"alert":{"action":"'),
DelimitedDataModelElement('action', b'"'),
FixedDataModelElement('gid_str', b'","gid":'),
DecimalIntegerValueModelElement('gid'),
FixedDataModelElement('signature_id_str', b',"signature_id":'),
DecimalIntegerValueModelElement('signature_id'),
FixedDataModelElement('rev_str', b',"rev":'),
DecimalIntegerValueModelElement('rev'),
FixedDataModelElement('signature_str', b',"signature":"'),
DelimitedDataModelElement('signature', b'"'),
FixedDataModelElement('category_str', b'","category":"'),
DelimitedDataModelElement('category', b'"'),
FixedDataModelElement('severity_str', b'","severity":'),
DecimalIntegerValueModelElement('severity'),
FixedDataModelElement('brack_str', b'}')
]),
http,
FixedDataModelElement('brack_str', b'}')
]),
])
])
return model
def build_analysis_pipeline(analysis_context):
"""Define the function to create pipeline for parsing the log
data. It has also to define an AtomizerFactory to instruct AMiner
how to process incoming data streams to create log atoms from
them."""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('Fixed Workload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Start Time', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', "de_AT.utf8", None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
AnyByteDataModelElement('AnyByteDataModelElement')
])))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers
# later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler
syslog_event_handler = SyslogWriterEventHandler(analysis_context)
from aminer.events import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers = [
stream_printer_event_handler, syslog_event_handler,
mail_notification_handler
]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.input import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis import WhitelistViolationDetector
whitelist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
whitelist_violation_detector = WhitelistViolationDetector(
analysis_context.aminer_config, whitelist_rules,
anomaly_event_handlers)
analysis_context.register_component(whitelist_violation_detector,
component_name="Whitelist")
atom_filter.add_handler(whitelist_violation_detector)
from aminer.analysis import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list), None)
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config,
'/model/DiskReport/Space',
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
2,
1,
0,
anomaly_event_handlers,
record_count_before_event=70000)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct AMiner how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
from aminer.parsing import FirstMatchModelElement, SequenceModelElement, DecimalFloatValueModelElement, FixedDataModelElement, \
DelimitedDataModelElement, AnyByteDataModelElement, FixedWordlistDataModelElement, DecimalIntegerValueModelElement, \
DateTimeModelElement, IpAddressDataModelElement, Base64StringModelElement, ElementValueBranchModelElement, HexStringModelElement, \
MultiLocaleDateTimeModelElement, OptionalMatchModelElement, RepeatedElementDataModelElement, VariableByteDataModelElement, \
WhiteSpaceLimitedDataModelElement
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_audit = [
SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '),
FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'),
FixedDataModelElement('placeholder', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='),
DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'),
DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='),
DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='),
DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode'),
FixedDataModelElement('ouid_string', b' ouid='),
DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='),
DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='),
DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='),
FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])
]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '),
FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'),
FixedDataModelElement('placeholder', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='),
DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='),
DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='),
FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='),
DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')
])
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', '%s.%s' %
(loc), None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets to the
# AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
SequenceModelElement('', [
FixedDataModelElement('FixedDME', b'Any:'),
AnyByteDataModelElement('AnyByteDataModelElement')
])
])))
alphabet = b'abcdef'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
anomaly_event_handlers = [stream_printer_event_handler]
# Now define the AtomizerFactory using the model. A simple line
# based one is usually sufficient.
from aminer.input import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers,
default_timestamp_paths=["/model/DailyCron/DTM"])
# Just report all unparsed atoms to the event handlers.
from aminer.input import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(
anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(
verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis import AllowlistViolationDetector
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config,
allowlist_rules,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None,
anomaly_event_handlers, 10)
analysis_context.register_component(parser_count,
component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config,
anomaly_event_handlers)
analysis_context.register_component(etd,
component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config,
anomaly_event_handlers,
etd,
silence_output_except_indicator=False,
output_log_line=False)
analysis_context.register_component(vtd,
component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
etd,
disc_div_thres=0.5)
analysis_context.register_component(
vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
check_rules_flag=True,
hypothesis_max_delta_time=1.0,
auto_include_flag=True)
analysis_context.register_component(
ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config,
['/model/Random'],
anomaly_event_handlers,
target_value_list=[1, 10, 100],
output_log_line=True)
analysis_context.register_component(match_filter,
component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function,
output_log_line=True)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)],
10,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config,
'/model/RandomTime',
modulo_time_bin_definition,
10,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
None, ['/model/Random'],
100,
10,
output_log_line=True)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
output_log_line=True,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
analysis_context.aminer_config,
['/model/type/path/name', '/model/type/syscall/syscall'],
anomaly_event_handlers,
id_path_list=['/model/type/path/id', '/model/type/syscall/id'],
min_allowed_time_diff=5,
auto_include_flag=True,
allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(
new_match_id_value_combo_detector,
component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config,
'/model/DiskReport/Space',
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5,
output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=10000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config,
rules,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct AMiner how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Start Time', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
anomaly_event_handlers = [stream_printer_event_handler]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.input import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis import AllowlistViolationDetector
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/Job Number'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config,
'/model/DiskReport/Space',
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def get_model():
"""Return a model to parse Audit logs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
seq = [
FixedDataModelElement('audit_str', b'audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('pid_str', b'): pid='),
VariableByteDataModelElement('pid', alphabet),
FixedDataModelElement('uid_str', b' uid='),
VariableByteDataModelElement('uid', alphabet),
FixedDataModelElement('auid_str', b' auid='),
VariableByteDataModelElement('auid', alphabet),
FixedDataModelElement('ses_str', b' ses='),
VariableByteDataModelElement('ses', alphabet),
FixedDataModelElement('msg2_str', b' msg='),
VariableByteDataModelElement('msg2', alphabet),
FirstMatchModelElement('fm', [
SequenceModelElement('acct', [
FixedDataModelElement('acct_str', b' acct='),
VariableByteDataModelElement('acct', alphabet)
]),
SequenceModelElement('comm', [
FixedDataModelElement('comm_str', b' comm='),
VariableByteDataModelElement('comm', alphabet)
]),
SequenceModelElement('cmd', [
FixedDataModelElement('cmd_str', b' cmd='),
VariableByteDataModelElement('cmd', alphabet)
])
]),
OptionalMatchModelElement(
'opt',
SequenceModelElement('opt_seq', [
FixedDataModelElement('exe_str', b' exe='),
VariableByteDataModelElement('exe', alphabet),
FixedDataModelElement('hostname_str', b' hostname='),
VariableByteDataModelElement('hostname', alphabet),
FixedDataModelElement('addr_str', b' addr='),
VariableByteDataModelElement('addr', alphabet)
])),
FixedDataModelElement('terminal_str', b' terminal='),
VariableByteDataModelElement('terminal', alphabet),
FixedDataModelElement('res_str', b' res='),
VariableByteDataModelElement('res', alphabet)
]
model = SequenceModelElement('model', [
FixedDataModelElement('type_str', b'type='),
FirstMatchModelElement('type', [
SequenceModelElement('execve', [
FixedDataModelElement('execve_str', b'EXECVE msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('argc_str', b'): argc='),
DecimalIntegerValueModelElement(
'argc',
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL),
FixedDataModelElement('a0_str', b' a0='),
VariableByteDataModelElement('a0', alphabet),
OptionalMatchModelElement(
'opt1',
SequenceModelElement('seq1', [
FixedDataModelElement('a1_str', b' a1='),
VariableByteDataModelElement('a1', alphabet),
OptionalMatchModelElement(
'opt2',
SequenceModelElement('seq2', [
FixedDataModelElement('a2_str', b' a2='),
VariableByteDataModelElement('a2', alphabet),
OptionalMatchModelElement(
'opt3',
SequenceModelElement(
'seq3', [
FixedDataModelElement(
'a3_str', b' a3='),
VariableByteDataModelElement(
'a3', alphabet),
OptionalMatchModelElement(
'opt4',
SequenceModelElement(
'seq4', [
FixedDataModelElement(
'a4_str', b' a4='),
VariableByteDataModelElement(
'a4', alphabet)
])),
OptionalMatchModelElement(
'opt5',
SequenceModelElement(
'seq5', [
FixedDataModelElement(
'a5_str', b' a5='),
VariableByteDataModelElement(
'a5', alphabet)
])),
OptionalMatchModelElement(
'opt6',
SequenceModelElement(
'seq6', [
FixedDataModelElement(
'a6_str', b' a6='),
VariableByteDataModelElement(
'a6', alphabet)
])),
OptionalMatchModelElement(
'opt7',
SequenceModelElement(
'seq7', [
FixedDataModelElement(
'a7_str', b' a7='),
VariableByteDataModelElement(
'a7', alphabet)
])),
OptionalMatchModelElement(
'opt8',
SequenceModelElement(
'seq8', [
FixedDataModelElement(
'a8_str', b' a8='),
VariableByteDataModelElement(
'a8', alphabet)
])),
OptionalMatchModelElement(
'opt9',
SequenceModelElement(
'seq9', [
FixedDataModelElement(
'a9_str', b' a9='),
VariableByteDataModelElement(
'a9', alphabet)
])),
OptionalMatchModelElement(
'opt10',
SequenceModelElement(
'seq10', [
FixedDataModelElement(
'a10_str',
b' a10='),
VariableByteDataModelElement(
'a10', alphabet)
])),
OptionalMatchModelElement(
'opt11',
SequenceModelElement(
'seq11', [
FixedDataModelElement(
'a11_str',
b' a11='),
VariableByteDataModelElement(
'a11', alphabet)
])),
OptionalMatchModelElement(
'opt12',
SequenceModelElement(
'seq12', [
FixedDataModelElement(
'a12_str',
b' a12='),
VariableByteDataModelElement(
'a12', alphabet)
])),
OptionalMatchModelElement(
'opt13',
SequenceModelElement(
'seq13', [
FixedDataModelElement(
'a13_str',
b' a13='),
VariableByteDataModelElement(
'a13', alphabet)
])),
OptionalMatchModelElement(
'opt14',
SequenceModelElement(
'seq14', [
FixedDataModelElement(
'a14_str',
b' a14='),
VariableByteDataModelElement(
'a14', alphabet)
]))
]))
]))
]))
]),
SequenceModelElement('proctitle', [
FixedDataModelElement('type_str', b'PROCTITLE msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('proctitle_str', b'): proctitle='),
VariableByteDataModelElement('proctitle', alphabet)
]),
SequenceModelElement('syscall', [
FixedDataModelElement('msg_str', b'SYSCALL msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_str', b'): arch='),
VariableByteDataModelElement('arch', alphabet),
FixedDataModelElement('syscall_str', b' syscall='),
DecimalIntegerValueModelElement(
'syscall',
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL),
FixedDataModelElement('success_str', b' success='),
VariableByteDataModelElement('success', alphabet),
FixedDataModelElement('exit_str', b' exit='),
VariableByteDataModelElement('exit', alphabet),
FixedDataModelElement('a0_str', b' a0='),
VariableByteDataModelElement('a0', alphabet),
FixedDataModelElement('a1_str', b' a1='),
VariableByteDataModelElement('a1', alphabet),
FixedDataModelElement('a2_str', b' a2='),
VariableByteDataModelElement('a2', alphabet),
FixedDataModelElement('a3_str', b' a3='),
VariableByteDataModelElement('a3', alphabet),
FixedDataModelElement('items_str', b' items='),
VariableByteDataModelElement('items', alphabet),
FixedDataModelElement('ppid_str', b' ppid='),
VariableByteDataModelElement('ppid', alphabet),
FixedDataModelElement('pid_str', b' pid='),
VariableByteDataModelElement('pid', alphabet),
FixedDataModelElement('auid_str', b' auid='),
VariableByteDataModelElement('auid', alphabet),
FixedDataModelElement('uid_str', b' uid='),
VariableByteDataModelElement('uid', alphabet),
FixedDataModelElement('gid_str', b' gid='),
VariableByteDataModelElement('gid', alphabet),
FixedDataModelElement('euid_str', b' euid='),
VariableByteDataModelElement('euid', alphabet),
FixedDataModelElement('suid_str', b' suid='),
VariableByteDataModelElement('suid', alphabet),
FixedDataModelElement('fsuid_str', b' fsuid='),
VariableByteDataModelElement('fsuid', alphabet),
FixedDataModelElement('egid_str', b' egid='),
VariableByteDataModelElement('egid', alphabet),
FixedDataModelElement('sgid_str', b' sgid='),
VariableByteDataModelElement('sgid', alphabet),
FixedDataModelElement('fsgid_str', b' fsgid='),
VariableByteDataModelElement('fsgid', alphabet),
FixedDataModelElement('tty_str', b' tty='),
VariableByteDataModelElement('tty', alphabet),
FixedDataModelElement('ses_str', b' ses='),
VariableByteDataModelElement('ses', alphabet),
FixedDataModelElement('comm_str', b' comm='),
VariableByteDataModelElement('comm', alphabet),
FixedDataModelElement('exe_str', b' exe='),
VariableByteDataModelElement('exe', alphabet),
FixedDataModelElement('key_str', b' key='),
VariableByteDataModelElement('key', alphabet)
]),
SequenceModelElement('path', [
FixedDataModelElement('msg_str', b'PATH msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_str', b'): item='),
DecimalIntegerValueModelElement(
'item',
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL),
FixedDataModelElement('name_str', b' name='),
VariableByteDataModelElement('name', alphabet),
FirstMatchModelElement('path', [
SequenceModelElement('nametype', [
FixedDataModelElement('nametype_str', b' nametype='),
VariableByteDataModelElement('nametype', alphabet)
]),
SequenceModelElement('inode', [
FixedDataModelElement('inode_str', b' inode='),
DecimalIntegerValueModelElement(
'inode',
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL),
FixedDataModelElement('dev_str', b' dev='),
VariableByteDataModelElement('dev', alphabet),
FixedDataModelElement('mode_str', b' mode='),
VariableByteDataModelElement('mode', alphabet),
FixedDataModelElement('ouid_str', b' ouid='),
VariableByteDataModelElement('ouid', alphabet),
FixedDataModelElement('ogid_str', b' ogid='),
VariableByteDataModelElement('ogid', alphabet),
FixedDataModelElement('rdev_str', b' rdev='),
VariableByteDataModelElement('rdev', alphabet),
FixedDataModelElement('nametype_str', b' nametype='),
VariableByteDataModelElement('nametype', alphabet)
])
])
]),
SequenceModelElement('login', [
FixedDataModelElement('msg1_str', b'LOGIN msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('pid_str', b'): pid='),
VariableByteDataModelElement('pid', alphabet),
FixedDataModelElement('uid_str', b' uid='),
VariableByteDataModelElement('uid', alphabet),
FixedDataModelElement('old_auid_str', b' old-auid='),
VariableByteDataModelElement('old_auid', alphabet),
FixedDataModelElement('auid_str', b' auid='),
VariableByteDataModelElement('auid', alphabet),
OptionalMatchModelElement(
'tty',
SequenceModelElement('tty', [
FixedDataModelElement('tty_str', b' tty='),
VariableByteDataModelElement('tty', alphabet)
])),
FixedDataModelElement('old_ses_str', b' old-ses='),
VariableByteDataModelElement('old_ses', alphabet),
FixedDataModelElement('ses_str', b' ses='),
VariableByteDataModelElement('ses', alphabet),
FixedDataModelElement('res_str', b' res='),
VariableByteDataModelElement('res', alphabet)
]),
SequenceModelElement('sockaddr', [
FixedDataModelElement('msg_str', b'SOCKADDR msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('saddr_str', b'): saddr='),
VariableByteDataModelElement('saddr', alphabet)
]),
SequenceModelElement('unknown', [
FixedDataModelElement('unknwon_str', b'UNKNOWN['),
DecimalIntegerValueModelElement(
'unknown_id',
value_sign_type=DecimalIntegerValueModelElement.
SIGN_TYPE_OPTIONAL),
FixedDataModelElement('msg_str', b'] msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('proctitle_str', b'): proctitle='),
VariableByteDataModelElement('proctitle', alphabet)
]),
SequenceModelElement(
'cred_refr',
[FixedDataModelElement('msg1_str', b'CRED_REFR msg=')] + seq),
SequenceModelElement(
'user_start',
[FixedDataModelElement('msg1_str', b'USER_START msg=')] + seq),
SequenceModelElement(
'user_acct',
[FixedDataModelElement('msg1_str', b'USER_ACCT msg=')] + seq),
SequenceModelElement(
'user_auth',
[FixedDataModelElement('msg1_str', b'USER_AUTH msg=')] + seq),
SequenceModelElement(
'cred_disp',
[FixedDataModelElement('msg1_str', b'CRED_DISP msg=')] + seq),
SequenceModelElement(
'service_start',
[FixedDataModelElement('msg1_str', b'SERVICE_START msg=')] +
seq),
SequenceModelElement(
'service_stop',
[FixedDataModelElement('msg1_str', b'SERVICE_STOP msg=')] +
seq),
SequenceModelElement(
'user_end',
[FixedDataModelElement('msg1_str', b'USER_END msg=')] + seq),
SequenceModelElement(
'user_cmd',
[FixedDataModelElement('msg1_str', b'USER_CMD msg=')] + seq),
SequenceModelElement(
'cred_acq',
[FixedDataModelElement('msg1_str', b'CRED_ACQ msg=')] + seq),
SequenceModelElement('user_bprm_fcaps', [
FixedDataModelElement('msg1_str', b'BPRM_FCAPS msg=audit('),
DateTimeModelElement('time', b'%s.%f'),
FixedDataModelElement('colon_str', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('fver_str', b'): fver='),
VariableByteDataModelElement('fver', alphabet),
FixedDataModelElement('fp_str', b' fp='),
VariableByteDataModelElement('fp', alphabet),
FixedDataModelElement('fi_str', b' fi='),
VariableByteDataModelElement('fi', alphabet),
FixedDataModelElement('fe_str', b' fe='),
VariableByteDataModelElement('fe', alphabet),
FixedDataModelElement('old_pp_str', b' old_pp='),
VariableByteDataModelElement('old_pp', alphabet),
FixedDataModelElement('old_pi_str', b' old_pi='),
VariableByteDataModelElement('old_pi', alphabet),
FixedDataModelElement('old_pe_str', b' old_pe='),
VariableByteDataModelElement('old_pe', alphabet),
FixedDataModelElement('new_pp_str', b' new_pp='),
VariableByteDataModelElement('new_pp', alphabet),
FixedDataModelElement('new_pi_str', b' new_pi='),
VariableByteDataModelElement('new_pi', alphabet),
FixedDataModelElement('new_pe_str', b' new_pe='),
VariableByteDataModelElement('new_pe', alphabet)
])
])
])
return model
def get_model():
"""Return a model to parse Syslogs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
user_info = SequenceModelElement('user_info', [
FixedDataModelElement('user_str', b'user=<'),
OptionalMatchModelElement(
'user', DelimitedDataModelElement('user', b'>')
),
FixedDataModelElement('method_str', b'>'),
OptionalMatchModelElement(
'method', SequenceModelElement('method', [
FixedDataModelElement('method_str', b', method='),
DelimitedDataModelElement('method', b','),
])
),
FixedDataModelElement('rip_str', b', rip='),
IpAddressDataModelElement('rip'),
FixedDataModelElement('lip_str', b', lip='),
IpAddressDataModelElement('lip'),
OptionalMatchModelElement(
'mpid', SequenceModelElement('mpid', [
FixedDataModelElement('mpid_str', b', mpid='),
DecimalIntegerValueModelElement('mpid'),
])
),
OptionalMatchModelElement(
'secured', FixedDataModelElement('secured_str', b', secured')
),
FixedDataModelElement('session_str', b', session=<'),
DelimitedDataModelElement('session', b'>'),
FixedDataModelElement('bracket_str', b'>'),
])
model = SequenceModelElement('model', [
DateTimeModelElement('time', b'%b %d %H:%M:%S'),
FixedDataModelElement('sp1', b' '),
DelimitedDataModelElement('host', b' '),
FirstMatchModelElement('service', [
SequenceModelElement('dovecot', [
FixedDataModelElement('dovecot_str', b' dovecot: '),
FirstMatchModelElement('imap', [
SequenceModelElement('imap', [
FixedDataModelElement('imap_str', b'imap('),
DelimitedDataModelElement('user', b')'),
FixedDataModelElement('bracket_str', b'): '),
FirstMatchModelElement('message', [
SequenceModelElement('logout', [
FixedDataModelElement('logout_str', b'Logged out in='),
DecimalIntegerValueModelElement('in'),
FixedDataModelElement('out_str', b' out='),
DecimalIntegerValueModelElement('out')
]),
SequenceModelElement('err_mail', [
FixedDataModelElement('mail_str', b'Error: Failed to autocreate mailbox INBOX: Internal error occurred. '
b'Refer to server log for more information. ['),
DelimitedDataModelElement('err_time', b']'),
FixedDataModelElement('brack', b']')
]),
SequenceModelElement('err_open', [
FixedDataModelElement('err_str', b'Error: '),
DelimitedDataModelElement('function_name', b'('),
FixedDataModelElement('brack_str1', b'('),
DelimitedDataModelElement('arg', b')'),
FixedDataModelElement('failed_str', b') failed: Permission denied (euid='),
DecimalIntegerValueModelElement('euid'),
FixedDataModelElement('brack_str2', b'('),
DelimitedDataModelElement('euid_user', b')'),
FixedDataModelElement('egid_str', b') egid='),
DecimalIntegerValueModelElement('egid'),
FixedDataModelElement('brack_str3', b'('),
DelimitedDataModelElement('egid_user', b')'),
FixedDataModelElement('perm_str', b') missing +w perm: '),
DelimitedDataModelElement('mail_path', b','),
FixedDataModelElement('group_str', b', we\'re not in group '),
DecimalIntegerValueModelElement('group_id'),
FixedDataModelElement('brack_str4', b'('),
DelimitedDataModelElement('group_name', b')'),
FixedDataModelElement('owned_str', b'), dir owned by '),
DelimitedDataModelElement('owner', b' '),
FixedDataModelElement('mode_str', b' mode='),
DelimitedDataModelElement('mode', b')'),
FixedDataModelElement('brack_str5', b')'),
OptionalMatchModelElement(
'set', SequenceModelElement('set', [
FixedDataModelElement('set_str', b' (set'),
DelimitedDataModelElement('param', b'='),
FixedDataModelElement('equal_str', b'='),
DelimitedDataModelElement('val', b')'),
FixedDataModelElement('brack_str6', b')')
])
)
]),
SequenceModelElement('err_mail', [
FixedDataModelElement('mail_str', b'Failed to autocreate mailbox INBOX: Internal error occurred. '
b'Refer to server log for more information. ['),
DelimitedDataModelElement('err_time', b']'),
FixedDataModelElement('brack', b']')
]),
]),
]),
SequenceModelElement('imap_login', [
FixedDataModelElement('imap_login_str', b'imap-login: '******'login', [
SequenceModelElement('disconnected_str', [
FixedDataModelElement('disconnected_str', b'Disconnected '),
FirstMatchModelElement('auth', [
SequenceModelElement('auth_failed', [
FixedDataModelElement('auth_failed_str', b'(auth failed, '),
DecimalIntegerValueModelElement('attempts'),
FixedDataModelElement('attempts_str', b' attempts in '),
]),
FixedDataModelElement('no_auth_str', b'(no auth attempts in '),
FixedDataModelElement('no_auth_str', b'(disconnected before auth was ready, waited '),
]),
DecimalIntegerValueModelElement('duration'),
FixedDataModelElement('secs_str', b' secs): '),
user_info
]),
SequenceModelElement('login', [
FixedDataModelElement('login_str', b'Login: '******'anvil', [
FixedDataModelElement('anvil_str', b'Error: anvil:'),
AnyByteDataModelElement('anvil_msg')
]),
SequenceModelElement('auth_responding', [
FixedDataModelElement('auth_responding_str', b'Warning: Auth process not responding, '
b'delayed sending initial response (greeting): '),
user_info
]),
]),
]),
SequenceModelElement('auth', [
FixedDataModelElement('auth_worker_str', b'auth: '),
AnyByteDataModelElement('message')
]),
SequenceModelElement('auth_worker', [
FixedDataModelElement('auth_worker_str', b'auth-worker('),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('brack', b'):'),
AnyByteDataModelElement('message')
]),
SequenceModelElement('master', [
FixedDataModelElement('master_str', b'master: '),
AnyByteDataModelElement('message')
])
]),
]),
SequenceModelElement('horde', [
FixedDataModelElement('horde_str', b' HORDE: '),
FirstMatchModelElement('horde', [
SequenceModelElement('imp', [
FixedDataModelElement('succ_str', b'[imp] '),
FirstMatchModelElement('imp', [
SequenceModelElement('login', [
FixedDataModelElement('succ_str', b'Login success for '),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('brack_str1', b' ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement('to_str', b') to {'),
DelimitedDataModelElement('imap_addr', b'}'),
FixedDataModelElement('brack_str2', b'}'),
]),
SequenceModelElement('message_sent', [
FixedDataModelElement('message_sent_str', b'Message sent to '),
VariableByteDataModelElement('user', b'.0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'),
OptionalMatchModelElement(
'mail', SequenceModelElement('mail', [
FixedDataModelElement('brack_mail1', b' <'),
DelimitedDataModelElement('mail', b'>'),
FixedDataModelElement('brack_mail2', b'>')
])
),
RepeatedElementDataModelElement(
'more_recepients_rep', SequenceModelElement('more_recepients', [
FixedDataModelElement('comma_str', b', '),
VariableByteDataModelElement('more_recepients_mail',
b'.0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'),
OptionalMatchModelElement(
'more_recepients_mail', SequenceModelElement('more_recepients', [
FixedDataModelElement('brack_more_recepients1', b' <'),
DelimitedDataModelElement('more_recepients', b'>'),
FixedDataModelElement('brack_more_recepients2', b'>')
])
),
])
),
FixedDataModelElement('from_str', b' from '),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('brack_str1', b' ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement('brack_str2', b')'),
]),
SequenceModelElement('login_failed', [
FixedDataModelElement('succ_str', b'FAILED LOGIN for '),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('brack_str1', b' ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement('to_str', b') to {'),
DelimitedDataModelElement('imap_addr', b'}'),
FixedDataModelElement('brack_str2', b'}'),
]),
SequenceModelElement('status', [
FixedDataModelElement('status_str', b'[status] Could not open mailbox "INBOX".'),
]),
SequenceModelElement('sync_token', [
FixedDataModelElement('sync_token_str', b'[getSyncToken] IMAP error reported by server.'),
]),
SequenceModelElement('auth_failed', [
FixedDataModelElement('auth_failed_str', b'[login] Authentication failed.'),
]),
]),
]),
SequenceModelElement('horde', [
FixedDataModelElement('succ_str', b'[horde] '),
FirstMatchModelElement('horde', [
SequenceModelElement('success', [
FixedDataModelElement('success_str', b'Login success for '),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('brack_str1', b' to horde ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement('brack_str2', b')'),
]),
SequenceModelElement('success', [
FixedDataModelElement('success_str', b'User '),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('brack_str1', b' logged out of Horde ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement('brack_str2', b')'),
]),
SequenceModelElement('login_failed', [
FixedDataModelElement('failed_str', b'FAILED LOGIN for '),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('to_horde_str', b' to horde ('),
IpAddressDataModelElement('ip'),
FixedDataModelElement('brack_str', b')'),
]),
])
]),
SequenceModelElement('function', [
FixedWordlistDataModelElement('horde_function', [b'[nag]', b'[turba]', b'[horde]']),
FixedDataModelElement('nag_str', b' PHP ERROR: '),
FirstMatchModelElement('php_error', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function_name1', b'('),
FixedDataModelElement('brack_str1', b'('),
OptionalMatchModelElement(
'arg1', DelimitedDataModelElement('arg1', b')')
),
FixedDataModelElement('failed_str', b') should be compatible with '),
DelimitedDataModelElement('function_name2', b'('),
FixedDataModelElement('brack_str2', b'('),
OptionalMatchModelElement(
'arg2', DelimitedDataModelElement('arg2', b')')
),
FixedDataModelElement('brack_str3', b')'),
]),
FixedDataModelElement('file_str', b'finfo_file(): Empty filename or path'),
FixedDataModelElement('header_str', b'Cannot modify header information - headers already sent')
])
]),
SequenceModelElement('guest', [
FixedDataModelElement('guest_str', b'Guest user is not authorized for Horde (Host: '),
IpAddressDataModelElement('ip'),
FixedDataModelElement('brack_str', b').')
]),
SequenceModelElement('php_error', [
FixedDataModelElement('php_error_str', b'PHP ERROR: '),
DelimitedDataModelElement('msg', b' ['),
]),
SequenceModelElement('free_msg', [
DelimitedDataModelElement('msg', b' ['),
])
]),
FixedDataModelElement('to_str', b' [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('line_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('of_str', b' of "'),
DelimitedDataModelElement('path', b'"'),
FixedDataModelElement('brack_str', b'"]')
]),
SequenceModelElement('cron', [
FixedDataModelElement('cron_str', b' CRON['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('brack_str1', b']: '),
FirstMatchModelElement('cron', [
SequenceModelElement('cmd', [
FixedDataModelElement('brack_str', b'('),
DelimitedDataModelElement('user', b')'),
FixedDataModelElement('cmd_str', b') CMD '),
AnyByteDataModelElement('cmd_msg')
]),
SequenceModelElement('session', [ # This only occurs in auth.log
DelimitedDataModelElement('pam', b'('),
FixedDataModelElement('brack_str', b'('),
DelimitedDataModelElement('name', b')'),
FixedDataModelElement('session_str', b'): session '),
FixedWordlistDataModelElement('status', [b'opened', b'closed']),
FixedDataModelElement('user_str', b' for user '),
VariableByteDataModelElement('user', alphabet),
OptionalMatchModelElement(
'uid', SequenceModelElement('uid', [
FixedDataModelElement('uid_str', b' by (uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('brack_str', b')')
])
)
])
])
]),
SequenceModelElement('auth', [ # This only occurs in auth.log
FixedDataModelElement('auth_str', b' auth: '),
DelimitedDataModelElement('pam', b'('),
FixedDataModelElement('brack_str', b'('),
DelimitedDataModelElement('name', b')'),
FixedDataModelElement('session_str', b'): authentication failure; logname='),
OptionalMatchModelElement(
'logname', DelimitedDataModelElement('logname', b' ')
),
FixedDataModelElement('uid_str', b' uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('euid_str', b' euid='),
DecimalIntegerValueModelElement('euid'),
FixedDataModelElement('tty_str', b' tty='),
DelimitedDataModelElement('tty', b' '),
FixedDataModelElement('ruser_str', b' ruser='******'ruser', b' '),
FixedDataModelElement('rhost_str', b' rhost='),
IpAddressDataModelElement('rhost'),
OptionalMatchModelElement(
'user', SequenceModelElement('user', [
FixedDataModelElement('user_str', b' user='******'user', alphabet)
])
)
]),
SequenceModelElement('systemd', [
FixedDataModelElement('systemd_str', b' systemd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('brack_str1', b']: '),
FirstMatchModelElement('systemd', [
FixedDataModelElement('php_starting_str', b'Starting Clean php session files...'),
FixedDataModelElement('php_started_str', b'Started Clean php session files.'),
FixedDataModelElement('php_starting_str', b'Starting Cleanup of Temporary Directories...'),
FixedDataModelElement('php_started_str', b'Started Cleanup of Temporary Directories.'),
FixedDataModelElement('php_started_str', b'Starting Daily apt upgrade and clean activities...'),
FixedDataModelElement('php_started_str', b'Started Daily apt upgrade and clean activities.'),
FixedDataModelElement('php_started_str', b'Starting Daily apt download activities...'),
FixedDataModelElement('php_started_str', b'Started Daily apt download activities.'),
FixedDataModelElement('php_started_str', b'Starting Security Auditing Service...'),
FixedDataModelElement('php_started_str', b'Started Security Auditing Service.'),
FixedDataModelElement('php_started_str', b'Stopping Security Auditing Service...'),
FixedDataModelElement('php_started_str', b'Stopped Security Auditing Service.'),
FixedDataModelElement('php_started_str', b'Reloading The Apache HTTP Server.'),
FixedDataModelElement('php_started_str', b'Reloaded The Apache HTTP Server.'),
FixedDataModelElement('php_started_str', b'Mounting Arbitrary Executable File Formats File System...'),
FixedDataModelElement('php_started_str', b'Mounted Arbitrary Executable File Formats File System.'),
SequenceModelElement('apt', [
FixedDataModelElement('apt_str', b'apt'),
AnyByteDataModelElement('apt_msg')
]),
SequenceModelElement('service', [
DelimitedDataModelElement('service', b':'),
FixedDataModelElement('col_str', b': '),
AnyByteDataModelElement('_msg')
])
]),
]),
SequenceModelElement('kernel', [
FixedDataModelElement('kernel_str', b' kernel'),
OptionalMatchModelElement(
'id', SequenceModelElement('id', [
FixedDataModelElement('brack_str', b'['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str2', b']')
])
),
FixedDataModelElement('col_str', b': '),
AnyByteDataModelElement('kernel_msg')
]),
SequenceModelElement('augenrules', [
FixedDataModelElement('augenrules_str', b' augenrules['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
AnyByteDataModelElement('augenrules_msg')
]),
SequenceModelElement('auditd', [
FixedDataModelElement('auditd_str', b' auditd['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
AnyByteDataModelElement('auditd_msg')
]),
SequenceModelElement('auditd2', [
FixedDataModelElement('auditd2_str', b' auditd: '),
AnyByteDataModelElement('auditd_msg')
]),
SequenceModelElement('audispd', [
FixedDataModelElement('audispd_str', b' audispd: '),
AnyByteDataModelElement('audispd_msg')
]),
SequenceModelElement('liblogging', [
FixedDataModelElement('liblogging_str', b' liblogging-stdlog: '),
AnyByteDataModelElement('liblogging_msg')
]),
SequenceModelElement('freshclam', [
FixedDataModelElement('freshclam_str', b' freshclam['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
AnyByteDataModelElement('freshclam_msg')
]),
SequenceModelElement('dhclient', [
FixedDataModelElement('dhclient_str', b' dhclient['),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('brack_str1', b']: '),
FirstMatchModelElement('dhclient', [
SequenceModelElement('dhcprequest', [
FixedDataModelElement('dhcprequest_str', b'DHCPREQUEST of '),
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('on_str', b' on '),
DelimitedDataModelElement('network_interface', b' '),
FixedDataModelElement('to_str', b' to '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('port_str', b' port '),
DecimalIntegerValueModelElement('port')
]),
SequenceModelElement('dhcpack', [
FixedDataModelElement('dhcpack_str', b'DHCPACK of '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('on_str', b' from '),
IpAddressDataModelElement('src_ip')
]),
SequenceModelElement('bound', [
FixedDataModelElement('bound_str', b'bound to '),
IpAddressDataModelElement('ip'),
FixedDataModelElement('renewal_str', b' -- renewal in '),
DecimalIntegerValueModelElement('seconds'),
FixedDataModelElement('seconds_str', b' seconds.')
]),
]),
]),
])
])
return model
def get_model():
"""Return a model to parse Exim logs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
model = SequenceModelElement('model', [
DateTimeModelElement('time', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('sp', b' '),
FirstMatchModelElement('fm', [
SequenceModelElement('start', [
FixedDataModelElement('start', b'Start queue run: pid='),
DecimalIntegerValueModelElement('pid'),
]),
SequenceModelElement('end', [
FixedDataModelElement('end', b'End queue run: pid='),
DecimalIntegerValueModelElement('pid'),
]),
SequenceModelElement('no_host_found', [
FixedDataModelElement('no_host_found_str', b'no host name found for IP address '),
IpAddressDataModelElement('ip'),
]),
SequenceModelElement('vrfy_failed', [
FixedDataModelElement('vrfy_failed_str', b'VRFY failed for '),
DelimitedDataModelElement('mail', b' '),
FixedDataModelElement('h_str', b' H='),
DelimitedDataModelElement('h', b' '),
FixedDataModelElement('sp1', b' ['),
IpAddressDataModelElement('ip'),
FixedDataModelElement('sp2', b']')
]),
SequenceModelElement('mail', [
DelimitedDataModelElement('id', b' '),
FirstMatchModelElement('dir', [
SequenceModelElement('dir_in', [
FixedDataModelElement('in', b' <= '),
FirstMatchModelElement('fm', [
SequenceModelElement('seq1', [
FixedDataModelElement('brack', b'<> '),
FirstMatchModelElement('fm', [
SequenceModelElement('r', [
FixedDataModelElement('r_str', b'R='),
DelimitedDataModelElement('r', b' '),
FixedDataModelElement('u_str', b' U='),
DelimitedDataModelElement('u', b' '),
]),
SequenceModelElement('h', [
FixedDataModelElement('h_str', b'H='),
DelimitedDataModelElement('h', b' '),
FixedDataModelElement('sp1', b' ['),
IpAddressDataModelElement('ip'),
FixedDataModelElement('sp1', b']'),
])
]),
FixedDataModelElement('sp2', b' P='),
DelimitedDataModelElement('p', b' '),
FixedDataModelElement('sp2', b' S='),
DecimalIntegerValueModelElement('s'),
]),
SequenceModelElement('seq2', [
DelimitedDataModelElement('mail', b' '),
FixedDataModelElement('user_str', b' U='),
DelimitedDataModelElement('user', b' '),
FixedDataModelElement('p_str', b' P='),
DelimitedDataModelElement('p', b' '),
FixedDataModelElement('s_str', b' S='),
DecimalIntegerValueModelElement('s'),
OptionalMatchModelElement(
'id', SequenceModelElement('id', [
FixedDataModelElement('id_str', b' id='),
AnyByteDataModelElement('id')
])
)
])
])
]),
SequenceModelElement('dir_out', [
FixedDataModelElement('in', b' => '),
DelimitedDataModelElement('name', b' '),
FixedDataModelElement('sp1', b' '),
OptionalMatchModelElement(
'mail_opt', SequenceModelElement('mail', [
FixedDataModelElement('brack1', b'('),
DelimitedDataModelElement('brack_mail', b')'),
FixedDataModelElement('brack2', b') '),
])),
FixedDataModelElement('sp2', b'<'),
DelimitedDataModelElement('mail', b'>'),
FixedDataModelElement('r_str', b'> R='),
DelimitedDataModelElement('r', b' '),
FixedDataModelElement('t_str', b' T='),
VariableByteDataModelElement('t', alphabet),
]),
SequenceModelElement('aster', [
FixedDataModelElement('aster', b' ** '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('headers_str', b' Too many "Received" headers - suspected mail loop')]),
FixedDataModelElement('completed', b' Completed'),
FixedDataModelElement('frozen', b' Message is frozen'),
FixedDataModelElement('frozen', b' Frozen (delivery error message)')
])
])])])
return model