def run_time_correlation_detector(self, number_of_rules):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
time_correlation_detector = TimeCorrelationDetector(
self.aminer_config,
2,
number_of_rules,
0, [self.stream_printer_event_handler],
record_count_before_event=self.waiting_time * 9000)
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
time_correlation_detector)
time_correlation_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
time_correlation_detector.__class__.__name__, avg, results,
'testCount=%d.' % number_of_rules)
def get_model():
"""Return a model to parse a su session information message after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('gidchange', [
FixedDataModelElement('s0', b'rsyslogd\'s groupid changed to '),
DecimalIntegerValueModelElement('gid')
]),
SequenceModelElement('statechange', [
FixedDataModelElement('s0',
b'[origin software="rsyslogd" swVersion="'),
DelimitedDataModelElement('version', b'"'),
FixedDataModelElement('s1', b'" x-pid="'),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s2',
b'" x-info="http://www.rsyslog.com"] '),
FirstMatchModelElement('type', [
FixedDataModelElement('HUPed', b'rsyslogd was HUPed'),
FixedDataModelElement('start', b'start')
])
]),
SequenceModelElement('uidchange', [
FixedDataModelElement('s0', b'rsyslogd\'s userid changed to '),
DecimalIntegerValueModelElement('uid')
])
]
model = SequenceModelElement('rsyslog', [
FixedDataModelElement('sname', b'rsyslogd: '),
FirstMatchModelElement('msg', type_children)
])
return model
def run_new_match_path_detector(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
t = round(time.time(), 3)
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
new_match_path_detector)
new_match_path_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
new_match_path_detector.__class__.__name__, avg, results,
self.different_pathes % number_of_pathes)
def getLogindModel(userNameModel=None):
"""This function defines how to parse a systemd logind daemon
message after any standard logging preamble, e.g. from syslog."""
if userNameModel is None:
userNameModel = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz-')
typeChildren = []
# FIXME: Will fail on username models including the dot at the end.
typeChildren.append(
SequenceModelElement('new session', [
FixedDataModelElement('s0', b'New session '),
DecimalIntegerValueModelElement('session'),
FixedDataModelElement('s1', b' of user '), userNameModel,
FixedDataModelElement('s2', b'.')
]))
typeChildren.append(
SequenceModelElement('removed session', [
FixedDataModelElement('s0', b'Removed session '),
DecimalIntegerValueModelElement('session'),
FixedDataModelElement('s1', b'.')
]))
model = SequenceModelElement('systemd-logind', [
FixedDataModelElement('sname', b'systemd-logind['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', typeChildren)
])
return model
def get_systemd_model():
"""Return the parsing model for messages directly from systemd."""
type_children = [
FixedDataModelElement('apt-daily-start',
b'Starting Daily apt activities...'),
FixedDataModelElement('apt-daily-started',
b'Started Daily apt activities.'),
SequenceModelElement('apt-daily-timer', [
FixedDataModelElement('s0', b'apt-daily.timer: Adding '),
OptionalMatchModelElement(
'hopt',
SequenceModelElement('hblock', [
DecimalIntegerValueModelElement('hours'),
FixedDataModelElement('s1', b'h ')
])),
DecimalIntegerValueModelElement('minutes'),
FixedDataModelElement('s2', b'min '),
DecimalFloatValueModelElement('seconds'),
FixedDataModelElement('s3', b's random time.')
]),
FixedDataModelElement('tmp-file-cleanup',
b'Starting Cleanup of Temporary Directories...'),
FixedDataModelElement('tmp-file-cleanup-started',
b'Started Cleanup of Temporary Directories.')
]
model = SequenceModelElement('systemd', [
FixedDataModelElement('sname', b'systemd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return a model for su session information messages after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('build-stack', [
FixedDataModelElement('s0',
b'building new pluginstance stack: \''),
DelimitedDataModelElement('stack', b'\''),
FixedDataModelElement('s1', b'\'')
]),
SequenceModelElement(
'nfct-event',
[
FixedDataModelElement('s0', b'[DESTROY] ORIG: SRC='),
IpAddressDataModelElement('osrcip'),
FixedDataModelElement('s1', b' DST='),
IpAddressDataModelElement('odstip'),
FixedDataModelElement('s2', b' PROTO='),
FixedWordlistDataModelElement('proto', [b'TCP', b'UDP']),
FixedDataModelElement('s3', b' SPT='),
DecimalIntegerValueModelElement('ospt'),
FixedDataModelElement('s4', b' DPT='),
DecimalIntegerValueModelElement('odpt'),
FixedDataModelElement('s5', b' PKTS='),
DecimalIntegerValueModelElement('opkts'),
FixedDataModelElement('s6', b' BYTES='),
DecimalIntegerValueModelElement('obytes'),
FixedDataModelElement('s7', b' , REPLY: SRC='),
IpAddressDataModelElement('rsrcip'),
FixedDataModelElement('s8', b' DST='),
IpAddressDataModelElement('rdstip'),
FixedDataModelElement('s9', b' PROTO='),
FixedWordlistDataModelElement('rproto', [b'TCP', b'UDP']),
FixedDataModelElement('s10', b' SPT='),
DecimalIntegerValueModelElement('rspt'),
FixedDataModelElement('s11', b' DPT='),
DecimalIntegerValueModelElement('rdpt'),
FixedDataModelElement('s12', b' PKTS='),
DecimalIntegerValueModelElement('rpkts'),
FixedDataModelElement('s13', b' BYTES='),
DecimalIntegerValueModelElement('rbytes'),
# No additional whitespace from Ubuntu Trusty 14.04 on.
OptionalMatchModelElement('tail',
FixedDataModelElement('s0', b' '))
]),
FixedDataModelElement('nfct-plugin',
b'NFCT plugin working in event mode'),
FixedDataModelElement('reopen', b'reopening capture file'),
FixedDataModelElement('signal',
b'signal received, calling pluginstances'),
FixedDataModelElement('uidchange', b'Changing UID / GID')
]
# Netflow entry
model = SequenceModelElement('ulogd', [
FixedDataModelElement('sname', b'ulogd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
new_time_model = SequenceModelElement('time_model', [
DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S '),
FixedWordlistDataModelElement('sign', [b'+', b'-']),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('bracket', b']')
])
host_name_model = VariableByteDataModelElement(
'host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identity_model = VariableByteDataModelElement(
'ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
user_name_model = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
request_method_model = FixedWordlistDataModelElement(
'method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT',
b'OPTIONS', b'TRACE', b'PATCH'
])
request_model = VariableByteDataModelElement(
'request',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+'
)
version_model = VariableByteDataModelElement('version', b'0123456789.')
status_code_model = DecimalIntegerValueModelElement('status')
size_model = DecimalIntegerValueModelElement('size')
user_agent_model = VariableByteDataModelElement(
'useragent',
b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+;:_ '
)
whitespace_str = b' '
model = SequenceModelElement('accesslog', [
host_name_model,
FixedDataModelElement('sp0', whitespace_str),
identity_model,
FixedDataModelElement('sp1', whitespace_str),
user_name_model,
FixedDataModelElement('sp2', whitespace_str),
new_time_model,
FixedDataModelElement('sp3', b' "'),
request_method_model,
FixedDataModelElement('sp4', whitespace_str),
request_model,
FixedDataModelElement('sp5', b' HTTP/'),
version_model,
FixedDataModelElement('sp6', b'" '),
status_code_model,
FixedDataModelElement('sp7', whitespace_str),
size_model,
FixedDataModelElement('sp8', b' "-" "'),
user_agent_model,
FixedDataModelElement('sp9', b'"'),
])
return model
def get_model():
"""Get the model."""
interface_name_model = VariableByteDataModelElement('interface', b'0123456789abcdefghijklmnopqrstuvwxyz.')
type_children = [
SequenceModelElement('exit', [
FixedDataModelElement('s0', b'ntpd exiting on signal '),
DecimalIntegerValueModelElement('signal')
]),
SequenceModelElement('listen-drop', [
FixedDataModelElement('s0', b'Listen and drop on '),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' '),
interface_name_model,
FixedDataModelElement('s2', b' '),
FirstMatchModelElement('address', [
IpAddressDataModelElement('ipv4'),
DelimitedDataModelElement('ipv6', b' ')
]),
FixedDataModelElement('s3', b' UDP 123')
]),
SequenceModelElement('listen-normal', [
FixedDataModelElement('s0', b'Listen normally on '),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' '),
interface_name_model,
FixedDataModelElement('s2', b' '),
IpAddressDataModelElement('ip'),
FirstMatchModelElement('msg', [
FixedDataModelElement('port-new', b':123'),
FixedDataModelElement('port-old', b' UDP 123')
])
]),
SequenceModelElement('listen-routing', [
FixedDataModelElement('s0', b'Listening on routing socket on fd #'),
DecimalIntegerValueModelElement('fd'),
FixedDataModelElement('s1', b' for interface updates')
]),
FixedDataModelElement('new-interfaces', b'new interface(s) found: waking up resolver'),
FixedDataModelElement('ntp-io', b'ntp_io: estimated max descriptors: 1024, initial socket boundary: 16'),
FixedDataModelElement('peers-refreshed', b'peers refreshed'),
SequenceModelElement('precision', [
FixedDataModelElement('s0', b'proto: precision = '),
DecimalFloatValueModelElement('precision'),
FixedDataModelElement('s1', b' usec')])]
model = SequenceModelElement('ntpd', [
FixedDataModelElement('sname', b'ntpd['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def run_atom_filters_match_value_filter(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
subhandler_filter = SubhandlerFilter([],
stop_when_handled_flag=True)
i = 0
dictionary = {}
while i < 1000000:
dictionary[i] = new_match_path_detector
i = i + 1
i = 0
while i < number_of_pathes:
match_value_filter = MatchValueFilter(
self.integerd + str(i % number_of_pathes), dictionary,
None)
subhandler_filter.add_handler(match_value_filter,
stop_when_handled_flag=True)
i = i + 1
t = round(time.time(), 3)
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
match_value_filter)
subhandler_filter.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
subhandler_filter.__class__.__name__, avg, results,
'%d different %ss with a dictionary of %ss.' %
(number_of_pathes, match_value_filter.__class__.__name__,
new_match_path_detector.__class__.__name__))
def get_model():
"""This method returns the model."""
type_children = [
FixedDataModelElement(
'warn-no-openat',
b'WARNING: SECURITY: No secure open yet due to missing openat in python!'
),
FixedDataModelElement(
'warn-no-OPATH',
b'WARNING: SECURITY: Open should use O_PATH, but not yet available in python'
),
FixedDataModelElement(
'warn-POSIX-acls',
b'WARNING: SECURITY: No checking for backdoor access via \
POSIX ACLs, use "getfacl" from "acl" package to check manually.'),
FixedDataModelElement(
'warn-no-linkat',
b'WARNING: SECURITY: unsafe unlink (unavailable unlinkat/linkat \
should be used, but not available in python)'),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('aminer', [
FixedDataModelElement('sname', b'AMiner['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def run_whitelist_violation_detector(self, number_of_pathes,
modulo_factor):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
rules = []
while i < number_of_pathes:
rules.append(
PathExistsMatchRule(
self.integerd + str(i % number_of_pathes), None))
i = i + 1
whitelist_violation_detector = WhitelistViolationDetector(
self.aminer_config, rules, [self.stream_printer_event_handler])
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
p = process_time()
r = random.randint(1, 100)
if r >= modulo_factor:
r = 2
else:
r = 1
seconds = seconds + process_time() - p
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % (number_of_pathes * r)),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(i % 100).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), t,
whitelist_violation_detector)
whitelist_violation_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
whitelist_violation_detector.__class__.__name__, avg, results,
'%d different PathExistsMatchRules and a moduloFactor of %d.' %
(number_of_pathes, modulo_factor))
def getModel(userNameModel=None):
"""This function defines how to parse a su session information message
after any standard logging preamble, e.g. from syslog."""
if userNameModel is None:
userNameModel = VariableByteDataModelElement(
'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
srcUserNameModel = VariableByteDataModelElement('srcuser', \
b'0123456789abcdefghijklmnopqrstuvwxyz.-')
typeChildren = []
typeChildren.append(
SequenceModelElement('su-good', [
FixedDataModelElement('s0', b'Successful su for '), userNameModel,
FixedDataModelElement('s1', b' by '), srcUserNameModel
]))
typeChildren.append(
SequenceModelElement('su-good', [
FixedDataModelElement('s0', b'+ '),
DelimitedDataModelElement('terminal', b' '),
FixedDataModelElement('s1', b' '), srcUserNameModel,
FixedDataModelElement('s2', b':'), userNameModel
]))
typeChildren.append(SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(su:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '),
userNameModel,
OptionalMatchModelElement('openby', \
SequenceModelElement('userinfo', [
FixedDataModelElement('s0', b' by (uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s1', b')')]))
]))
model = SequenceModelElement('su', [
FixedDataModelElement('sname', b'su['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', typeChildren)
])
return model
def get_model():
"""Return a model to parse Apache Access logs from the AIT-LDS."""
alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]'
model = SequenceModelElement('model', [
FirstMatchModelElement('client_ip', [
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('localhost', b'::1')
]),
FixedDataModelElement('sp1', b' '),
VariableByteDataModelElement('client_id', alphabet),
FixedDataModelElement('sp2', b' '),
VariableByteDataModelElement('user_id', alphabet),
FixedDataModelElement('sp3', b' ['),
DateTimeModelElement('time', b'%d/%b/%Y:%H:%M:%S'),
FixedDataModelElement('sp4', b' +'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('sp5', b'] "'),
FirstMatchModelElement('fm', [
FixedDataModelElement('dash', b'-'),
SequenceModelElement('request', [
FixedWordlistDataModelElement('method', [
b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH']),
FixedDataModelElement('sp6', b' '),
DelimitedDataModelElement('request', b' ', b'\\'),
FixedDataModelElement('sp7', b' '),
DelimitedDataModelElement('version', b'"'),
])
]),
FixedDataModelElement('sp8', b'" '),
DecimalIntegerValueModelElement('status_code'),
FixedDataModelElement('sp9', b' '),
DecimalIntegerValueModelElement('content_size'),
OptionalMatchModelElement(
'combined', SequenceModelElement('combined', [
FixedDataModelElement('sp10', b' "'),
DelimitedDataModelElement('referer', b'"', b'\\'),
FixedDataModelElement('sp11', b'" "'),
DelimitedDataModelElement('user_agent', b'"', b'\\'),
FixedDataModelElement('sp12', b'"'),
])),
])
return model
def run_missing_match_path_value_detector(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
path_list = []
while i < number_of_pathes:
path_list.append(self.integerd + str(i % number_of_pathes))
i = i + 1
missing_match_path_list_value_detector = MissingMatchPathListValueDetector(
self.aminer_config, path_list,
[self.stream_printer_event_handler], 'Default', True, 3600,
86400)
seconds = time.time()
t = seconds
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context = MatchContext(str(1).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
p = process_time()
r = random.randint(0, 100)
seconds = seconds + process_time() - p
delta = int(r / 100)
t = t + 4000 * delta
log_atom = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
missing_match_path_list_value_detector)
missing_match_path_list_value_detector.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
missing_match_path_list_value_detector.__class__.__name__, avg,
results, self.different_pathes % number_of_pathes)
def get_model():
"""Return a model to parse Suricata Fast logs from the AIT-LDS."""
model = SequenceModelElement('model', [
DateTimeModelElement('time', b'%m/%d/%Y-%H:%M:%S.%f'),
FixedDataModelElement('brack_str1', b' [**] ['),
DecimalIntegerValueModelElement('id1'),
FixedDataModelElement('sep1', b':'),
DecimalIntegerValueModelElement('id2'),
FixedDataModelElement('sep2', b':'),
DecimalIntegerValueModelElement('id3'),
FixedDataModelElement('sep3', b'] '),
DelimitedDataModelElement('message', b' [**] '),
FixedDataModelElement('classification_str',
b' [**] [Classification: '),
DelimitedDataModelElement('classification', b']'),
FixedDataModelElement('priority_str', b'] [Priority: '),
DecimalIntegerValueModelElement('priority'),
FixedDataModelElement('brack_str1', b'] {'),
DelimitedDataModelElement('conn', b'}'),
FixedDataModelElement('brack_str2', b'} '),
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('arrow_str', b' -> '),
IpAddressDataModelElement('dst_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('dst_port'),
])
return model
class TimeCorrelationDetectorTest(TestBase):
__expected_string = '%s Correlation report\nTimeCorrelationDetector: "%s" (%d lines)\n '
string = b'25537 uid=2'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
fixed_dme = FixedDataModelElement('s1', string)
decimal_integer_value_me = DecimalIntegerValueModelElement('d1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_first_match_me = MatchContext(string)
first_match_me = FirstMatchModelElement('f1', [fixed_dme, decimal_integer_value_me])
match_element_first_match_me = first_match_me.get_match_element('first', match_context_first_match_me)
match_context_first_match_me2 = MatchContext(string)
first_match_me2 = FirstMatchModelElement('f2', [decimal_integer_value_me, fixed_dme])
match_element_first_match_me2 = first_match_me2.get_match_element('second', match_context_first_match_me2)
def test1_normal_report(self):
"""This test case unit the creation of a report. As the rules are chosen randomly this test can not be very specific in checking
the actual values of the report."""
description = "Test1TimeCorrelationDetector"
time_correlation_detector = TimeCorrelationDetector(self.aminer_config, 2, 1, 0, [self.stream_printer_event_handler],
record_count_before_event=10)
self.analysis_context.register_component(time_correlation_detector, component_name=description)
t = time.time()
for i in range(0, 10):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t).strftime(self.datetime_format_string), description, 10)))
self.reset_output_stream()
for i in range(0, 10):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t + 9).strftime(self.datetime_format_string), description, 20)))
self.reset_output_stream()
for i in range(10, 15):
logAtomSequenceME = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME)
logAtomSequenceME2 = LogAtom(self.fixed_dme.fixed_data, ParserMatch(self.match_element_first_match_me2), t + i,
time_correlation_detector)
time_correlation_detector.receive_atom(logAtomSequenceME2)
self.assertTrue(self.output_stream.getvalue().startswith(
self.__expected_string % (datetime.fromtimestamp(t + 14).strftime(self.datetime_format_string), description, 30)))
def get_model(user_name_model=None):
"""This function defines how to parse a cron message logged via syslog after any standard logging preamble, e.g. from syslog."""
if user_name_model is None:
user_name_model = VariableByteDataModelElement('user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
type_children = [
SequenceModelElement('exec', [
FixedDataModelElement('s0', b'('),
user_name_model,
FixedDataModelElement('s1', b') CMD '),
AnyByteDataModelElement('command')
]),
SequenceModelElement('pam', [
FixedDataModelElement('s0', b'pam_unix(cron:session): session '),
FixedWordlistDataModelElement('change', [b'opened', b'closed']),
FixedDataModelElement('s1', b' for user '),
user_name_model,
OptionalMatchModelElement('openby', FixedDataModelElement('default', b' by (uid=0)'))
])
]
model = FirstMatchModelElement('cron', [
SequenceModelElement('std', [
FixedDataModelElement('sname', b'CRON['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msgtype', type_children)
]),
SequenceModelElement('low', [
FixedDataModelElement('sname', b'cron['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: (*system*'),
DelimitedDataModelElement('rname', b') RELOAD ('),
FixedDataModelElement('s1', b') RELOAD ('),
DelimitedDataModelElement('fname', b')'),
FixedDataModelElement('s2', b')'), ])
])
return model
def getModel():
"""This function returns the model."""
typeChildren = []
typeChildren.append(SequenceModelElement('sent', [
FixedDataModelElement('s0', b'Sent mail for '),
DelimitedDataModelElement('to-addr', b' ('),
FixedDataModelElement('s1', b' ('),
DelimitedDataModelElement('status', b') uid='),
FixedDataModelElement('s2', b') uid='),
DecimalIntegerValueModelElement('uid'),
FixedDataModelElement('s3', b' username='******'username', b' outbytes='),
FixedDataModelElement('s4', b' outbytes='),
DecimalIntegerValueModelElement('bytes'),
]))
model = SequenceModelElement('ssmtp', [
FixedDataModelElement('sname', b'sSMTP['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', typeChildren)])
return model
def run_timestamps_unsorted_detector(self, reset_factor):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
timestamps_unsorted_detector = TimestampsUnsortedDetector(
self.aminer_config, [self.stream_printer_event_handler])
seconds = time.time()
s = seconds
i = 0
mini = 100
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
p = process_time()
r = random.randint(1, 100)
seconds = seconds + process_time() - p
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element),
s + min(r, mini),
timestamps_unsorted_detector)
timestamps_unsorted_detector.receive_atom(log_atom)
if mini > r:
mini = r
else:
mini = mini + reset_factor
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
timestamps_unsorted_detector.__class__.__name__, avg, results,
'a resetFactor of %f.' % reset_factor)
def getModel(timeModel=None):
timeModel = DateTimeModelElement('time', b'[%d/%b/%Y:%H:%M:%S +0000]')
hostNameModel = VariableByteDataModelElement('host', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
identityModel = VariableByteDataModelElement('ident', b'-.01234567890abcdefghijklmnopqrstuvwxyz:')
userNameModel = VariableByteDataModelElement('user', b'0123456789abcdefghijklmnopqrstuvwxyz.-')
requestMethodModel = FixedWordlistDataModelElement('method', [b'GET', b'POST', b'PUT', b'HEAD', b'DELETE', b'CONNECT', b'OPTIONS', b'TRACE', b'PATCH'])
requestModel = VariableByteDataModelElement('request', b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-/()[]{}!$%&=<?*+')
versionModel = VariableByteDataModelElement('version', b'0123456789.')
statuscodeModel = DecimalIntegerValueModelElement('status')
sizeModel = DecimalIntegerValueModelElement('size')
useragentModel = VariableByteDataModelElement('useragent', b'0123456789abcdefghijklmnopqrstuvwxyz.-/()[]{}!$%&=<?*+')
model = SequenceModelElement('accesslog', [
hostNameModel,
FixedDataModelElement('sp0', b' '),
identityModel,
FixedDataModelElement('sp1', b' '),
userNameModel,
FixedDataModelElement('sp2', b' '),
timeModel,
FixedDataModelElement('sp3', b' "'),
requestMethodModel,
FixedDataModelElement('sp4', b' '),
requestModel,
FixedDataModelElement('sp5', b' HTTP/'),
versionModel,
# AnyByteDataModelElement('any')
FixedDataModelElement('sp6', b'" '),
statuscodeModel,
FixedDataModelElement('sp7', b' '),
sizeModel,
FixedDataModelElement('sp8', b' "-" "'),
useragentModel,
FixedDataModelElement('sp9', b'"'),
])
return model
def run_match_value_stream_writer(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
i = 0
path_list = []
parsing_model = []
while i < number_of_pathes / 2:
path_list.append('match/integer/d' + str(i % number_of_pathes))
path_list.append('match/integer/s' + str(i % number_of_pathes))
parsing_model.append(
DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE))
parsing_model.append(
FixedDataModelElement('s' + str(i % number_of_pathes),
b' Euro '))
i = i + 1
sequence_model_element = SequenceModelElement(
'integer', parsing_model)
match_value_stream_writer = MatchValueStreamWriter(
self.output_stream, path_list, b';', b'-')
t = time.time()
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
data = b''
p = process_time()
for j in range(
1,
int(number_of_pathes / 2) + number_of_pathes % 2 + 1):
data = data + str(j).encode() + b' Euro '
seconds = seconds + process_time() - p
match_context = MatchContext(data)
match_element = sequence_model_element.get_match_element(
'match', match_context)
log_atom = LogAtom(match_element.match_object,
ParserMatch(match_element), t,
match_value_stream_writer)
match_value_stream_writer.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
match_value_stream_writer.__class__.__name__, avg, results,
self.different_pathes % number_of_pathes)
def run_timestamp_correction_filters(self, number_of_pathes):
results = [None] * self.iterations
avg = 0
z = 0
while z < self.iterations:
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default', True)
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[new_match_path_detector])
seconds = time.time()
i = 0
while int(time.time() - seconds) < self.waiting_time:
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd' + str(i % number_of_pathes),
DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
p = process_time()
r = random.randint(1, 1000000)
seconds = seconds + process_time() - p
match_context = MatchContext(str(i).encode())
match_element = decimal_integer_value_me.get_match_element(
'integer', match_context)
log_atom = LogAtom(match_element.match_string,
ParserMatch(match_element), seconds - r,
simple_monotonic_timestamp_adjust)
simple_monotonic_timestamp_adjust.receive_atom(log_atom)
i = i + 1
results[z] = i
z = z + 1
avg = avg + i
avg = avg / self.iterations
type(self).result = self.result + self.result_string % (
simple_monotonic_timestamp_adjust.__class__.__name__, avg, results,
'a %s and %d different path(es).' %
(new_match_path_detector.__class__.__name__, number_of_pathes))
def get_model():
"""Return a model to parse Apache Error logs from the AIT-LDS."""
model = SequenceModelElement('model', [
FixedDataModelElement('sp1', b'['),
FixedWordlistDataModelElement('day', [b'Mon', b'Tue', b'Wed', b'Thu', b'Fri', b'Sat', b'Sun']),
FixedDataModelElement('sp2', b' '),
DateTimeModelElement('time', b'%b %d %H:%M:%S.%f %Y'),
FixedDataModelElement('error_str', b'] [:error] [pid '),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('sp3', b'] [client '),
IpAddressDataModelElement('client_ip'),
FixedDataModelElement('colon', b':'),
DecimalIntegerValueModelElement('client_port'),
FixedDataModelElement('php', b'] PHP '),
FirstMatchModelElement('fphp', [
SequenceModelElement('warning', [
FixedDataModelElement('warning_str', b'Warning: '),
FirstMatchModelElement('warning', [
SequenceModelElement('declaration', [
FixedDataModelElement('declaration_str', b'Declaration of '),
DelimitedDataModelElement('function', b')'),
FixedDataModelElement('compatible_str', b') should be compatible with '),
DelimitedDataModelElement('function2', b')'),
FixedDataModelElement('compatible_str', b') in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer')]),
SequenceModelElement('system', [
FixedDataModelElement('system_str', b'system(): Cannot execute a blank command in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')])])]),
SequenceModelElement('notice', [
FixedDataModelElement('notice_str', b'Notice: Undefined index: '),
DelimitedDataModelElement('command', b' '),
FixedDataModelElement('sp', b' in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line')]),
SequenceModelElement('deprecated', [
FixedDataModelElement('deprecated_str', b'Deprecated: Methods with the same name as their class '
b'will not be constructors in a future version of PHP; '),
DelimitedDataModelElement('class', b' '),
FixedDataModelElement('constructor_str', b' has a deprecated constructor in '),
DelimitedDataModelElement('path', b' '),
FixedDataModelElement('compatible_str', b' on line '),
DecimalIntegerValueModelElement('line'),
FixedDataModelElement('referer_str', b', referer: '),
AnyByteDataModelElement('referer'),
])])])
return model
def get_model():
"""This method returns the model."""
type_children = [
FixedDataModelElement('start',
b' * Starting Tomcat servlet engine tomcat7'),
FixedDataModelElement('stop',
b' * Stopping Tomcat servlet engine tomcat7'),
FixedDataModelElement('done', b' ...done.'),
AnyByteDataModelElement('unparsed')
]
model = SequenceModelElement('tomcat7', [
FixedDataModelElement('sname', b'tomcat7['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_tmp_files_model():
"""This function defines how to parse a systemd tmpfiles daemon message after any standard logging preamble, e.g. from syslog."""
type_children = [
SequenceModelElement('duplicate', [
FixedDataModelElement('s0', b'[/usr/lib/tmpfiles.d/var.conf:14] Duplicate line for path "'),
DelimitedDataModelElement('path', b'", ignoring.'),
FixedDataModelElement('s2', b'", ignoring.')
])
]
# Will fail on username models including the dot at the end.
model = SequenceModelElement('systemd-tmpfiles', [
FixedDataModelElement('sname', b'systemd-tmpfiles['),
DecimalIntegerValueModelElement('pid'),
FixedDataModelElement('s0', b']: '),
FirstMatchModelElement('msg', type_children)
])
return model
def get_model():
"""Return a model to parse Suricata Event logs from the AIT-LDS."""
conn = SequenceModelElement('conn', [
FixedDataModelElement('src_ip_str', b'"src_ip":"'),
FirstMatchModelElement('ip', [
SequenceModelElement('ipv4', [
IpAddressDataModelElement('src_ip'),
FixedDataModelElement('src_port_str', b'","src_port":'),
DecimalIntegerValueModelElement('src_port'),
FixedDataModelElement('dest_ip_str', b',"dest_ip":"'),
IpAddressDataModelElement('dest_ip'),
FixedDataModelElement('dest_port_str', b'","dest_port":'),
DecimalIntegerValueModelElement('dest_port'),
FixedDataModelElement('proto_str', b',"proto":"'),
DelimitedDataModelElement('proto', b'"'),
FixedDataModelElement('quote', b'"')
]),
SequenceModelElement('ipv6', [
DelimitedDataModelElement('src_ip', b'"'),
FixedDataModelElement('dest_ip_str', b'","dest_ip":"'),
DelimitedDataModelElement('dest_ip', b'"'),
FixedDataModelElement('proto_str', b'","proto":"'),
DelimitedDataModelElement('proto', b'"'),
FixedDataModelElement('icmp_type_str', b'","icmp_type":'),
DecimalIntegerValueModelElement('icmp_type'),
FixedDataModelElement('icmp_code_str', b',"icmp_code":'),
DecimalIntegerValueModelElement('icmp_code'),
]),
])
])
http = SequenceModelElement('http', [
FixedDataModelElement('hostname_str', b',"http":{"hostname":"'),
DelimitedDataModelElement('hostname', b'"'),
FixedDataModelElement('url_str', b'","url":"'),
DelimitedDataModelElement('url', b'"', escape=b'\\'),
FixedDataModelElement('http_user_agent_str', b'","http_user_agent":"'),
DelimitedDataModelElement('http_user_agent', b'"'),
OptionalMatchModelElement(
'content_type', SequenceModelElement('content_type', [
FixedDataModelElement('http_content_type_str', b'","http_content_type":"'),
DelimitedDataModelElement('http_content_type', b'"'),
])),
OptionalMatchModelElement(
'http_refer', SequenceModelElement('http_refer', [
FixedDataModelElement('http_refer_str', b'","http_refer":"'),
DelimitedDataModelElement('http_refer', b'"'),
])),
FixedDataModelElement('http_method_str', b'","http_method":"'),
DelimitedDataModelElement('http_method', b'"'),
FixedDataModelElement('protocol_str', b'","protocol":"'),
DelimitedDataModelElement('protocol', b'"'),
FixedDataModelElement('quote_str', b'"'),
OptionalMatchModelElement(
'status', SequenceModelElement('status', [
FixedDataModelElement('status_str', b',"status":'),
DecimalIntegerValueModelElement('status'),
])),
OptionalMatchModelElement(
'redirect', SequenceModelElement('redirect', [
FixedDataModelElement('redirect_str', b',"redirect":"'),
DelimitedDataModelElement('redirect', b'"'),
FixedDataModelElement('quote_str', b'"')
])),
FixedDataModelElement('length_str', b',"length":'),
DecimalIntegerValueModelElement('length'),
FixedDataModelElement('brack_str', b'}')
])
model = SequenceModelElement('model', [
FixedDataModelElement('time_str', b'{"timestamp":"'),
DateTimeModelElement('time', b'%Y-%m-%dT%H:%M:%S.%f'),
FixedDataModelElement('plus_sign', b'+'),
DecimalIntegerValueModelElement('tz'),
FixedDataModelElement('comma_str', b'",'),
OptionalMatchModelElement(
'flow_id', SequenceModelElement('flow_id', [
FixedDataModelElement('flow_id_str', b'"flow_id":'),
DecimalIntegerValueModelElement('flow_id'),
FixedDataModelElement('comma_str', b',')])),
OptionalMatchModelElement(
'in_iface', SequenceModelElement('in_iface', [
FixedDataModelElement('in_iface_str', b'"in_iface":"'),
DelimitedDataModelElement('in_iface', b'"'),
FixedDataModelElement('comma_str', b'",')])),
FixedDataModelElement('event_type_str', b'"event_type":"'),
FirstMatchModelElement('event_type', [
SequenceModelElement('dns', [
FixedDataModelElement('dns_str', b'dns",'),
conn,
SequenceModelElement('dns', [
FixedDataModelElement('type_str', b',"dns":{"type":"'),
DelimitedDataModelElement('type', b'"'),
FixedDataModelElement('id_str', b'","id":'),
DecimalIntegerValueModelElement('id'),
OptionalMatchModelElement(
'rcode', SequenceModelElement('rcode', [
FixedDataModelElement('rcode_str', b',"rcode":"'),
DelimitedDataModelElement('rcode', b'"'),
FixedDataModelElement('quote_str', b'"')])),
FixedDataModelElement('rrname_str', b',"rrname":"'),
DelimitedDataModelElement('rrname', b'"'),
OptionalMatchModelElement('rrtype', SequenceModelElement('rrtype', [
FixedDataModelElement('rrtype_str', b'","rrtype":"'),
DelimitedDataModelElement('rrtype', b'"')])),
FixedDataModelElement('quote', b'"'),
OptionalMatchModelElement(
'tx_id', SequenceModelElement('tx_id', [
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id')])),
OptionalMatchModelElement('ttl', SequenceModelElement('ttl', [
FixedDataModelElement('ttl_str', b',"ttl":'),
DecimalIntegerValueModelElement('ttl')])),
OptionalMatchModelElement(
'rdata', SequenceModelElement('rdata', [
FixedDataModelElement('rdata_str', b',"rdata":"'),
DelimitedDataModelElement('rdata', b'"'),
FixedDataModelElement('quote_str', b'"')])),
FixedDataModelElement('brack_str', b'}}')
]),
]),
SequenceModelElement('flow', [
FixedDataModelElement('flow_str', b'flow",'),
conn,
OptionalMatchModelElement(
'app_proto', SequenceModelElement('app_proto', [
FixedDataModelElement('app_proto_str', b',"app_proto":"'),
DelimitedDataModelElement('app_proto', b'"'),
FixedDataModelElement('quote_str', b'"')
])
),
OptionalMatchModelElement(
'app_proto_tc', SequenceModelElement('app_proto_tc', [
FixedDataModelElement('app_proto_tc_str', b',"app_proto_tc":"'),
DelimitedDataModelElement('app_proto_tc', b'"'),
FixedDataModelElement('quote_str', b'"')
])
),
SequenceModelElement('flow', [
FixedDataModelElement('pkts_toserver_str', b',"flow":{"pkts_toserver":'),
DecimalIntegerValueModelElement('pkts_toserver'),
FixedDataModelElement('pkts_toclient_str', b',"pkts_toclient":'),
DecimalIntegerValueModelElement('pkts_toclient'),
FixedDataModelElement('bytes_toserver_str', b',"bytes_toserver":'),
DecimalIntegerValueModelElement('bytes_toserver'),
FixedDataModelElement('bytes_toclient_str', b',"bytes_toclient":'),
DecimalIntegerValueModelElement('bytes_toclient'),
FixedDataModelElement('start_str', b',"start":"'),
DelimitedDataModelElement('start', b'"'),
FixedDataModelElement('end_str', b'","end":"'),
DelimitedDataModelElement('end', b'"'),
FixedDataModelElement('age_str', b'","age":'),
DecimalIntegerValueModelElement('age'),
FixedDataModelElement('state_str', b',"state":"'),
DelimitedDataModelElement('state', b'"'),
FixedDataModelElement('reason_str', b'","reason":"'),
DelimitedDataModelElement('reason', b'"'),
FixedDataModelElement('alerted_str', b'","alerted":'),
FixedWordlistDataModelElement('alerted', [b'true', b'false']),
FixedDataModelElement('brack_str1', b'}'),
OptionalMatchModelElement(
'tcp', SequenceModelElement('tcp', [
FixedDataModelElement('tcp_flags_str', b',"tcp":{"tcp_flags":"'),
HexStringModelElement('tcp_flags'),
FixedDataModelElement('tcp_flags_ts_str', b'","tcp_flags_ts":"'),
HexStringModelElement('tcp_flags_ts'),
FixedDataModelElement('tcp_flags_tc_str', b'","tcp_flags_tc":"'),
HexStringModelElement('tcp_flags_tc'),
OptionalMatchModelElement(
'flags', SequenceModelElement('flags', [
FixedDataModelElement('syn_str', b'","syn":'),
FixedWordlistDataModelElement('syn', [b'true', b'false']),
OptionalMatchModelElement(
'fin', SequenceModelElement('fin', [
FixedDataModelElement('fin_str', b',"fin":'),
FixedWordlistDataModelElement('fin', [b'true', b'false']),
])
),
OptionalMatchModelElement(
'rst', SequenceModelElement('rst', [
FixedDataModelElement('rst_str', b',"rst":'),
FixedWordlistDataModelElement('rst', [b'true', b'false']),
])
),
OptionalMatchModelElement(
'psh', SequenceModelElement('psh', [
FixedDataModelElement('psh_str', b',"psh":'),
FixedWordlistDataModelElement('psh', [b'true', b'false']),
])
),
FixedDataModelElement('ack_str', b',"ack":'),
FixedWordlistDataModelElement('ack', [b'true', b'false']),
FixedDataModelElement('tcp_state_str', b',"state":"'),
DelimitedDataModelElement('tcp_state', b'"'),
])
),
FixedDataModelElement('tcp_brack_str', b'"}'),
])
),
FixedDataModelElement('brack_str2', b'}')
]),
]),
SequenceModelElement('http', [
FixedDataModelElement('http_str', b'http",'),
conn,
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
http,
FixedDataModelElement('brack_str', b'}')
]),
SequenceModelElement('fileinfo', [
FixedDataModelElement('fileinfo_str', b'fileinfo",'),
conn,
http,
FixedDataModelElement('app_proto_str', b',"app_proto":"'),
DelimitedDataModelElement('app_proto', b'"'),
SequenceModelElement('fileinfo', [
FixedDataModelElement('fileinfo_str', b'","fileinfo":{'),
OptionalMatchModelElement(
'filename', SequenceModelElement('filename', [
FixedDataModelElement('filename_str', b'"filename":"'),
DelimitedDataModelElement('filename', b'"'),
FixedDataModelElement('quote_str', b'",')
])
),
FixedDataModelElement('state_str', b'"state":"'),
DelimitedDataModelElement('state', b'"'),
FixedDataModelElement('stored_str', b'","stored":'),
FixedWordlistDataModelElement('stored', [b'true', b'false']),
FixedDataModelElement('size_str', b',"size":'),
DecimalIntegerValueModelElement('size'),
FixedDataModelElement('tx_id_str', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
FixedDataModelElement('brack_str', b'}}')
]),
]),
SequenceModelElement('stats', [
FixedDataModelElement('stats_str', b'stats",'),
FixedDataModelElement('uptime_str', b'"stats":{"uptime":'),
DecimalIntegerValueModelElement('uptime'),
SequenceModelElement('capture', [
FixedDataModelElement('capture_str', b',"capture":{'),
FixedDataModelElement('kernel_packets_str', b'"kernel_packets":'),
DecimalIntegerValueModelElement('kernel_packets'),
FixedDataModelElement('kernel_drops_str', b',"kernel_drops":'),
DecimalIntegerValueModelElement('kernel_drops'),
FixedDataModelElement('brack_str', b'}')
]),
SequenceModelElement('decoder', [
FixedDataModelElement('pkts_str', b',"decoder":{"pkts":'),
DecimalIntegerValueModelElement('pkts'),
FixedDataModelElement('bytes_str', b',"bytes":'),
DecimalIntegerValueModelElement('bytes'),
FixedDataModelElement('invalid_str', b',"invalid":'),
DecimalIntegerValueModelElement('invalid'),
FixedDataModelElement('ipv4_str', b',"ipv4":'),
DecimalIntegerValueModelElement('ipv4'),
FixedDataModelElement('ipv6_str', b',"ipv6":'),
DecimalIntegerValueModelElement('ipv6'),
FixedDataModelElement('ethernet_str', b',"ethernet":'),
DecimalIntegerValueModelElement('ethernet'),
FixedDataModelElement('raw_str', b',"raw":'),
DecimalIntegerValueModelElement('raw'),
FixedDataModelElement('null_str', b',"null":'),
DecimalIntegerValueModelElement('null'),
FixedDataModelElement('sll_str', b',"sll":'),
DecimalIntegerValueModelElement('sll'),
FixedDataModelElement('tcp_str', b',"tcp":'),
DecimalIntegerValueModelElement('tcp'),
FixedDataModelElement('udp_str', b',"udp":'),
DecimalIntegerValueModelElement('udp'),
FixedDataModelElement('sctp_str', b',"sctp":'),
DecimalIntegerValueModelElement('sctp'),
FixedDataModelElement('icmpv4_str', b',"icmpv4":'),
DecimalIntegerValueModelElement('icmpv4'),
FixedDataModelElement('icmpv6_str', b',"icmpv6":'),
DecimalIntegerValueModelElement('icmpv6'),
FixedDataModelElement('ppp_str', b',"ppp":'),
DecimalIntegerValueModelElement('ppp'),
FixedDataModelElement('pppoe_str', b',"pppoe":'),
DecimalIntegerValueModelElement('pppoe'),
FixedDataModelElement('gre_str', b',"gre":'),
DecimalIntegerValueModelElement('gre'),
FixedDataModelElement('vlan_str', b',"vlan":'),
DecimalIntegerValueModelElement('vlan'),
FixedDataModelElement('vlan_qinq_str', b',"vlan_qinq":'),
DecimalIntegerValueModelElement('vlan_qinq'),
FixedDataModelElement('teredo_str', b',"teredo":'),
DecimalIntegerValueModelElement('teredo'),
FixedDataModelElement('ipv4_in_ipv6_str', b',"ipv4_in_ipv6":'),
DecimalIntegerValueModelElement('ipv4_in_ipv6'),
FixedDataModelElement('ipv6_in_ipv6_str', b',"ipv6_in_ipv6":'),
DecimalIntegerValueModelElement('ipv6_in_ipv6'),
FixedDataModelElement('mpls_str', b',"mpls":'),
DecimalIntegerValueModelElement('mpls'),
FixedDataModelElement('avg_pkt_size_str', b',"avg_pkt_size":'),
DecimalIntegerValueModelElement('avg_pkt_size'),
FixedDataModelElement('max_pkt_size_str', b',"max_pkt_size":'),
DecimalIntegerValueModelElement('max_pkt_size'),
FixedDataModelElement('erspan_str', b',"erspan":'),
DecimalIntegerValueModelElement('erspan'),
SequenceModelElement('ipraw', [
FixedDataModelElement('invalid_ip_version_str', b',"ipraw":{"invalid_ip_version":'),
DecimalIntegerValueModelElement('invalid_ip_version'),
]),
SequenceModelElement('ltnull', [
FixedDataModelElement('ipraw_pkt_too_small_str', b'},"ltnull":{"pkt_too_small":'),
DecimalIntegerValueModelElement('ipraw_pkt_too_small'),
FixedDataModelElement('unsupported_type', b',"unsupported_type":'),
DecimalIntegerValueModelElement('unsupported_type'),
]),
SequenceModelElement('dce', [
FixedDataModelElement('dce_pkt_too_small_str', b'},"dce":{"pkt_too_small":'),
DecimalIntegerValueModelElement('dce_pkt_too_small'),
FixedDataModelElement('brack_str', b'}')
])
]),
SequenceModelElement('flow', [
FixedDataModelElement('memcap_str', b'},"flow":{"memcap":'),
DecimalIntegerValueModelElement('memcap'),
FixedDataModelElement('spare_str', b',"spare":'),
DecimalIntegerValueModelElement('spare'),
FixedDataModelElement('emerg_mode_entered_str', b',"emerg_mode_entered":'),
DecimalIntegerValueModelElement('emerg_mode_entered'),
FixedDataModelElement('emerg_mode_over_str', b',"emerg_mode_over":'),
DecimalIntegerValueModelElement('emerg_mode_over'),
FixedDataModelElement('tcp_reuse_str', b',"tcp_reuse":'),
DecimalIntegerValueModelElement('tcp_reuse'),
FixedDataModelElement('memuse_str', b',"memuse":'),
DecimalIntegerValueModelElement('memuse'),
]),
SequenceModelElement('defrag', [
SequenceModelElement('ipv4', [
FixedDataModelElement('fragments_str', b'},"defrag":{"ipv4":{"fragments":'),
DecimalIntegerValueModelElement('fragments'),
FixedDataModelElement('reassembled_str', b',"reassembled":'),
DecimalIntegerValueModelElement('reassembled_str'),
FixedDataModelElement('timeouts_str', b',"timeouts":'),
DecimalIntegerValueModelElement('timeouts'),
]),
SequenceModelElement('ipv6', [
FixedDataModelElement('fragments_str', b'},"ipv6":{"fragments":'),
DecimalIntegerValueModelElement('fragments'),
FixedDataModelElement('reassembled_str', b',"reassembled":'),
DecimalIntegerValueModelElement('reassembled_str'),
FixedDataModelElement('timeouts_str', b',"timeouts":'),
DecimalIntegerValueModelElement('timeouts'),
]),
FixedDataModelElement('max_frag_hits_str', b'},"max_frag_hits":'),
DecimalIntegerValueModelElement('max_frag_hits'),
]),
SequenceModelElement('tcp', [
FixedDataModelElement('sessions_str', b'},"tcp":{"sessions":'),
DecimalIntegerValueModelElement('sessions'),
FixedDataModelElement('ssn_memcap_drop_str', b',"ssn_memcap_drop":'),
DecimalIntegerValueModelElement('ssn_memcap_drop'),
FixedDataModelElement('pseudo_str', b',"pseudo":'),
DecimalIntegerValueModelElement('pseudo'),
FixedDataModelElement('pseudo_failed_str', b',"pseudo_failed":'),
DecimalIntegerValueModelElement('pseudo_failed'),
FixedDataModelElement('invalid_checksum_str', b',"invalid_checksum":'),
DecimalIntegerValueModelElement('invalid_checksum'),
FixedDataModelElement('no_flow_str', b',"no_flow":'),
DecimalIntegerValueModelElement('no_flow'),
FixedDataModelElement('syn_str', b',"syn":'),
DecimalIntegerValueModelElement('syn'),
FixedDataModelElement('synack_str', b',"synack":'),
DecimalIntegerValueModelElement('synack'),
FixedDataModelElement('rst_str', b',"rst":'),
DecimalIntegerValueModelElement('rst'),
FixedDataModelElement('segment_memcap_drop_str', b',"segment_memcap_drop":'),
DecimalIntegerValueModelElement('segment_memcap_drop'),
FixedDataModelElement('stream_depth_reached_str', b',"stream_depth_reached":'),
DecimalIntegerValueModelElement('stream_depth_reached'),
FixedDataModelElement('reassembly_gap_str', b',"reassembly_gap":'),
DecimalIntegerValueModelElement('reassembly_gap'),
FixedDataModelElement('memuse_str', b',"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('reassembly_memuse_str', b',"reassembly_memuse":'),
DecimalIntegerValueModelElement('reassembly_memuse'),
]),
SequenceModelElement('detect', [
FixedDataModelElement('alert_str', b'},"detect":{"alert":'),
DecimalIntegerValueModelElement('alert')
]),
SequenceModelElement('app_layer', [
SequenceModelElement('flow', [
FixedDataModelElement('http_str', b'},"app_layer":{"flow":{"http":'),
DecimalIntegerValueModelElement('http'),
FixedDataModelElement('ftp_str', b',"ftp":'),
DecimalIntegerValueModelElement('ftp'),
FixedDataModelElement('smtp_str', b',"smtp":'),
DecimalIntegerValueModelElement('smtp'),
FixedDataModelElement('tls_str', b',"tls":'),
DecimalIntegerValueModelElement('tls'),
FixedDataModelElement('ssh_str', b',"ssh":'),
DecimalIntegerValueModelElement('ssh'),
FixedDataModelElement('imap_str', b',"imap":'),
DecimalIntegerValueModelElement('imap'),
FixedDataModelElement('msn_str', b',"msn":'),
DecimalIntegerValueModelElement('msn'),
FixedDataModelElement('smb_str', b',"smb":'),
DecimalIntegerValueModelElement('smb'),
FixedDataModelElement('dcerpc_tcp_str', b',"dcerpc_tcp":'),
DecimalIntegerValueModelElement('dcerpc_tcp'),
FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'),
DecimalIntegerValueModelElement('dns_tcp'),
FixedDataModelElement('failed_tcp_str', b',"failed_tcp":'),
DecimalIntegerValueModelElement('failed_tcp'),
FixedDataModelElement('dcerpc_udp_str', b',"dcerpc_udp":'),
DecimalIntegerValueModelElement('dcerpc_udp'),
FixedDataModelElement('dns_udp_str', b',"dns_udp":'),
DecimalIntegerValueModelElement('dns_udp'),
FixedDataModelElement('failed_udp_str', b',"failed_udp":'),
DecimalIntegerValueModelElement('failed_udp'),
]),
SequenceModelElement('tx', [
FixedDataModelElement('http_str', b'},"tx":{"http":'),
DecimalIntegerValueModelElement('http'),
FixedDataModelElement('smtp_str', b',"smtp":'),
DecimalIntegerValueModelElement('smtp'),
FixedDataModelElement('tls_str', b',"tls":'),
DecimalIntegerValueModelElement('tls'),
FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'),
DecimalIntegerValueModelElement('dns_tcp'),
FixedDataModelElement('dns_udp_str', b',"dns_udp":'),
DecimalIntegerValueModelElement('dns_udp'),
])
]),
SequenceModelElement('flow_mgr', [
FixedDataModelElement('closed_pruned_str', b'}},"flow_mgr":{"closed_pruned":'),
DecimalIntegerValueModelElement('closed_pruned'),
FixedDataModelElement('new_pruned_str', b',"new_pruned":'),
DecimalIntegerValueModelElement('new_pruned'),
FixedDataModelElement('est_pruned_str', b',"est_pruned":'),
DecimalIntegerValueModelElement('est_pruned'),
FixedDataModelElement('bypassed_pruned_str', b',"bypassed_pruned":'),
DecimalIntegerValueModelElement('bypassed_pruned'),
FixedDataModelElement('flows_checked_str', b',"flows_checked":'),
DecimalIntegerValueModelElement('flows_checked'),
FixedDataModelElement('flows_notimeout_str', b',"flows_notimeout":'),
DecimalIntegerValueModelElement('flows_notimeout'),
FixedDataModelElement('flows_timeout_str', b',"flows_timeout":'),
DecimalIntegerValueModelElement('flows_timeout'),
FixedDataModelElement('flows_timeout_inuse_str', b',"flows_timeout_inuse":'),
DecimalIntegerValueModelElement('flows_timeout_inuse'),
FixedDataModelElement('flows_removed_str', b',"flows_removed":'),
DecimalIntegerValueModelElement('flows_removed'),
FixedDataModelElement('rows_checked_str', b',"rows_checked":'),
DecimalIntegerValueModelElement('rows_checked'),
FixedDataModelElement('rows_skipped_str', b',"rows_skipped":'),
DecimalIntegerValueModelElement('rows_skipped'),
FixedDataModelElement('rows_empty_str', b',"rows_empty":'),
DecimalIntegerValueModelElement('rows_empty'),
FixedDataModelElement('rows_busy_str', b',"rows_busy":'),
DecimalIntegerValueModelElement('rows_busy'),
FixedDataModelElement('rows_maxlen_str', b',"rows_maxlen":'),
DecimalIntegerValueModelElement('rows_maxlen'),
]),
SequenceModelElement('dns', [
FixedDataModelElement('memuse_str', b'},"dns":{"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('memcap_state_str', b',"memcap_state":'),
DecimalIntegerValueModelElement('memcap_state'),
FixedDataModelElement('memcap_global_str', b',"memcap_global":'),
DecimalIntegerValueModelElement('memcap_global'),
]),
SequenceModelElement('http', [
FixedDataModelElement('memuse_str', b'},"http":{"memuse":'),
DecimalIntegerValueModelElement('memuse'),
FixedDataModelElement('memcap_str', b',"memcap":'),
DecimalIntegerValueModelElement('memcap'),
]),
FixedDataModelElement('quote_str', b'}}}')
]),
SequenceModelElement('tls', [
FixedDataModelElement('tls_str', b'tls",'),
conn,
SequenceModelElement('tls', [
FixedDataModelElement('subject_str', b',"tls":{"subject":"'),
DelimitedDataModelElement('subject', b'"'),
FixedDataModelElement('issuerdn_str', b'","issuerdn":"'),
DelimitedDataModelElement('issuerdn', b'"'),
FixedDataModelElement('fingerprint_str', b'","fingerprint":"'),
DelimitedDataModelElement('fingerprint', b'"'),
OptionalMatchModelElement(
'sni', SequenceModelElement('sni', [
FixedDataModelElement('sni_str', b'","sni":"'),
DelimitedDataModelElement('sni', b'"'),
])
),
FixedDataModelElement('version_str', b'","version":"'),
DelimitedDataModelElement('version', b'"'),
FixedDataModelElement('notbefore_str', b'","notbefore":"'),
DelimitedDataModelElement('notbefore', b'"'),
FixedDataModelElement('notafter_str', b'","notafter":"'),
DelimitedDataModelElement('notafter', b'"'),
]),
FixedDataModelElement('brack_str', b'"}}')
]),
SequenceModelElement('alert', [
FixedDataModelElement('alert_str', b'alert",'),
conn,
OptionalMatchModelElement(
'tx_id', SequenceModelElement('tx_id', [
FixedDataModelElement('tx_id', b',"tx_id":'),
DecimalIntegerValueModelElement('tx_id'),
])),
SequenceModelElement('alert', [
FixedDataModelElement('action_str', b',"alert":{"action":"'),
DelimitedDataModelElement('action', b'"'),
FixedDataModelElement('gid_str', b'","gid":'),
DecimalIntegerValueModelElement('gid'),
FixedDataModelElement('signature_id_str', b',"signature_id":'),
DecimalIntegerValueModelElement('signature_id'),
FixedDataModelElement('rev_str', b',"rev":'),
DecimalIntegerValueModelElement('rev'),
FixedDataModelElement('signature_str', b',"signature":"'),
DelimitedDataModelElement('signature', b'"'),
FixedDataModelElement('category_str', b'","category":"'),
DelimitedDataModelElement('category', b'"'),
FixedDataModelElement('severity_str', b'","severity":'),
DecimalIntegerValueModelElement('severity'),
FixedDataModelElement('brack_str', b'}')
]),
http,
FixedDataModelElement('brack_str', b'}')
]),
])
])
return model
class TimeCorrelationViolationDetectorTest(TestBase):
"""Unittests for the TimeCorrelationViolationDetector."""
_expected_string = '%s Correlation rule "%s" violated\nTimeCorrelationViolationDetector: "%s" (%d lines)\n FAIL: '
_expected_string_too_early = _expected_string + 'B-Event for "%s" (%s) was found too early!\n\n\n'
_expected_string_too_late = _expected_string + 'B-Event for "%s" (%s) was not found in time!\n\n\n'
_expected_string_different_attributes = _expected_string + '"%s" (%s) %d is not equal %d\n\n\n'
model = '/model'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
service_children1 = [
FixedDataModelElement('Value1Key', b'Value1: '),
FixedDataModelElement('Value1Value', b'fixed Value1'),
FixedDataModelElement('Value2Key', b', Value2: '),
DecimalIntegerValueModelElement('Value2Value'),
FixedDataModelElement('Value3Key', b', Value3: '),
FixedDataModelElement('Value3Value', b'fixed Value3'),
FixedDataModelElement('Value4Key', b', Value4: '),
FixedDataModelElement('Value4Value', b'fixed Value4')
]
service_children2 = [
FixedDataModelElement('Value1Key', b'Value1: '),
FixedDataModelElement('Value1Value', b'fixed Value1'),
FixedDataModelElement('Value2Key', b', Value2: '),
FixedDataModelElement('Value2Value', b'fixed Value2'),
FixedDataModelElement('Value3Key', b', Value3: '),
DecimalIntegerValueModelElement('Value3Value'),
FixedDataModelElement('Value4Key', b', Value4: '),
FixedDataModelElement('Value4Value', b'fixed Value4')
]
match_context1 = MatchContext(
b'Value1: fixed Value1, Value2: 22500, Value3: fixed Value3, Value4: fixed Value4'
)
match_context2 = MatchContext(
b'Value1: fixed Value1, Value2: fixed Value2, Value3: 22500, Value4: fixed Value4'
)
match_context2_different = MatchContext(
b'Value1: fixed Value1, Value2: fixed Value2, Value3: 22501, Value4: fixed Value4'
)
seq1 = SequenceModelElement('sequence1', service_children1)
seq2 = SequenceModelElement('sequence2', service_children2)
match_element1 = seq1.get_match_element(model, match_context1)
match_element2 = seq2.get_match_element(model, match_context2)
match_element2_different = seq2.get_match_element(
model, match_context2_different)
def setUp(self):
"""Set up the rules for the TimeCorrelationViolationDetector."""
TestBase.setUp(self)
self.correlation_rule = CorrelationRule(
'Correlation',
1,
1.2,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/sequence1/Value2Value',
'/model/sequence2/Value3Value')])
self.a_class_selector = EventClassSelector('Selector1',
[self.correlation_rule],
None)
self.b_class_selector = EventClassSelector('Selector2', None,
[self.correlation_rule])
self.rules = []
self.rules.append(
Rules.PathExistsMatchRule('/model/sequence1/Value2Key',
self.a_class_selector))
self.rules.append(
Rules.PathExistsMatchRule('/model/sequence2/Value3Key',
self.b_class_selector))
def test1_check_status_ok(self):
"""
In this test case the status is OK after receiving the expected data and no error message is returned.
The output of the do_timer-method is also tested in this test case.
"""
description = "Test1TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(
self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(
time_correlation_violation_detector, component_name=description)
log_atom1 = LogAtom(self.match_context1.match_data,
ParserMatch(self.match_element1), time.time(),
self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data,
ParserMatch(self.match_element2),
time.time() + 1, self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(self.output_stream.getvalue(), "")
def test2_check_status_not_found_error(self):
"""
In this test case the second log line is not found and an appropriate error message is expected from the check_status-method.
The output of the do_timer-method is also tested in this test case.
"""
description = "Test2TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(
self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(
time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data,
ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
r = self.correlation_rule.check_status(t + 2)
self.assertEqual(
r[0], 'FAIL: B-Event for "%s" (%s) was not found in time!\n' %
(self.match_element1.get_match_string().decode(),
self.a_class_selector.action_id))
def test3_check_status_before_expected_timespan(self):
"""
In this test case the second log line is found too early. An appropriate error message is expected from the check_status-method.
The output of the do_timer-method is also tested in this test case.
"""
description = "Test3TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(
self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(
time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data,
ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data,
ParserMatch(self.match_element2), time.time(),
self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(
self.output_stream.getvalue(), self._expected_string_too_early %
(datetime.fromtimestamp(t).strftime(
self.datetime_format_string), self.correlation_rule.rule_id,
description, 1, self.match_element1.get_match_string().decode(),
self.a_class_selector.action_id))
def test4_check_status_after_expected_timespan(self):
"""
In this test case the second log line is found too late. An appropriate error message is expected from the check_status-method.
The output of the do_timer-method is also tested in this test case.
"""
description = "Test4TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(
self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(
time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data,
ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data,
ParserMatch(self.match_element2), t + 5, self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(
self.output_stream.getvalue(), self._expected_string_too_late %
(datetime.fromtimestamp(t).strftime(
self.datetime_format_string), self.correlation_rule.rule_id,
description, 1, self.match_element1.get_match_string().decode(),
self.a_class_selector.action_id))
def test5_check_status_attributes_not_matching(self):
"""
In this test case the second log line has different attributes than expected.
An appropriate error message is expected from the check_status-method. The output of the do_timer-method is also tested in this
test case.
"""
description = "Test5TimeCorrelationViolationDetector"
time_correlation_violation_detector = TimeCorrelationViolationDetector(
self.analysis_context.aminer_config, self.rules,
[self.stream_printer_event_handler])
self.analysis_context.register_component(
time_correlation_violation_detector, component_name=description)
t = time.time()
log_atom1 = LogAtom(self.match_context1.match_data,
ParserMatch(self.match_element1), t, self)
time_correlation_violation_detector.receive_atom(log_atom1)
log_atom2 = LogAtom(self.match_context2.match_data,
ParserMatch(self.match_element2_different), t + 1,
self)
time_correlation_violation_detector.receive_atom(log_atom2)
time_correlation_violation_detector.do_timer(time.time())
self.assertEqual(
self.output_stream.getvalue(),
self._expected_string_different_attributes %
(datetime.fromtimestamp(t).strftime(
self.datetime_format_string), self.correlation_rule.rule_id,
description, 1, self.match_element1.get_match_string().decode(),
self.a_class_selector.action_id, 22500, 22501))
def test6_prepare_history_entry(self):
"""
In this test case the prepare_history_entry-method is tested with multiple artefact_match_parameters.
Also the case of not finding a parameter is tested.
"""
t = time.time()
p1 = ParserMatch(self.match_element1)
p2 = ParserMatch(self.match_element2)
log_atom1 = LogAtom(self.match_context1.match_data, p1, t, self)
log_atom2 = LogAtom(self.match_context2.match_data, p2, t + 5, self)
result = self.correlation_rule.prepare_history_entry(
self.a_class_selector, log_atom1)
self.assertEqual(result, [t, 0, self.a_class_selector, p1, 22500])
result = self.correlation_rule.prepare_history_entry(
self.b_class_selector, log_atom2)
self.assertEqual(result, [t + 5, 0, self.b_class_selector, p2, 22500])
class EnhancedNewMatchPathValueComboDetectorTest(TestBase):
__expected_string = '%s New value combination(s) detected\n%s: "%s" (%d lines)\n%s\n\n'
__expected_whitelisting_string = 'Whitelisted path(es) %s with %s in %s'
fixed_dme = FixedDataModelElement('s1', b'25537 uid=')
fixed_dme2 = FixedDataModelElement('s2', b' uid=2')
decimal_integer_value_me = DecimalIntegerValueModelElement(
'd1', DecimalIntegerValueModelElement.SIGN_TYPE_NONE,
DecimalIntegerValueModelElement.PAD_TYPE_NONE)
match_context_sequence_me = MatchContext(b'25537 uid=2')
seq = SequenceModelElement('seq', [fixed_dme, decimal_integer_value_me])
match_element_sequence_me = seq.get_match_element(
'first', match_context_sequence_me)
match_context_sequence_me2 = MatchContext(b'25537 uid=2')
seq2 = SequenceModelElement('seq2', [decimal_integer_value_me, fixed_dme2])
match_element_sequence_me2 = seq2.get_match_element(
'second', match_context_sequence_me2)
first_seq_s1 = 'first/seq/s1'
first_seq_d1 = 'first/seq/d1'
datetime_format_string = '%Y-%m-%d %H:%M:%S'
exp_str = " first/seq: b'25537 uid=2'\n " + first_seq_s1 + ": b'25537 uid='\n " + first_seq_d1 + \
": 2\n{(b'25537 uid=', 2): [%s, %s, 1]}"
exp_str2 = " {(b'25537 uid=', 2): [%s, %s, 1]}\nb'25537 uid=2'"
def test1_log_atom_not_known(self):
"""This test case checks the correct processing of unknown log lines, which in reality means that an anomaly has been found. The
output is directed to an output stream and compared for accuracy. The auto_include_flag is False and the output must be repeatable
on second run."""
description = "Test1EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.exp_str2 % (t, t)))
self.reset_output_stream()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t + 2,
enhanced_new_match_path_value_combo_detector)
# repeating should produce the same result with new extraData.
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 2).strftime(
self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1,
" {(b'25537 uid=', 2): [%s, %s, 2]}\nb'25537 uid=2'" %
(t, t + 2)))
self.reset_output_stream()
enhanced_new_match_path_value_combo_detector2 = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(
enhanced_new_match_path_value_combo_detector2.receive_atom(
log_atom_sequence_me2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(25537, b' uid=2'): [%s, %s, 1]}\nb'25537 uid=2'" % (t, t)))
def test2_log_atom_known(self):
"""This test case checks the functionality of the auto_include_flag. If the same MatchElement is processed a second time and the
auto_include_flag was True, no event must be triggered."""
description = "Test2EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.exp_str2 % (t, t)))
self.reset_output_stream()
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
# repeating should NOT produce the same result, only persist the new extraData.
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(self.output_stream.getvalue(), '')
self.reset_output_stream()
enhanced_new_match_path_value_combo_detector2 = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['second/seq2/d1', 'second/seq2/s2'],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector2, description + "2")
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector2)
# other MatchElement
self.assertTrue(
enhanced_new_match_path_value_combo_detector2.receive_atom(
log_atom_sequence_me2))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(25537, b' uid=2'): [%s, %s, 1]}\nb'25537 uid=2'" % (t, t)))
def test3_log_atom_known_from_persisted_data(self):
"""The persisting and reading of permitted log lines should be checked with this test."""
description = "Test3EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = round(time.time(), 3)
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertTrue(
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t).strftime(self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description, 1, self.exp_str2 % (t, t)))
enhanced_new_match_path_value_combo_detector.do_persist()
self.reset_output_stream()
other_enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
False,
output_log_line=False)
self.analysis_context.register_component(
other_enhanced_new_match_path_value_combo_detector,
description + "2")
other_log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t + 2,
other_enhanced_new_match_path_value_combo_detector)
self.assertTrue(
other_enhanced_new_match_path_value_combo_detector.receive_atom(
other_log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 2).strftime(
self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(b'25537 uid=', 2): [%s, %s, 2]}\nb'25537 uid=2'" %
(t, t + 2)))
self.reset_output_stream()
other_log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t + 5,
other_enhanced_new_match_path_value_combo_detector)
self.assertTrue(
other_enhanced_new_match_path_value_combo_detector.receive_atom(
other_log_atom_sequence_me))
self.assertEqual(
self.output_stream.getvalue(), self.__expected_string %
(datetime.fromtimestamp(t + 5).strftime(
self.datetime_format_string),
enhanced_new_match_path_value_combo_detector.__class__.__name__,
description + "2", 1,
" {(b'25537 uid=', 2): [%s, %s, 3]}\nb'25537 uid=2'" %
(t, t + 5)))
def test4_whitelist_event_with_known_and_unknown_paths(self):
"""This test case checks in which cases an event is triggered and compares with expected results."""
description = "Test4EnhancedNewMatchPathValueComboDetector"
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, [self.first_seq_s1, self.first_seq_d1],
[self.stream_printer_event_handler],
'Default',
False,
True,
output_log_line=False)
self.analysis_context.register_component(
enhanced_new_match_path_value_combo_detector, description)
t = time.time()
log_atom_sequence_me = LogAtom(
self.match_element_sequence_me.get_match_string(),
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
self.assertEqual(
enhanced_new_match_path_value_combo_detector.whitelist_event(
'Analysis.%s' % enhanced_new_match_path_value_combo_detector.
__class__.__name__, [
log_atom_sequence_me,
[self.match_element_sequence_me.get_path()]
], [
log_atom_sequence_me,
self.match_element_sequence_me.get_path()
], None), self.__expected_whitelisting_string %
(', '.join(
enhanced_new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me.get_path(), log_atom_sequence_me))
log_atom_sequence_me2 = LogAtom(
self.match_element_sequence_me2.get_match_string(),
ParserMatch(self.match_element_sequence_me2), t,
enhanced_new_match_path_value_combo_detector)
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
self.assertEqual(
enhanced_new_match_path_value_combo_detector.whitelist_event(
'Analysis.%s' % enhanced_new_match_path_value_combo_detector.
__class__.__name__, [
log_atom_sequence_me2,
[self.match_element_sequence_me2.get_path()]
], [
log_atom_sequence_me2,
self.match_element_sequence_me2.get_path()
], None), self.__expected_whitelisting_string %
(', '.join(
enhanced_new_match_path_value_combo_detector.target_path_list),
self.match_element_sequence_me2.path, log_atom_sequence_me2))
def test5save_metadata(self):
"""This test case checks the correctness of the metadata informations"""
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
self.aminer_config, ['first/f1/s1'],
[self.stream_printer_event_handler],
'Default',
False,
True,
None,
output_log_line=False)
t = 1
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_sequence_me), t,
enhanced_new_match_path_value_combo_detector)
enhanced_new_match_path_value_combo_detector.receive_atom(
log_atom_sequence_me)
self.assertEqual(
enhanced_new_match_path_value_combo_detector.known_values_dict.get(
(self.fixed_dme.fixed_data, (t, t, 1))), None)
def build_analysis_pipeline(analysis_context):
"""Define the function to create pipeline for parsing the log
data. It has also to define an AtomizerFactory to instruct AMiner
how to process incoming data streams to create log atoms from
them."""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('Fixed Workload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Start Time', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', "de_AT.utf8", None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets
# to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
AnyByteDataModelElement('AnyByteDataModelElement')
])))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers
# later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler
syslog_event_handler = SyslogWriterEventHandler(analysis_context)
from aminer.events import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers = [
stream_printer_event_handler, syslog_event_handler,
mail_notification_handler
]
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.input import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis import WhitelistViolationDetector
whitelist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
whitelist_violation_detector = WhitelistViolationDetector(
analysis_context.aminer_config, whitelist_rules,
anomaly_event_handlers)
analysis_context.register_component(whitelist_violation_detector,
component_name="Whitelist")
atom_filter.add_handler(whitelist_violation_detector)
from aminer.analysis import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list), None)
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config,
'/model/DiskReport/Space',
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
2,
1,
0,
anomaly_event_handlers,
record_count_before_event=70000)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct AMiner how to process incoming data streams to create log atoms from them.
"""
date_format_string = b'%Y-%m-%d %H:%M:%S'
cron = b' cron['
# Build the parsing model:
from aminer.parsing import FirstMatchModelElement, SequenceModelElement, DecimalFloatValueModelElement, FixedDataModelElement, \
DelimitedDataModelElement, AnyByteDataModelElement, FixedWordlistDataModelElement, DecimalIntegerValueModelElement, \
DateTimeModelElement, IpAddressDataModelElement, Base64StringModelElement, ElementValueBranchModelElement, HexStringModelElement, \
MultiLocaleDateTimeModelElement, OptionalMatchModelElement, RepeatedElementDataModelElement, VariableByteDataModelElement, \
WhiteSpaceLimitedDataModelElement
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPU Workload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', date_format_string)
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', date_format_string),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', cron),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
service_children_audit = [
SequenceModelElement('path', [
FixedDataModelElement('type', b'type=PATH '),
FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'),
FixedDataModelElement('placeholder', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('item_string', b'): item='),
DecimalIntegerValueModelElement('item'),
FixedDataModelElement('name_string', b' name="'),
DelimitedDataModelElement('name', b'"'),
FixedDataModelElement('inode_string', b'" inode='),
DecimalIntegerValueModelElement('inode'),
FixedDataModelElement('dev_string', b' dev='),
DelimitedDataModelElement('dev', b' '),
FixedDataModelElement('mode_string', b' mode='),
DecimalIntegerValueModelElement('mode'),
FixedDataModelElement('ouid_string', b' ouid='),
DecimalIntegerValueModelElement('ouid'),
FixedDataModelElement('ogid_string', b' ogid='),
DecimalIntegerValueModelElement('ogid'),
FixedDataModelElement('rdev_string', b' rdev='),
DelimitedDataModelElement('rdev', b' '),
FixedDataModelElement('nametype_string', b' nametype='),
FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])
]),
SequenceModelElement('syscall', [
FixedDataModelElement('type', b'type=SYSCALL '),
FixedDataModelElement('msg_audit', b'msg=audit('),
DelimitedDataModelElement('msg', b':'),
FixedDataModelElement('placeholder', b':'),
DecimalIntegerValueModelElement('id'),
FixedDataModelElement('arch_string', b'): arch='),
DelimitedDataModelElement('arch', b' '),
FixedDataModelElement('syscall_string', b' syscall='),
DecimalIntegerValueModelElement('syscall'),
FixedDataModelElement('success_string', b' success='),
FixedWordlistDataModelElement('success', [b'yes', b'no']),
FixedDataModelElement('exit_string', b' exit='),
DecimalIntegerValueModelElement('exit'),
AnyByteDataModelElement('remainding_data')
])
]
service_children_parsing_model_element = [
DateTimeModelElement('DateTimeModelElement',
b'Current DateTime: %d.%m.%Y %H:%M:%S'),
DecimalFloatValueModelElement('DecimalFloatValueModelElement',
value_sign_type='optional'),
DecimalIntegerValueModelElement('DecimalIntegerValueModelElement',
value_sign_type='optional',
value_pad_type='blank'),
SequenceModelElement('', [
DelimitedDataModelElement('DelimitedDataModelElement', b';'),
FixedDataModelElement('FixedDataModelElement', b';')
])
]
# ElementValueBranchModelElement
fixed_data_me1 = FixedDataModelElement("fixed1", b'match ')
fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String')
fixed_wordlist_data_model_element = FixedWordlistDataModelElement(
"wordlist", [b'data: ', b'string: '])
decimal_integer_value_model_element = DecimalIntegerValueModelElement(
"decimal")
service_children_parsing_model_element.append(
ElementValueBranchModelElement(
'ElementValueBranchModelElement',
FirstMatchModelElement("first", [
SequenceModelElement(
"seq1",
[fixed_data_me1, fixed_wordlist_data_model_element]),
SequenceModelElement("seq2", [
fixed_data_me1, fixed_wordlist_data_model_element,
fixed_data_me2
])
]), "wordlist", {
0: decimal_integer_value_model_element,
1: fixed_data_me2
}))
service_children_parsing_model_element.append(
HexStringModelElement('HexStringModelElement'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
FixedDataModelElement('FixedDataModelElement',
b'Gateway IP-Address: '),
IpAddressDataModelElement('IpAddressDataModelElement')
]))
import locale
loc = locale.getlocale()
if loc == (None, None):
loc = ('en_US', 'utf8')
service_children_parsing_model_element.append(
MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement',
[(b'%b %d %Y', '%s.%s' %
(loc), None)]))
service_children_parsing_model_element.append(
RepeatedElementDataModelElement(
'RepeatedElementDataModelElement',
SequenceModelElement('SequenceModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'drawn number: '),
DecimalIntegerValueModelElement(
'DecimalIntegerValueModelElement')
]), 1))
service_children_parsing_model_element.append(
VariableByteDataModelElement('VariableByteDataModelElement', b'-@#'))
service_children_parsing_model_element.append(
SequenceModelElement('', [
WhiteSpaceLimitedDataModelElement(
'WhiteSpaceLimitedDataModelElement'),
FixedDataModelElement('', b' ')
]))
# The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches.
service_children_parsing_model_element.append(
Base64StringModelElement('Base64StringModelElement'))
# The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets to the
# AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted.
service_children_parsing_model_element.append(
OptionalMatchModelElement(
'OptionalMatchModelElement',
FirstMatchModelElement('FirstMatchModelElement', [
FixedDataModelElement('FixedDataModelElement',
b'The-searched-element-was-found!'),
SequenceModelElement('', [
FixedDataModelElement('FixedDME', b'Any:'),
AnyByteDataModelElement('AnyByteDataModelElement')
])
])))
alphabet = b'abcdef'
service_children_ecd = []
for _, char in enumerate(alphabet):
char = bytes([char])
service_children_ecd.append(FixedDataModelElement(char.decode(), char))
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address),
FirstMatchModelElement('type', service_children_audit),
FirstMatchModelElement('ECD', service_children_ecd),
FirstMatchModelElement('ParsingME',
service_children_parsing_model_element)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filter = AtomFilters.SubhandlerFilter(None)
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filter])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(analysis_context)
anomaly_event_handlers = [stream_printer_event_handler]
# Now define the AtomizerFactory using the model. A simple line
# based one is usually sufficient.
from aminer.input import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers,
default_timestamp_paths=["/model/DailyCron/DTM"])
# Just report all unparsed atoms to the event handlers.
from aminer.input import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filter.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=False)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="SimpleUnparsedHandler")
verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(
anomaly_event_handlers, parsing_model)
atom_filter.add_handler(verbose_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(
verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filter.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis import AllowlistViolationDetector
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config,
allowlist_rules,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filter.add_handler(allowlist_violation_detector)
from aminer.analysis import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None,
anomaly_event_handlers, 10)
analysis_context.register_component(parser_count,
component_name="ParserCount")
atom_filter.add_handler(parser_count)
from aminer.analysis.EventTypeDetector import EventTypeDetector
etd = EventTypeDetector(analysis_context.aminer_config,
anomaly_event_handlers)
analysis_context.register_component(etd,
component_name="EventTypeDetector")
atom_filter.add_handler(etd)
from aminer.analysis.VariableTypeDetector import VariableTypeDetector
vtd = VariableTypeDetector(analysis_context.aminer_config,
anomaly_event_handlers,
etd,
silence_output_except_indicator=False,
output_log_line=False)
analysis_context.register_component(vtd,
component_name="VariableTypeDetector")
atom_filter.add_handler(vtd)
from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector
vtd = VariableCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
etd,
disc_div_thres=0.5)
analysis_context.register_component(
vtd, component_name="VariableCorrelationDetector")
atom_filter.add_handler(vtd)
from aminer.analysis import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
check_rules_flag=True,
hypothesis_max_delta_time=1.0,
auto_include_flag=True)
analysis_context.register_component(
ecd, component_name="EventCorrelationDetector")
atom_filter.add_handler(ecd)
from aminer.analysis import MatchFilter
match_filter = MatchFilter(analysis_context.aminer_config,
['/model/Random'],
anomaly_event_handlers,
target_value_list=[1, 10, 100],
output_log_line=True)
analysis_context.register_component(match_filter,
component_name="MatchFilter")
atom_filter.add_handler(match_filter)
from aminer.analysis import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filter.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=True,
tuple_transformation_function=tuple_transformation_function,
output_log_line=True)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filter.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)],
10,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filter.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config,
'/model/RandomTime',
modulo_time_bin_definition,
10,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filter.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
None, ['/model/Random'],
100,
10,
output_log_line=True)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filter.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filter.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
output_log_line=True,
auto_include_flag=True)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filter.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
analysis_context.aminer_config,
['/model/type/path/name', '/model/type/syscall/syscall'],
anomaly_event_handlers,
id_path_list=['/model/type/path/id', '/model/type/syscall/id'],
min_allowed_time_diff=5,
auto_include_flag=True,
allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(
new_match_id_value_combo_detector,
component_name="NewMatchIdValueComboDetector")
atom_filter.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=True,
output_log_line=True)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filter.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config,
'/model/DiskReport/Space',
anomaly_event_handlers,
auto_include_flag=True,
default_interval=2,
realert_interval=5,
output_log_line=True)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filter.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=10000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filter.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config,
rules,
anomaly_event_handlers,
output_log_line=True)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filter.add_handler(time_correlation_violation_detector)
