def __init__(self, rule_id, min_time_delta, max_time_delta, max_artefacts_a_for_single_b=1, artefact_match_parameters=None):
"""
Create the correlation rule.
@param artefact_match_parameters if not none, two artefacts A and B will be only treated as correlated when all the
parsed artefact attributes identified by the list of attribute path tuples match.
@param min_time_delta minimal delta in seconds, that artefact B may be observed after artefact A. Negative values are allowed
as artefact B may be found before A.
"""
self.rule_id = rule_id
self.min_time_delta = min_time_delta
self.max_time_delta = max_time_delta
self.max_artefacts_a_for_single_b = max_artefacts_a_for_single_b
self.artefact_match_parameters = artefact_match_parameters
self.history_a_events = []
self.history_b_events = []
self.last_timestamp_seen = 0.0
self.correlation_history = LogarithmicBackoffHistory(10)
def __init__(self, object_history=None, debug_match_result=False, match_action=None):
"""
Create a DebugHistoryMatchRule object.
@param object_history use this ObjectHistory to collect the LogAtoms. When None, a default LogarithmicBackoffHistory for 10 items.
"""
if object_history is None:
object_history = LogarithmicBackoffHistory(10)
elif not isinstance(object_history, ObjectHistory):
msg = 'object_history is not an instance of ObjectHistory'
logging.getLogger(DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
self.object_history = object_history
self.debug_match_result = debug_match_result
self.match_action = match_action
def __init__(self, max_items):
"""Initialize the history component."""
LogarithmicBackoffHistory.__init__(self, max_items)
self.event_id = 0
class CorrelationRule:
"""
This class defines a correlation rule to match artefacts A and B.
A hidden event A* always triggers at least one artefact A and the the hidden event B*, thus triggering also at least one artefact B.
"""
def __init__(self,
rule_id,
min_time_delta,
max_time_delta,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=None):
"""
Create the correlation rule.
@param artefact_match_parameters if not none, two artefacts A and B will be only treated as correlated when all the
parsed artefact attributes identified by the list of attribute path tuples match.
@param min_time_delta minimal delta in seconds, that artefact B may be observed after artefact A. Negative values are allowed
as artefact B may be found before A.
"""
self.rule_id = rule_id
self.min_time_delta = min_time_delta
self.max_time_delta = max_time_delta
self.max_artefacts_a_for_single_b = max_artefacts_a_for_single_b
self.artefact_match_parameters = artefact_match_parameters
self.history_a_events = []
self.history_b_events = []
self.last_timestamp_seen = 0.0
self.correlation_history = LogarithmicBackoffHistory(10)
def update_artefact_a(self, selector, log_atom):
"""Append entry to the event history A."""
history_entry = self.prepare_history_entry(selector, log_atom)
# Check if event A could be discarded immediately.
self.history_a_events.append(history_entry)
def update_artefact_b(self, selector, log_atom):
"""Append entry to the event history B."""
history_entry = self.prepare_history_entry(selector, log_atom)
# Check if event B could be discarded immediately.
self.history_b_events.append(history_entry)
def check_status(self, newest_timestamp, max_violations=20):
"""@return None if status is OK. Return a tuple containing a descriptive message and a list of violating log data lines on error."""
# This part of code would be good target to be implemented as native library with optimized algorithm in future.
a_pos = 0
check_range = len(self.history_a_events)
violation_logs = []
violation_message = ''
num_violations = 0
while a_pos < check_range:
deleted = False
check_range = len(self.history_a_events)
a_event = self.history_a_events[a_pos]
if a_event is None:
continue
a_event_time = a_event[0]
b_pos = 0
while b_pos < len(self.history_b_events):
b_event = self.history_b_events[b_pos]
if b_event is None:
continue
b_event_time = b_event[0]
delta = b_event_time - a_event_time
if delta < self.min_time_delta:
# See if too early, if yes go to next element. As we will not check again any older aEvents in this loop, skip
# all bEvents up to this position in future runs.
if b_pos < len(self.history_b_events):
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: B-Event for \"%s\" (%s) was found too early!\n' % (
violation_line, a_event[2].action_id)
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
break
continue
# Too late, no other b_event may match this a_event
if delta > self.max_time_delta:
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: B-Event for \"%s\" (%s) was not found in time!\n' % (
violation_line, a_event[2].action_id)
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
break
# So time range is OK, see if match parameters are also equal.
violation_found = False
for check_pos in range(4, len(a_event)): # skipcq: PTC-W0060
if a_event[check_pos] != b_event[check_pos]:
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: \"%s\" (%s) %s is not equal %s\n' % (
violation_line, a_event[2].action_id,
a_event[check_pos], b_event[check_pos])
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
violation_found = True
break
check_pos = check_pos + 1
if violation_found:
continue
# We want to keep a history of good matches to ease diagnosis of correlation failures. Keep information about current line
# for reference.
self.correlation_history.add_object(
(a_event[3].match_element.match_string,
a_event[2].action_id,
b_event[3].match_element.match_string,
b_event[2].action_id))
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
b_pos = b_pos + 1
if deleted is False:
a_pos = a_pos + 1
# After checking all aEvents before a_pos were cleared, otherwise they violate a correlation rule.
for a_pos in range(0, check_range):
a_event = self.history_a_events[a_pos]
if a_event is None:
continue
delta = newest_timestamp - a_event[0]
if delta > self.max_time_delta:
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: B-Event for \"%s\" (%s) was not found in time!\n' % (
violation_line, a_event[2].action_id)
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
break
if num_violations > max_violations:
violation_message += '... (%d more)\n' % (num_violations -
max_violations)
if num_violations != 0 and len(
self.correlation_history.get_history()) > 0:
violation_message += 'Historic examples:\n'
for record in self.correlation_history.get_history():
violation_message += ' "%s" (%s) ==> "%s" (%s)\n' % (
record[0].decode(), record[1], record[2].decode(),
record[3])
if num_violations == 0:
return None
return violation_message, violation_logs
def prepare_history_entry(self, selector, log_atom):
"""Return a history entry for a parser match."""
parser_match = log_atom.parser_match
timestamp = log_atom.get_timestamp()
if timestamp is None:
timestamp = time.time()
length = 4
if self.artefact_match_parameters is not None:
length += len(self.artefact_match_parameters)
result = [None] * length
result[0] = timestamp
result[1] = 0
result[2] = selector
result[3] = parser_match
if result[0] < self.last_timestamp_seen:
msg = 'Timestamps unsorted!'
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
self.last_timestamp_seen = result[0]
if self.artefact_match_parameters is not None:
pos = 4
v_dict = parser_match.get_match_dictionary()
for artefact_match_parameter in self.artefact_match_parameters:
for param_path in artefact_match_parameter:
match_element = v_dict.get(param_path, None)
if match_element is not None:
result[pos] = match_element.match_object
pos += 1
return result