def __init__(self, rule_id, min_time_delta, max_time_delta, max_artefacts_a_for_single_b=1, artefact_match_parameters=None):
"""Create the correlation rule.
@param artefact_match_parameters if not none, two artefacts A and B will be only treated as correlated when all the
parsed artefact attributes identified by the list of attribute path tuples match.
@param min_time_delta minimal delta in seconds, that artefact B may be observed after artefact A. Negative values are allowed
as artefact B may be found before A. """
self.rule_id = rule_id
self.min_time_delta = min_time_delta
self.max_time_delta = max_time_delta
self.max_artefacts_a_for_single_b = max_artefacts_a_for_single_b
self.artefact_match_parameters = artefact_match_parameters
self.history_a_events = []
self.history_b_events = []
self.last_timestamp_seen = 0.0
self.correlation_history = LogarithmicBackoffHistory(10)
def __init__(self, object_history=None, debug_match_result=False, match_action=None):
"""Create a DebugHistoryMatchRule object. @param object_history use this ObjectHistory to collect the
LogAtoms. When None, a default LogarithmicBackoffHistory for 10 items."""
if object_history is None:
object_history = LogarithmicBackoffHistory(10)
elif not isinstance(object_history, ObjectHistory):
raise Exception('objectHistory is not an instance of ObjectHistory')
self.object_history = object_history
self.debug_match_result = debug_match_result
self.match_action = match_action
def __init__(self, ruleId, minTimeDelta, maxTimeDelta, maxArtefactsAForSingleB=1,
artefactMatchParameters=None):
"""Create the correlation rule.
@param artefactMatchParameters if not none, two artefacts
A and B will be only treated as correlated when all the
parsed artefact attributes identified by the list of attribute
path tuples match.
@param minTimeDelta minimal delta in seconds, that artefact
B may be observed after artefact A. Negative values are allowed
as artefact B may be found before A.
"""
self.ruleId = ruleId
self.minTimeDelta = minTimeDelta
self.maxTimeDelta = maxTimeDelta
self.maxArtefactsAForSingleB = maxArtefactsAForSingleB
self.artefactMatchParameters = artefactMatchParameters
self.historyAEvents = []
self.historyBEvents = []
self.lastTimestampSeen = 0.0
self.correlationHistory = LogarithmicBackoffHistory(10)
def __init__(self,
object_history=None,
debug_match_result=False,
match_action=None):
"""
Create a DebugHistoryMatchRule object.
@param object_history use this ObjectHistory to collect the LogAtoms. When None, a default LogarithmicBackoffHistory for 10 items.
"""
if object_history is None:
object_history = LogarithmicBackoffHistory(10)
elif not isinstance(object_history, ObjectHistory):
msg = 'object_history is not an instance of ObjectHistory'
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
self.object_history = object_history
self.debug_match_result = debug_match_result
self.match_action = match_action
class CorrelationRule:
"""This class defines a correlation rule to match artefacts
A and B, where a hidden event A* always triggers at least one
artefact A and the the hidden event B*, thus triggering also
at least one artefact B."""
def __init__(self, ruleId, minTimeDelta, maxTimeDelta, maxArtefactsAForSingleB=1,
artefactMatchParameters=None):
"""Create the correlation rule.
@param artefactMatchParameters if not none, two artefacts
A and B will be only treated as correlated when all the
parsed artefact attributes identified by the list of attribute
path tuples match.
@param minTimeDelta minimal delta in seconds, that artefact
B may be observed after artefact A. Negative values are allowed
as artefact B may be found before A.
"""
self.ruleId = ruleId
self.minTimeDelta = minTimeDelta
self.maxTimeDelta = maxTimeDelta
self.maxArtefactsAForSingleB = maxArtefactsAForSingleB
self.artefactMatchParameters = artefactMatchParameters
self.historyAEvents = []
self.historyBEvents = []
self.lastTimestampSeen = 0.0
self.correlationHistory = LogarithmicBackoffHistory(10)
def updateArtefactA(self, selector, logAtom):
"""Append entry to the event history A."""
historyEntry = self.prepareHistoryEntry(selector, logAtom)
# FIXME: Check if event A could be discarded immediately.
self.historyAEvents.append(historyEntry)
def updateArtefactB(self, selector, logAtom):
"""Append entry to the event history B."""
historyEntry = self.prepareHistoryEntry(selector, logAtom)
# FIXME: Check if event B could be discarded immediately.
self.historyBEvents.append(historyEntry)
def checkStatus(self, newestTimestamp, maxViolations=20):
"""@return None if status is OK. Returns a tuple containing
a descriptive message and a list of violating log data lines
on error."""
# FIXME: This part of code would be good target to be implemented
# as native library with optimized algorithm in future.
aPos = 0
bPosStart = 0
for aPos in range(0, len(self.historyAEvents)):
aEvent = self.historyAEvents[aPos]
if aEvent is None:
continue
aEventTime = aEvent[0]
if newestTimestamp-aEventTime <= self.maxTimeDelta:
# This event is so new, that timewindow for related event has
# not expired yet.
t = datetime.fromtimestamp(time.time())
if newestTimestamp-aEventTime == 0 and \
aEventTime-(t-datetime.fromtimestamp(0)).total_seconds() <= self.maxTimeDelta:
violationLine = aEvent[3].matchElement.matchString
if isinstance(violationLine, bytes):
violationLine = violationLine.decode("utf-8")
violationMessage = 'FAIL: B-Event for \"%s\" (%s) not found!' % \
(violationLine, aEvent[2].actionId)
violationLogs = []
violationLogs.append(violationLine)
del self.historyAEvents[aPos]
return (violationMessage, violationLogs)
for bPos in range(bPosStart, len(self.historyBEvents)):
bEvent = self.historyBEvents[bPos]
if bEvent is None:
continue
bEventTime = bEvent[0]
delta = bEventTime-aEventTime
if delta < self.minTimeDelta:
# See if too early, if yes go to next element. As we will not
# check again any older aEvents in this loop, skip all bEvents
# up to this position in future runs.
bPosStart = bPos+1
if bPosStart == len(self.historyBEvents):
violationLine = aEvent[3].matchElement.matchString
if isinstance(violationLine, bytes):
violationLine = violationLine.decode("utf-8")
violationMessage = 'FAIL: B-Event for \"%s\" (%s) was found too early!' % \
(violationLine, aEvent[2].actionId)
violationLogs = []
violationLogs.append(violationLine)
del self.historyAEvents[aPos]
return (violationMessage, violationLogs)
continue
# Too late, no other bEvent may match this aEvent
if delta > self.maxTimeDelta:
violationLine = aEvent[3].matchElement.matchString
if isinstance(violationLine, bytes):
violationLine = violationLine.decode("utf-8")
violationMessage = 'FAIL: B-Event for \"%s\" (%s) was not found in time!' % \
(violationLine, aEvent[2].actionId)
violationLogs = []
violationLogs.append(violationLine)
del self.historyAEvents[aPos]
return (violationMessage, violationLogs)
break
# So time range is OK, see if match parameters are also equal.
checkPos = 4
for checkPos in range(4, len(aEvent)):
if aEvent[checkPos] != bEvent[checkPos]:
violationLine = aEvent[3].matchElement.matchString
if isinstance(violationLine, bytes):
violationLine = violationLine.decode("utf-8")
violationMessage = 'FAIL: \"%s\" (%s) %s is not equal %s' % ( \
violationLine, aEvent[2].actionId, aEvent[checkPos], bEvent[checkPos])
violationLogs = []
violationLogs.append(violationLine)
del self.historyAEvents[aPos]
return (violationMessage, violationLogs)
break
checkPos = checkPos+1
if checkPos != len(aEvent):
continue
# We found the match. Mark aEvent as done.
self.historyAEvents[aPos] = None
# See how many eEvents this bEvent might collect. Clean it also
# when limit was reached.
bEvent[1] += 1
if bEvent[1] == self.maxArtefactsAForSingleB:
self.historyBEvents[bPos] = None
# We want to keep a history of good matches to ease diagnosis
# of correlation failures. Keep information about current line
# for reference.
self.correlationHistory.addObject((aEvent[3].matchElement.matchString, aEvent[2].actionId, \
bEvent[3].matchElement.matchString, bEvent[2].actionId))
aPos += 1
break
# After checking all aEvents before aPos were cleared, otherwise
# they violate a correlation rule.
checkRange = aPos
violationLogs = []
violationMessage = ''
numViolations = 0
for aPos in range(0, checkRange):
aEvent = self.historyAEvents[aPos]
if aEvent is None:
continue
numViolations += 1
if numViolations > maxViolations:
continue
violationLine = aEvent[3].matchElement.matchString
if isinstance(violationLine, bytes):
violationLine = violationLine.decode("utf-8")
violationMessage += 'FAIL: \"%s\" (%s)' % (violationLine, aEvent[2].actionId)
violationLogs.append(violationLine)
if numViolations > maxViolations:
violationMessage += '... (%d more)\n' % (numViolations-maxViolations)
if numViolations != 0 and self.correlationHistory.getHistory():
violationMessage += 'Historic examples:\n'
for record in self.correlationHistory.getHistory():
violationMessage += ' "%s" (%s) ==> "%s" (%s)\n' % record
# Prune out all handled event records
self.historyAEvents = self.historyAEvents[checkRange:]
self.historyBEvents = self.historyBEvents[bPosStart:]
if numViolations == 0:
return None
return (violationMessage, violationLogs)
def prepareHistoryEntry(self, selector, logAtom):
"""Return a history entry for a parser match."""
parserMatch = logAtom.parserMatch
length = 4
if self.artefactMatchParameters is not None:
length += len(self.artefactMatchParameters)
result = [None]*length
result[0] = logAtom.getTimestamp()
result[1] = 0
result[2] = selector
result[3] = parserMatch
if result[0] is None:
result[0] = datetime.fromtimestamp(time.time())
if isinstance(result[0], datetime):
result[0] = (result[0]-datetime.fromtimestamp(0)).total_seconds()
if result[0] < self.lastTimestampSeen:
raise Exception('Unsorted!')
self.lastTimestampSeen = result[0]
if self.artefactMatchParameters is not None:
pos = 4
vDict = parserMatch.getMatchDictionary()
for artefactMatchParameter in self.artefactMatchParameters:
for paramPath in artefactMatchParameter:
matchElement = vDict.get(paramPath, None)
if matchElement is not None:
result[pos] = matchElement.matchObject
pos += 1
return result
def __init__(self, max_items):
"""Initialize the history component."""
LogarithmicBackoffHistory.__init__(self, max_items)
self.event_id = 0
class CorrelationRule:
"""This class defines a correlation rule to match artefacts A and B, where a hidden event A* always triggers at least one
artefact A and the the hidden event B*, thus triggering also at least one artefact B."""
def __init__(self, rule_id, min_time_delta, max_time_delta, max_artefacts_a_for_single_b=1, artefact_match_parameters=None):
"""Create the correlation rule.
@param artefact_match_parameters if not none, two artefacts A and B will be only treated as correlated when all the
parsed artefact attributes identified by the list of attribute path tuples match.
@param min_time_delta minimal delta in seconds, that artefact B may be observed after artefact A. Negative values are allowed
as artefact B may be found before A. """
self.rule_id = rule_id
self.min_time_delta = min_time_delta
self.max_time_delta = max_time_delta
self.max_artefacts_a_for_single_b = max_artefacts_a_for_single_b
self.artefact_match_parameters = artefact_match_parameters
self.history_a_events = []
self.history_b_events = []
self.last_timestamp_seen = 0.0
self.correlation_history = LogarithmicBackoffHistory(10)
def update_artefact_a(self, selector, log_atom):
"""Append entry to the event history A."""
history_entry = self.prepare_history_entry(selector, log_atom)
# Check if event A could be discarded immediately.
self.history_a_events.append(history_entry)
def update_artefact_b(self, selector, log_atom):
"""Append entry to the event history B."""
history_entry = self.prepare_history_entry(selector, log_atom)
# Check if event B could be discarded immediately.
self.history_b_events.append(history_entry)
def check_status(self, newest_timestamp, max_violations=20):
"""@return None if status is OK. Returns a tuple containing a descriptive message and a list of violating log data lines
on error."""
# This part of code would be good target to be implemented as native library with optimized algorithm in future.
a_pos = 0
check_range = len(self.history_a_events)
violation_logs = []
violation_message = ''
num_violations = 0
while a_pos < check_range:
deleted = False
check_range = len(self.history_a_events)
a_event = self.history_a_events[a_pos]
if a_event is None:
continue
a_event_time = a_event[0]
b_pos = 0
while b_pos < len(self.history_b_events):
b_event = self.history_b_events[b_pos]
if b_event is None:
continue
b_event_time = b_event[0]
delta = b_event_time - a_event_time
if delta < self.min_time_delta:
# See if too early, if yes go to next element. As we will not check again any older aEvents in this loop, skip
# all bEvents up to this position in future runs.
if b_pos < len(self.history_b_events):
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: B-Event for \"%s\" (%s) was found too early!\n' % (
violation_line, a_event[2].action_id)
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
break
continue
# Too late, no other b_event may match this a_event
if delta > self.max_time_delta:
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: B-Event for \"%s\" (%s) was not found in time!\n' % (
violation_line, a_event[2].action_id)
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
break
# So time range is OK, see if match parameters are also equal.
violation_found = False
for check_pos in range(4, len(a_event)):
if a_event[check_pos] != b_event[check_pos]:
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: \"%s\" (%s) %s is not equal %s\n' % (
violation_line, a_event[2].action_id, a_event[check_pos], b_event[check_pos])
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
violation_found = True
break
check_pos = check_pos + 1
if violation_found:
continue
# We want to keep a history of good matches to ease diagnosis of correlation failures. Keep information about current line
# for reference.
self.correlation_history.add_object((a_event[3].match_element.match_string, a_event[2].action_id,
b_event[3].match_element.match_string, b_event[2].action_id))
del self.history_a_events[a_pos]
del self.history_b_events[b_pos]
deleted = True
check_range = check_range - 1
b_pos = b_pos + 1
if deleted is False:
a_pos = a_pos + 1
# After checking all aEvents before a_pos were cleared, otherwise they violate a correlation rule.
for a_pos in range(0, check_range):
a_event = self.history_a_events[a_pos]
if a_event is None:
continue
delta = newest_timestamp - a_event[0]
if delta > self.max_time_delta:
violation_line = a_event[3].match_element.match_string
if isinstance(violation_line, bytes):
violation_line = violation_line.decode()
if num_violations <= max_violations:
violation_message += 'FAIL: B-Event for \"%s\" (%s) was not found in time!\n' % (
violation_line, a_event[2].action_id)
violation_logs.append(violation_line)
del self.history_a_events[a_pos]
deleted = True
check_range = check_range - 1
num_violations = num_violations + 1
break
if num_violations > max_violations:
violation_message += '... (%d more)\n' % (num_violations - max_violations)
if num_violations != 0 and len(self.correlation_history.get_history()) > 0:
violation_message += 'Historic examples:\n'
for record in self.correlation_history.get_history():
violation_message += ' "%s" (%s) ==> "%s" (%s)\n' % (record[0].decode(), record[1], record[2].decode(), record[3])
if num_violations == 0:
return None
return violation_message, violation_logs
def prepare_history_entry(self, selector, log_atom):
"""Return a history entry for a parser match."""
parser_match = log_atom.parser_match
timestamp = log_atom.get_timestamp()
if timestamp is None:
timestamp = time.time()
length = 4
if self.artefact_match_parameters is not None:
length += len(self.artefact_match_parameters)
result = [None] * length
result[0] = timestamp
result[1] = 0
result[2] = selector
result[3] = parser_match
if result[0] < self.last_timestamp_seen:
raise Exception('Unsorted!')
self.last_timestamp_seen = result[0]
if self.artefact_match_parameters is not None:
pos = 4
v_dict = parser_match.get_match_dictionary()
for artefact_match_parameter in self.artefact_match_parameters:
for param_path in artefact_match_parameter:
match_element = v_dict.get(param_path, None)
if match_element is not None:
result[pos] = match_element.match_object
pos += 1
return result