def run(self, records):
"""Run rules against the records sent from the Classifier function
Args:
records (list): Dictionaries of records sent from the classifier function
Record Format:
{
'cluster': 'prod',
'log_schema_type': 'cloudwatch:cloudtrail',
'record': {
'key': 'value'
},
'service': 'kinesis',
'resource': 'kinesis_stream_name'
'data_type': 'json'
}
Returns:
list: Alerts that have been triggered by this data
"""
LOGGER.info('Processing %d records', len(records))
# Extract any threat intelligence matches from the records
self._extract_threat_intel(records)
alerts = []
for payload in records:
rules = Rule.rules_for_log_type(payload['log_schema_type'])
if not rules:
LOGGER.debug('No rules to process for %s', payload)
continue
for rule in rules:
# subkey check
if not self._process_subkeys(payload['record'], rule):
continue
# matcher check
if not rule.check_matchers(payload['record']):
continue
alert = self._rule_analysis(payload, rule)
if alert:
alerts.append(alert)
self._alert_forwarder.send_alerts(alerts)
# Only log rule info here if this is deployed in Lambda
# During testing, this gets logged at the end and printing here could be confusing
# since stress testing calls this method multiple times
if self._in_lambda:
LOGGER.info(get_rule_stats(True))
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS,
len(alerts))
return alerts
def run(self, input_payload):
"""Process rules on a record.
Gather a list of rules based on the record's datasource type.
For each rule, evaluate the record through all listed matchers
and the rule itself to determine if a match occurs.
Returns:
A tuple(list, list).
First return is a list of Alert instances.
Second return is a list of payload instance with normalized records.
"""
alerts = []
# store normalized records for future process in Threat Intel
normalized_records = []
payload = copy(input_payload)
rules = Rule.rules_for_log_type(payload.log_source)
if not rules:
LOGGER.debug('No rules to process for %s', payload)
return alerts, normalized_records
# fetch all datatypes info from rules and run data normalization before
# rule match to improve performance. So one record will be normalized only
# once by all normalized datatypes from all rules.
datatypes_set = {
datatype
for rule in rules if rule.datatypes for datatype in rule.datatypes
}
if datatypes_set:
for record in payload.records:
self._apply_normalization(record, normalized_records,
datatypes_set, payload)
for record in payload.records:
for rule in rules:
# subkey check
if not self.process_subkeys(record, payload.type, rule):
continue
# matcher check
if not rule.check_matchers(record):
continue
self.rule_analysis(record, rule, payload, alerts)
return alerts, normalized_records
def run(self, input_payload):
"""Process rules on a record.
Gather a list of rules based on the record's datasource type.
For each rule, evaluate the record through all listed matchers
and the rule itself to determine if a match occurs.
Returns:
A tuple(list, list).
First return is a list of Alert instances.
Second return is a list of payload instance with normalized records.
"""
alerts = []
# store normalized records for future process in Threat Intel
normalized_records = []
payload = copy(input_payload)
rules = Rule.rules_for_log_type(payload.log_source)
if not rules:
LOGGER.debug('No rules to process for %s', payload)
return alerts, normalized_records
for record in payload.records:
for rule in rules:
# subkey check
if not self.process_subkeys(record, payload.type, rule):
continue
# matcher check
if not rule.check_matchers(record):
continue
if rule.datatypes:
# When rule 'datatypes' option is defined, rules engine will
# apply data normalization to all the record.
record_copy = self._apply_normalization(
record, normalized_records, rule, payload)
self.rule_analysis(record_copy, rule, payload, alerts)
else:
self.rule_analysis(record, rule, payload, alerts)
return alerts, normalized_records