def run(self, records):
"""Run rules against the records sent from the Classifier function
Args:
records (list): Dictionaries of records sent from the classifier function
Record Format:
{
'cluster': 'prod',
'log_schema_type': 'cloudwatch:cloudtrail',
'record': {
'key': 'value'
},
'service': 'kinesis',
'resource': 'kinesis_stream_name'
'data_type': 'json'
}
Returns:
list: Alerts that have been triggered by this data
"""
LOGGER.info('Processing %d records', len(records))
# Extract any threat intelligence matches from the records
self._extract_threat_intel(records)
alerts = []
for payload in records:
rules = Rule.rules_for_log_type(payload['log_schema_type'])
if not rules:
LOGGER.debug('No rules to process for %s', payload)
continue
for rule in rules:
# subkey check
if not self._process_subkeys(payload['record'], rule):
continue
# matcher check
if not rule.check_matchers(payload['record']):
continue
alert = self._rule_analysis(payload, rule)
if alert:
alerts.append(alert)
self._alert_forwarder.send_alerts(alerts)
# Only log rule info here if this is deployed in Lambda
# During testing, this gets logged at the end and printing here could be confusing
# since stress testing calls this method multiple times
if self._in_lambda:
LOGGER.info(get_rule_stats(True))
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS,
len(alerts))
return alerts
def run(self, input_payload):
"""Process rules on a record.
Gather a list of rules based on the record's datasource type.
For each rule, evaluate the record through all listed matchers
and the rule itself to determine if a match occurs.
Returns:
A tuple(list, list).
First return is a list of Alert instances.
Second return is a list of payload instance with normalized records.
"""
alerts = []
# store normalized records for future process in Threat Intel
normalized_records = []
payload = copy(input_payload)
rules = Rule.rules_for_log_type(payload.log_source)
if not rules:
LOGGER.debug('No rules to process for %s', payload)
return alerts, normalized_records
# fetch all datatypes info from rules and run data normalization before
# rule match to improve performance. So one record will be normalized only
# once by all normalized datatypes from all rules.
datatypes_set = {
datatype
for rule in rules if rule.datatypes for datatype in rule.datatypes
}
if datatypes_set:
for record in payload.records:
self._apply_normalization(record, normalized_records,
datatypes_set, payload)
for record in payload.records:
for rule in rules:
# subkey check
if not self.process_subkeys(record, payload.type, rule):
continue
# matcher check
if not rule.check_matchers(record):
continue
self.rule_analysis(record, rule, payload, alerts)
return alerts, normalized_records
def run(self, input_payload):
"""Process rules on a record.
Gather a list of rules based on the record's datasource type.
For each rule, evaluate the record through all listed matchers
and the rule itself to determine if a match occurs.
Returns:
A tuple(list, list).
First return is a list of Alert instances.
Second return is a list of payload instance with normalized records.
"""
alerts = []
# store normalized records for future process in Threat Intel
normalized_records = []
payload = copy(input_payload)
rules = Rule.rules_for_log_type(payload.log_source)
if not rules:
LOGGER.debug('No rules to process for %s', payload)
return alerts, normalized_records
for record in payload.records:
for rule in rules:
# subkey check
if not self.process_subkeys(record, payload.type, rule):
continue
# matcher check
if not rule.check_matchers(record):
continue
if rule.datatypes:
# When rule 'datatypes' option is defined, rules engine will
# apply data normalization to all the record.
record_copy = self._apply_normalization(
record, normalized_records, rule, payload)
self.rule_analysis(record_copy, rule, payload, alerts)
else:
self.rule_analysis(record, rule, payload, alerts)
return alerts, normalized_records
def threat_intel_match(self, payload_with_normalized_records):
"""Apply Threat Intelligence on normalized records
Args:
payload_with_normalized_records (list): A list of payload instances.
And it pre_parsed_record is replaced by normalized record. The
reason to pass a copy of payload into Threat Intelligence is because
alerts require to include payload metadata (payload.log_source,
payload.type, payload.service and payload.entity).
Returns:
list: A list of Alerts triggered by Threat Intelligence.
"""
alerts = []
if self._threat_intel:
ioc_records = self._threat_intel.threat_detection(
payload_with_normalized_records)
rules = Rule.rules_with_datatypes()
if ioc_records:
for ioc_record in ioc_records:
for rule in rules:
self.rule_analysis(ioc_record.pre_parsed_record, rule,
ioc_record, alerts)
return alerts
def test_process_subkeys_nested_records(self):
"""Rules Engine - Required Subkeys with Nested Records"""
def cloudtrail_us_east_logs(rec):
return ('us-east' in rec['awsRegion']
and 'AWS' in rec['requestParameters']['program'])
rule_attrs = Rule(cloudtrail_us_east_logs,
rule_name='cloudtrail_us_east_logs',
matchers=[],
datatypes=[],
logs=['test_log_type_json_nested'],
merge_by_keys=[],
merge_window_mins=0,
outputs=['s3:sample_bucket'],
req_subkeys={'requestParameters': ['program']},
context={})
data = json.dumps({
'Records': [
{
'eventVersion': '1.05',
'eventID': '2',
'eventTime': '3',
'requestParameters': {
'program': 'AWS CLI'
},
'eventType': 'CreateSomeResource',
'responseElements': 'Response',
'awsRegion': 'us-east-1',
'eventName': 'CreateResource',
'userIdentity': {
'name': 'john',
'key': 'AVC124313414'
},
'eventSource': 'Kinesis',
'requestID': '12345',
'userAgent': 'AWS CLI v1.3109',
'sourceIPAddress': '127.0.0.1',
'recipientAccountId': '123456123456'
},
{
'eventVersion': '1.05',
'eventID': '2',
'eventTime': '3',
'requestParameters': {
'program': 'AWS UI'
},
'eventType': 'CreateSomeOtherResource',
'responseElements': 'Response',
'awsRegion': 'us-east-2',
'eventName': 'CreateResource',
'userIdentity': {
'name': 'ann',
'key': 'AD114313414'
},
'eventSource': 'Lambda',
'requestID': '12345',
'userAgent': 'Google Chrome 42',
'sourceIPAddress': '127.0.0.2',
'recipientAccountId': '123456123456'
},
{
'eventVersion': '1.05',
'eventID': '2',
'eventTime': '3',
# Translates from null in JSON to None in Python
'requestParameters': None,
'eventType': 'CreateSomeResource',
'responseElements': 'Response',
'awsRegion': 'us-east-1',
'eventName': 'CreateResource',
'userIdentity': {
'name': 'john',
'key': 'AVC124313414'
},
'eventSource': 'Kinesis',
'requestID': '12345',
'userAgent': 'AWS CLI',
'sourceIPAddress': '127.0.0.1',
'recipientAccountId': '123456123456'
}
]
})
schema = self.config['logs']['test_cloudtrail']['schema']
options = self.config['logs']['test_cloudtrail']['configuration']
parser_class = get_parser('json')
parser = parser_class(options)
parsed_result = parser.parse(schema, data)
valid_record = [
rec for rec in parsed_result
if rec['requestParameters'] is not None
][0]
valid_subkey_check = RulesEngine.process_subkeys(
valid_record, 'json', rule_attrs)
assert_true(valid_subkey_check)
invalid_record = [
rec for rec in parsed_result if rec['requestParameters'] is None
][0]
invalid_subkey_check = RulesEngine.process_subkeys(
invalid_record, 'json', rule_attrs)
assert_false(invalid_subkey_check)
def local_rule_names(self):
"""Names of locally loaded rules"""
return set(Rule.rule_names())