def right_to_left_character(rec):
"""
author: @javutin
description: Malicious files can be disguised by using a encoding trick that uses the
unicode character U+202E, also known as right-to-left-override (RLO).
The trick hides a potentially malicious extension and makes them appear
harmless.
reference: https://krebsonsecurity.com/2011/09/right-to-left-override-aids-email-attacks
playbook: (a) verify the file has an RTLO character and that the file is malicious
ATT&CK Tactic: Defense Evasion
ATT&CK Technique: Obfuscated Files or Information
ATT&CK URL: https://attack.mitre.org/wiki/Technique/T1027
"""
# Unicode character U+202E, right-to-left-override (RLO)
rlo = '\u202e'
commands = Normalizer.get_values_for_normalized_type(rec, 'command')
for command in commands:
if isinstance(command, str) and rlo in command:
return True
paths = Normalizer.get_values_for_normalized_type(rec, 'path')
for path in paths:
if isinstance(path, str) and rlo in path:
return True
file_names = Normalizer.get_values_for_normalized_type(rec, 'file_name')
for file_name in file_names:
if isinstance(file_name, str) and rlo in file_name:
return True
return False
def test_normalize_corner_case(self):
"""Normalizer - Normalize - Corner Case"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'normalized_key':
NormalizedType(log_type, 'normalized_key',
['original_key', 'original_key']),
'account':
self._normalized_type_account()
}
}
record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
}
}
Normalizer.normalize(record, log_type)
expected_record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
},
'streamalert_normalization': {
'streamalert_record_id': MOCK_RECORD_ID,
'normalized_key': [{
'values': ['fizzbuzz'],
'function': None
}]
}
}
assert_equal(record, expected_record)
def test_normalize_none_defined(self, log_mock):
"""Normalizer - Normalize, No Types Defined"""
log_type = 'cloudtrail'
Normalizer._types_config = {}
Normalizer.normalize(self._test_record(), log_type)
log_mock.assert_called_with(
'No normalized types defined for log type: %s', log_type)
def test_normalize(self):
"""Normalizer - Normalize"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'region': {'region', 'awsRegion'},
'sourceAccount': {'account', 'accountId'}
}
}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert:normalization': {
'region': ['region_name'],
'sourceAccount': [123456]
}
}
assert_equal(record, expected_record)
def test_normalize_corner_case(self):
"""Normalizer - Normalize - Corner Case"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'normalized_key': {'normalized_key', 'original_key'},
'sourceAccount': {'account', 'accountId'}
}
}
record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
}
}
Normalizer.normalize(record, log_type)
expected_record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
},
'streamalert:normalization': {
'normalized_key': ['fizzbuzz']
}
}
assert_equal(record, expected_record)
def _classify_payload(self, payload):
"""Run the payload through the classification logic to determine the data type
Args:
payload (StreamPayload): StreamAlert payload object being processed
"""
# Get logs defined for the service/entity in the config
logs_config = self._load_logs_for_resource(payload.service(), payload.resource)
if not logs_config:
LOGGER.error(
'No log types defined for resource [%s] in sources configuration for service [%s]',
payload.resource,
payload.service()
)
return
for record in payload.pre_parse():
# Increment the processed size using the length of this record
self._processed_size += len(record)
# Get the parser for this data
self._process_log_schemas(record, logs_config)
LOGGER.debug('Parsed and classified payload: %s', bool(record))
payload.fully_classified = payload.fully_classified and record
if not record:
self._log_bad_records(record, 1)
continue
LOGGER.debug(
'Classified %d record(s) with schema: %s',
len(record.parsed_records),
record.log_schema_type
)
# Even if the parser was successful, there's a chance it
# could not parse all records, so log them here as invalid
self._log_bad_records(record, len(record.invalid_records))
for parsed_rec in record.parsed_records:
#
# In Normalization v1, the normalized types are defined based on log source
# (e.g. osquery, cloudwatch etc) and this will be deprecated.
# In Normalization v2, the normalized types are defined based on log type
# (e.g. osquery:differential, cloudwatch:cloudtrail, cloudwatch:events etc)
#
Normalizer.normalize(parsed_rec, record.log_schema_type)
self._payloads.append(record)
def test_key_does_not_exist(self):
"""Normalizer - Normalize, Key Does Not Exist"""
test_record = {'accountId': 123456, 'region': 'region_name'}
normalized_types = {
'region': self._normalized_type_region(),
'account': NormalizedType('test_log_type', 'account',
['accountId']),
# There is no IP value in record, so normalization should not include this
'ipv4': self._normalized_type_ip()
}
expected_results = {
'streamalert_record_id': MOCK_RECORD_ID,
'account': [{
'values': ['123456'],
'function': None
}],
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}]
}
results = Normalizer.match_types(test_record, normalized_types)
assert_equal(results, expected_results)
def test_load_from_config(self):
"""Normalizer - Load From Config"""
config = {
'logs': {
'cloudtrail': {
'schema': {},
'configuration': {
'normalization': {
'region': ['path', 'to', 'awsRegion'],
'sourceAccount': ['path', 'to', 'accountId']
}
}
}
}
}
normalizer = Normalizer.load_from_config(config)
expected_config = {
'cloudtrail': {
'region':
NormalizedType('cloudtrail', 'region',
['path', 'to', 'awsRegion']),
'sourceAccount':
NormalizedType('cloudtrail', 'sourceAccount',
['path', 'to', 'accountId'])
}
}
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_match_types(self):
"""Normalizer - Match Types"""
normalized_types = {
'region': self._normalized_type_region(),
'account': self._normalized_type_account(),
'ipv4': self._normalized_type_ip()
}
expected_results = {
'streamalert_record_id':
MOCK_RECORD_ID,
'account': [{
'values': ['123456'],
'function': None
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}, {
'values': ['1.1.1.2'],
'function': 'source ip address'
}],
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}, {
'values': ['region_name'],
'function': 'AWS region'
}]
}
results = Normalizer.match_types(self._test_record(), normalized_types)
assert_equal(results, expected_results)
def test_load_from_config_from_log_conf(self):
"""Normalizer - Load normalization config from "logs" field in the config"""
config = {
'logs': {
'cloudwatch:events': {
'schema': {
'account': 'string',
'source': 'string',
'key': 'string'
},
'parser': 'json',
'configuration': {
'normalization': {
'event_name': ['detail', 'eventName'],
'region': [{
'path': ['region'],
'function': 'aws region information'
}, {
'path': ['detail', 'awsRegion'],
'function': 'aws region information'
}],
'ip_address': [{
'path': ['detail', 'sourceIPAddress'],
'function':
'source ip address'
}]
}
}
}
}
}
expected_config = {
'cloudwatch:events': {
'event_name':
NormalizedType('cloudwatch:events', 'event_name',
['detail', 'eventName']),
'region':
NormalizedType('cloudwatch:events', 'region',
[{
'path': ['region'],
'function': 'aws region information'
}, {
'path': ['detail', 'awsRegion'],
'function': 'aws region information'
}]),
'ip_address':
NormalizedType('cloudwatch:events', 'ip_address',
[{
'path': ['detail', 'sourceIPAddress'],
'function': 'source ip address'
}])
}
}
normalizer = Normalizer.load_from_config(config)
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_normalize(self):
"""Normalizer - Normalize"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'region': self._normalized_type_region(),
'ipv4': self._normalized_type_ip()
}
}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert_normalization': {
'streamalert_record_id':
MOCK_RECORD_ID,
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}, {
'values': ['region_name'],
'function': 'AWS region'
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}, {
'values': ['1.1.1.2'],
'function': 'source ip address'
}]
}
}
assert_equal(record, expected_record)
def test_load_from_config_with_flag(self):
"""Normalizer - Load From Config with send_to_artifacts flag"""
config = {
'logs': {
'cloudwatch:flow_logs': {
'schema': {
'source': 'string',
'destination': 'string',
'destport': 'string'
},
'configuration': {
'normalization': {
'ip_address': [{
'path': ['destination'],
'function':
'Destination IP addresses'
}],
'port': [{
'path': ['destport'],
'function': 'Destination port number',
'send_to_artifacts': False
}]
}
}
}
}
}
normalizer = Normalizer.load_from_config(config)
record = {
'source': '1.1.1.2',
'destination': '2.2.2.2',
'destport': '54321'
}
normalizer.normalize(record, 'cloudwatch:flow_logs')
expect_result = {
'source': '1.1.1.2',
'destination': '2.2.2.2',
'destport': '54321',
'streamalert_normalization': {
'streamalert_record_id':
MOCK_RECORD_ID,
'ip_address': [{
'values': ['2.2.2.2'],
'function': 'Destination IP addresses'
}],
'port': [{
'values': ['54321'],
'function': 'Destination port number',
'send_to_artifacts': False
}]
}
}
assert_equal(record, expect_result)
def test_get_values_for_normalized_type_none(self):
"""Normalizer - Get Values for Normalized Type, None"""
record = {
'sourceIPAddress': '1.1.1.3',
'streamalert_normalization': {}
}
assert_equal(
Normalizer.get_values_for_normalized_type(record, 'ip_v4'), set())
def __init__(self):
# Create some objects to be cached if they have not already been created
Classifier._config = Classifier._config or config.load_config(validate=True)
Classifier._firehose_client = (
Classifier._firehose_client or FirehoseClient.load_from_config(
prefix=self.config['global']['account']['prefix'],
firehose_config=self.config['global'].get('infrastructure', {}).get('firehose', {}),
log_sources=self.config['logs']
)
)
Classifier._sqs_client = Classifier._sqs_client or SQSClient()
# Setup the normalization logic
Normalizer.load_from_config(self.config)
self._cluster = os.environ['CLUSTER']
self._payloads = []
self._failed_record_count = 0
self._processed_size = 0
def test_get_values_for_normalized_type(self):
"""Normalizer - Get Values for Normalized Type"""
expected_result = {'1.1.1.3'}
record = {
'sourceIPAddress': '1.1.1.3',
'streamalert:normalization': {
'ip_v4': expected_result,
}
}
assert_equal(
Normalizer.get_values_for_normalized_type(record, 'ip_v4'),
expected_result)
def test_match_types_list(self):
"""Normalizer - Match Types, List of Values"""
normalized_types = {
'ipv4': ['sourceIPAddress'],
}
expected_results = {'ipv4': ['1.1.1.2', '1.1.1.3']}
test_record = {
'account': 123456,
'sourceIPAddress': ['1.1.1.2', '1.1.1.3']
}
results = Normalizer.match_types(test_record, normalized_types)
assert_equal(results, expected_results)
def test_match_types(self):
"""Normalizer - Match Types"""
normalized_types = {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId'],
'ipv4': ['destination', 'source', 'sourceIPAddress']
}
expected_results = {
'sourceAccount': [123456],
'ipv4': ['1.1.1.2', '1.1.1.3'],
'region': ['region_name']
}
results = Normalizer.match_types(self._test_record(), normalized_types)
assert_equal(results, expected_results)
def test_empty_value(self):
"""Normalizer - Normalize, Empty Value"""
test_record = {
'account': 123456,
'region': '' # This value is empty so should not be stored
}
normalized_types = {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId'],
'ipv4': ['sourceIPAddress']
}
expected_results = {'sourceAccount': [123456]}
results = Normalizer.match_types(test_record, normalized_types)
assert_equal(results, expected_results)
def test_key_does_not_exist(self):
"""Normalizer - Normalize, Key Does Not Exist"""
test_record = {'accountId': 123456, 'region': 'region_name'}
normalized_types = {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId'],
# There is no IP value in record, so normalization should not include this
'ipv4': ['sourceIPAddress']
}
expected_results = {
'sourceAccount': [123456],
'region': ['region_name']
}
results = Normalizer.match_types(test_record, normalized_types)
assert_equal(results, expected_results)
def test_match_types_multiple(self):
"""Normalizer - Match Types, Mutiple Sub-keys"""
normalized_types = {
'account': ['account'],
'region': ['region', 'awsRegion'],
'ipv4': ['destination', 'source', 'sourceIPAddress'],
'userName': ['userName', 'owner', 'invokedBy']
}
expected_results = {
'account': [123456],
'ipv4': ['1.1.1.2', '1.1.1.3'],
'region': ['region_name'],
'userName': ['Alice', 'signin.amazonaws.com']
}
results = Normalizer.match_types(self._test_record(), normalized_types)
assert_equal(results, expected_results)
def test_load_from_config_deprecate_normalized_types(self):
"""Normalizer - Load normalization config and deprecate conf/normalized_types.json
"""
config = {
'logs': {
'cloudwatch:events': {
'schema': {
'account': 'string',
'source': 'string',
'key': 'string'
},
'parser': 'json',
'configuration': {
'normalization': {
'ip_address': [{
'path': ['path', 'to', 'sourceIPAddress'],
'function':
'source ip address'
}]
}
}
},
'other_log_type': {}
},
'normalized_types': {
'cloudwatch': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
}
expected_config = {
'cloudwatch:events': {
'ip_address':
NormalizedType('cloudwatch:events', 'ip_address',
[{
'path': ['path', 'to', 'sourceIPAddress'],
'function': 'source ip address'
}])
}
}
normalizer = Normalizer.load_from_config(config)
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_load_from_config(self):
"""Normalizer - Load From Config"""
config = {
'normalized_types': {
'cloudtrail': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
}
normalizer = Normalizer.load_from_config(config)
expected_config = {
'cloudtrail': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_match_condition(self):
"""Normalizer - Test match condition with different conditions"""
record = self._test_record()
condition = {'path': ['account'], 'is': '123456'}
assert_true(Normalizer._match_condition(record, condition))
condition = {'path': ['account'], 'is_not': '123456'}
assert_false(Normalizer._match_condition(record, condition))
condition = {'path': ['detail', 'awsRegion'], 'contains': 'region'}
assert_true(Normalizer._match_condition(record, condition))
condition = {'path': ['detail', 'awsRegion'], 'contains': 'not_region'}
assert_false(Normalizer._match_condition(record, condition))
condition = {
'path': ['detail', 'userIdentity', 'userName'],
'not_contains': 'alice'
}
assert_false(Normalizer._match_condition(record, condition))
condition = {'path': ['sourceIPAddress'], 'in': ['1.1.1.2', '1.1.1.3']}
assert_true(Normalizer._match_condition(record, condition))
condition = {
'path': ['sourceIPAddress'],
'not_in': ['1.1.1.2', '1.1.1.3']
}
assert_false(Normalizer._match_condition(record, condition))
# Only support extract one condition. The result is not quaranteed if multiple conditions
# configured. In this test case, it is because 'not_in' condition is checked before
# 'contains'
condition = {
'path': ['detail', 'userIdentity', 'invokedBy'],
'contains': 'amazonaws.com',
'not_in': ['signin.amazonaws.com', 's3.amazonaws.com']
}
assert_false(Normalizer._match_condition(record, condition))
def test_match_types_multiple(self):
"""Normalizer - Match Types, Mutiple Sub-keys"""
normalized_types = {
'account': self._normalized_type_account(),
'ipv4': self._normalized_type_ip(),
'region': self._normalized_type_region(),
'user_identity': self._normalized_type_user_identity()
}
expected_results = {
'streamalert_record_id':
MOCK_RECORD_ID,
'account': [{
'values': ['123456'],
'function': None
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}, {
'values': ['1.1.1.2'],
'function': 'source ip address'
}],
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}, {
'values': ['region_name'],
'function': 'AWS region'
}],
'user_identity': [{
'values': ['Alice'],
'function': 'User name'
}, {
'values': ['signin.amazonaws.com'],
'function': 'Service name'
}]
}
results = Normalizer.match_types(self._test_record(), normalized_types)
assert_equal(results, expected_results)
def test_empty_value(self):
"""Normalizer - Normalize, Empty Value"""
test_record = {
'account': 123456,
'region': '' # This value is empty so should not be stored
}
normalized_types = {
'region': self._normalized_type_region(),
'account': self._normalized_type_account(),
'ipv4': self._normalized_type_ip()
}
expected_results = {
'streamalert_record_id': MOCK_RECORD_ID,
'account': [{
'values': ['123456'],
'function': None
}]
}
results = Normalizer.match_types(test_record, normalized_types)
assert_equal(results, expected_results)
def test_load_from_config_exist_types_config(self):
"""Normalizer - Load normalized_types from conf when it was loaded previously"""
Normalizer._types_config = {'normalized_type1': {}}
assert_equal(Normalizer.load_from_config({'foo': 'bar'}), Normalizer)
def test_normalize_condition(self):
"""Normalizer - Test normalization when condition applied"""
log_type = 'cloudtrail'
region = NormalizedType(
'test_log_type', 'region', [{
'path': ['region'],
'function': 'AWS region'
}, {
'path': ['detail', 'awsRegion'],
'function': 'AWS region',
'condition': {
'path': ['detail', 'userIdentity', 'userName'],
'not_in': ['alice', 'bob']
}
}])
ipv4 = NormalizedType('test_log_type', 'ip_address',
[{
'path': ['sourceIPAddress'],
'function': 'source ip address',
'condition': {
'path': ['account'],
'is': '123456'
}
}, {
'path': ['detail', 'source'],
'function': 'source ip address',
'condition': {
'path': ['account'],
'is_not': '123456'
}
}])
Normalizer._types_config = {log_type: {'region': region, 'ipv4': ipv4}}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert_normalization': {
'streamalert_record_id': MOCK_RECORD_ID,
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}]
}
}
assert_equal(record, expected_record)
def test_load_from_config_empty(self):
"""Normalizer - Load From Config, Empty"""
normalizer = Normalizer.load_from_config({})
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, None)