def test_load_from_config(self):
"""Normalizer - Load From Config"""
config = {
'logs': {
'cloudtrail': {
'schema': {},
'configuration': {
'normalization': {
'region': ['path', 'to', 'awsRegion'],
'sourceAccount': ['path', 'to', 'accountId']
}
}
}
}
}
normalizer = Normalizer.load_from_config(config)
expected_config = {
'cloudtrail': {
'region':
NormalizedType('cloudtrail', 'region',
['path', 'to', 'awsRegion']),
'sourceAccount':
NormalizedType('cloudtrail', 'sourceAccount',
['path', 'to', 'accountId'])
}
}
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_load_from_config_from_log_conf(self):
"""Normalizer - Load normalization config from "logs" field in the config"""
config = {
'logs': {
'cloudwatch:events': {
'schema': {
'account': 'string',
'source': 'string',
'key': 'string'
},
'parser': 'json',
'configuration': {
'normalization': {
'event_name': ['detail', 'eventName'],
'region': [{
'path': ['region'],
'function': 'aws region information'
}, {
'path': ['detail', 'awsRegion'],
'function': 'aws region information'
}],
'ip_address': [{
'path': ['detail', 'sourceIPAddress'],
'function':
'source ip address'
}]
}
}
}
}
}
expected_config = {
'cloudwatch:events': {
'event_name':
NormalizedType('cloudwatch:events', 'event_name',
['detail', 'eventName']),
'region':
NormalizedType('cloudwatch:events', 'region',
[{
'path': ['region'],
'function': 'aws region information'
}, {
'path': ['detail', 'awsRegion'],
'function': 'aws region information'
}]),
'ip_address':
NormalizedType('cloudwatch:events', 'ip_address',
[{
'path': ['detail', 'sourceIPAddress'],
'function': 'source ip address'
}])
}
}
normalizer = Normalizer.load_from_config(config)
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_key_does_not_exist(self):
"""Normalizer - Normalize, Key Does Not Exist"""
test_record = {'accountId': 123456, 'region': 'region_name'}
normalized_types = {
'region': self._normalized_type_region(),
'account': NormalizedType('test_log_type', 'account',
['accountId']),
# There is no IP value in record, so normalization should not include this
'ipv4': self._normalized_type_ip()
}
expected_results = {
'streamalert_record_id': MOCK_RECORD_ID,
'account': [{
'values': ['123456'],
'function': None
}],
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}]
}
results = Normalizer.match_types(test_record, normalized_types)
assert_equal(results, expected_results)
def test_normalize_corner_case(self):
"""Normalizer - Normalize - Corner Case"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'normalized_key':
NormalizedType(log_type, 'normalized_key',
['original_key', 'original_key']),
'account':
self._normalized_type_account()
}
}
record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
}
}
Normalizer.normalize(record, log_type)
expected_record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
},
'streamalert_normalization': {
'streamalert_record_id': MOCK_RECORD_ID,
'normalized_key': [{
'values': ['fizzbuzz'],
'function': None
}]
}
}
assert_equal(record, expected_record)
def _normalized_type_region(cls):
return NormalizedType('test_log_type', 'region',
[{
'path': ['region'],
'function': 'AWS region'
}, {
'path': ['detail', 'awsRegion'],
'function': 'AWS region'
}])
def _normalized_type_ip(cls):
return NormalizedType('test_log_type', 'ip_address',
[{
'path': ['sourceIPAddress'],
'function': 'source ip address'
}, {
'path': ['detail', 'source'],
'function': 'source ip address'
}])
def _normalized_type_user_identity(cls):
return NormalizedType(
'test_log_type', 'user_identity',
[{
'path': ['detail', 'userIdentity', 'userName'],
'function': 'User name'
}, {
'path': ['detail', 'userIdentity', 'invokedBy'],
'function': 'Service name'
}])
def test_load_from_config_deprecate_normalized_types(self):
"""Normalizer - Load normalization config and deprecate conf/normalized_types.json
"""
config = {
'logs': {
'cloudwatch:events': {
'schema': {
'account': 'string',
'source': 'string',
'key': 'string'
},
'parser': 'json',
'configuration': {
'normalization': {
'ip_address': [{
'path': ['path', 'to', 'sourceIPAddress'],
'function':
'source ip address'
}]
}
}
},
'other_log_type': {}
},
'normalized_types': {
'cloudwatch': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
}
expected_config = {
'cloudwatch:events': {
'ip_address':
NormalizedType('cloudwatch:events', 'ip_address',
[{
'path': ['path', 'to', 'sourceIPAddress'],
'function': 'source ip address'
}])
}
}
normalizer = Normalizer.load_from_config(config)
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def _normalized_type_account(cls):
return NormalizedType('test_log_type', 'account', ['account'])
def test_normalize_condition(self):
"""Normalizer - Test normalization when condition applied"""
log_type = 'cloudtrail'
region = NormalizedType(
'test_log_type', 'region', [{
'path': ['region'],
'function': 'AWS region'
}, {
'path': ['detail', 'awsRegion'],
'function': 'AWS region',
'condition': {
'path': ['detail', 'userIdentity', 'userName'],
'not_in': ['alice', 'bob']
}
}])
ipv4 = NormalizedType('test_log_type', 'ip_address',
[{
'path': ['sourceIPAddress'],
'function': 'source ip address',
'condition': {
'path': ['account'],
'is': '123456'
}
}, {
'path': ['detail', 'source'],
'function': 'source ip address',
'condition': {
'path': ['account'],
'is_not': '123456'
}
}])
Normalizer._types_config = {log_type: {'region': region, 'ipv4': ipv4}}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert_normalization': {
'streamalert_record_id': MOCK_RECORD_ID,
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}]
}
}
assert_equal(record, expected_record)