def test_load_from_config(self): """Normalizer - Load From Config""" config = { 'logs': { 'cloudtrail': { 'schema': {}, 'configuration': { 'normalization': { 'region': ['path', 'to', 'awsRegion'], 'sourceAccount': ['path', 'to', 'accountId'] } } } } } normalizer = Normalizer.load_from_config(config) expected_config = { 'cloudtrail': { 'region': NormalizedType('cloudtrail', 'region', ['path', 'to', 'awsRegion']), 'sourceAccount': NormalizedType('cloudtrail', 'sourceAccount', ['path', 'to', 'accountId']) } } assert_equal(normalizer, Normalizer) assert_equal(normalizer._types_config, expected_config)
def test_load_from_config_from_log_conf(self): """Normalizer - Load normalization config from "logs" field in the config""" config = { 'logs': { 'cloudwatch:events': { 'schema': { 'account': 'string', 'source': 'string', 'key': 'string' }, 'parser': 'json', 'configuration': { 'normalization': { 'event_name': ['detail', 'eventName'], 'region': [{ 'path': ['region'], 'function': 'aws region information' }, { 'path': ['detail', 'awsRegion'], 'function': 'aws region information' }], 'ip_address': [{ 'path': ['detail', 'sourceIPAddress'], 'function': 'source ip address' }] } } } } } expected_config = { 'cloudwatch:events': { 'event_name': NormalizedType('cloudwatch:events', 'event_name', ['detail', 'eventName']), 'region': NormalizedType('cloudwatch:events', 'region', [{ 'path': ['region'], 'function': 'aws region information' }, { 'path': ['detail', 'awsRegion'], 'function': 'aws region information' }]), 'ip_address': NormalizedType('cloudwatch:events', 'ip_address', [{ 'path': ['detail', 'sourceIPAddress'], 'function': 'source ip address' }]) } } normalizer = Normalizer.load_from_config(config) assert_equal(normalizer, Normalizer) assert_equal(normalizer._types_config, expected_config)
def test_key_does_not_exist(self): """Normalizer - Normalize, Key Does Not Exist""" test_record = {'accountId': 123456, 'region': 'region_name'} normalized_types = { 'region': self._normalized_type_region(), 'account': NormalizedType('test_log_type', 'account', ['accountId']), # There is no IP value in record, so normalization should not include this 'ipv4': self._normalized_type_ip() } expected_results = { 'streamalert_record_id': MOCK_RECORD_ID, 'account': [{ 'values': ['123456'], 'function': None }], 'region': [{ 'values': ['region_name'], 'function': 'AWS region' }] } results = Normalizer.match_types(test_record, normalized_types) assert_equal(results, expected_results)
def test_normalize_corner_case(self): """Normalizer - Normalize - Corner Case""" log_type = 'cloudtrail' Normalizer._types_config = { log_type: { 'normalized_key': NormalizedType(log_type, 'normalized_key', ['original_key', 'original_key']), 'account': self._normalized_type_account() } } record = { 'unrelated_key': 'foobar', 'original_key': { 'original_key': 'fizzbuzz', } } Normalizer.normalize(record, log_type) expected_record = { 'unrelated_key': 'foobar', 'original_key': { 'original_key': 'fizzbuzz', }, 'streamalert_normalization': { 'streamalert_record_id': MOCK_RECORD_ID, 'normalized_key': [{ 'values': ['fizzbuzz'], 'function': None }] } } assert_equal(record, expected_record)
def _normalized_type_region(cls): return NormalizedType('test_log_type', 'region', [{ 'path': ['region'], 'function': 'AWS region' }, { 'path': ['detail', 'awsRegion'], 'function': 'AWS region' }])
def _normalized_type_ip(cls): return NormalizedType('test_log_type', 'ip_address', [{ 'path': ['sourceIPAddress'], 'function': 'source ip address' }, { 'path': ['detail', 'source'], 'function': 'source ip address' }])
def _normalized_type_user_identity(cls): return NormalizedType( 'test_log_type', 'user_identity', [{ 'path': ['detail', 'userIdentity', 'userName'], 'function': 'User name' }, { 'path': ['detail', 'userIdentity', 'invokedBy'], 'function': 'Service name' }])
def test_load_from_config_deprecate_normalized_types(self): """Normalizer - Load normalization config and deprecate conf/normalized_types.json """ config = { 'logs': { 'cloudwatch:events': { 'schema': { 'account': 'string', 'source': 'string', 'key': 'string' }, 'parser': 'json', 'configuration': { 'normalization': { 'ip_address': [{ 'path': ['path', 'to', 'sourceIPAddress'], 'function': 'source ip address' }] } } }, 'other_log_type': {} }, 'normalized_types': { 'cloudwatch': { 'region': ['region', 'awsRegion'], 'sourceAccount': ['account', 'accountId'] } } } expected_config = { 'cloudwatch:events': { 'ip_address': NormalizedType('cloudwatch:events', 'ip_address', [{ 'path': ['path', 'to', 'sourceIPAddress'], 'function': 'source ip address' }]) } } normalizer = Normalizer.load_from_config(config) assert_equal(normalizer, Normalizer) assert_equal(normalizer._types_config, expected_config)
def _normalized_type_account(cls): return NormalizedType('test_log_type', 'account', ['account'])
def test_normalize_condition(self): """Normalizer - Test normalization when condition applied""" log_type = 'cloudtrail' region = NormalizedType( 'test_log_type', 'region', [{ 'path': ['region'], 'function': 'AWS region' }, { 'path': ['detail', 'awsRegion'], 'function': 'AWS region', 'condition': { 'path': ['detail', 'userIdentity', 'userName'], 'not_in': ['alice', 'bob'] } }]) ipv4 = NormalizedType('test_log_type', 'ip_address', [{ 'path': ['sourceIPAddress'], 'function': 'source ip address', 'condition': { 'path': ['account'], 'is': '123456' } }, { 'path': ['detail', 'source'], 'function': 'source ip address', 'condition': { 'path': ['account'], 'is_not': '123456' } }]) Normalizer._types_config = {log_type: {'region': region, 'ipv4': ipv4}} record = self._test_record() Normalizer.normalize(record, log_type) expected_record = { 'account': 123456, 'region': 'region_name', 'detail': { 'awsRegion': 'region_name', 'source': '1.1.1.2', 'userIdentity': { "userName": "******", "invokedBy": "signin.amazonaws.com" } }, 'sourceIPAddress': '1.1.1.3', 'streamalert_normalization': { 'streamalert_record_id': MOCK_RECORD_ID, 'region': [{ 'values': ['region_name'], 'function': 'AWS region' }], 'ipv4': [{ 'values': ['1.1.1.3'], 'function': 'source ip address' }] } } assert_equal(record, expected_record)