def test_load_from_config(self):
"""Normalizer - Load From Config"""
config = {
'logs': {
'cloudtrail': {
'schema': {},
'configuration': {
'normalization': {
'region': ['path', 'to', 'awsRegion'],
'sourceAccount': ['path', 'to', 'accountId']
}
}
}
}
}
normalizer = Normalizer.load_from_config(config)
expected_config = {
'cloudtrail': {
'region':
NormalizedType('cloudtrail', 'region',
['path', 'to', 'awsRegion']),
'sourceAccount':
NormalizedType('cloudtrail', 'sourceAccount',
['path', 'to', 'accountId'])
}
}
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_load_from_config_from_log_conf(self):
"""Normalizer - Load normalization config from "logs" field in the config"""
config = {
'logs': {
'cloudwatch:events': {
'schema': {
'account': 'string',
'source': 'string',
'key': 'string'
},
'parser': 'json',
'configuration': {
'normalization': {
'event_name': ['detail', 'eventName'],
'region': [{
'path': ['region'],
'function': 'aws region information'
}, {
'path': ['detail', 'awsRegion'],
'function': 'aws region information'
}],
'ip_address': [{
'path': ['detail', 'sourceIPAddress'],
'function':
'source ip address'
}]
}
}
}
}
}
expected_config = {
'cloudwatch:events': {
'event_name':
NormalizedType('cloudwatch:events', 'event_name',
['detail', 'eventName']),
'region':
NormalizedType('cloudwatch:events', 'region',
[{
'path': ['region'],
'function': 'aws region information'
}, {
'path': ['detail', 'awsRegion'],
'function': 'aws region information'
}]),
'ip_address':
NormalizedType('cloudwatch:events', 'ip_address',
[{
'path': ['detail', 'sourceIPAddress'],
'function': 'source ip address'
}])
}
}
normalizer = Normalizer.load_from_config(config)
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def __init__(self):
# Create some objects to be cached if they have not already been created
Classifier._config = Classifier._config or config.load_config(validate=True)
Classifier._firehose_client = (
Classifier._firehose_client or FirehoseClient.load_from_config(
prefix=self.config['global']['account']['prefix'],
firehose_config=self.config['global'].get('infrastructure', {}).get('firehose', {}),
log_sources=self.config['logs']
)
)
Classifier._sqs_client = Classifier._sqs_client or SQSClient()
# Setup the normalization logic
Normalizer.load_from_config(self.config)
self._cluster = os.environ['CLUSTER']
self._payloads = []
self._failed_record_count = 0
self._processed_size = 0
def test_load_from_config_with_flag(self):
"""Normalizer - Load From Config with send_to_artifacts flag"""
config = {
'logs': {
'cloudwatch:flow_logs': {
'schema': {
'source': 'string',
'destination': 'string',
'destport': 'string'
},
'configuration': {
'normalization': {
'ip_address': [{
'path': ['destination'],
'function':
'Destination IP addresses'
}],
'port': [{
'path': ['destport'],
'function': 'Destination port number',
'send_to_artifacts': False
}]
}
}
}
}
}
normalizer = Normalizer.load_from_config(config)
record = {
'source': '1.1.1.2',
'destination': '2.2.2.2',
'destport': '54321'
}
normalizer.normalize(record, 'cloudwatch:flow_logs')
expect_result = {
'source': '1.1.1.2',
'destination': '2.2.2.2',
'destport': '54321',
'streamalert_normalization': {
'streamalert_record_id':
MOCK_RECORD_ID,
'ip_address': [{
'values': ['2.2.2.2'],
'function': 'Destination IP addresses'
}],
'port': [{
'values': ['54321'],
'function': 'Destination port number',
'send_to_artifacts': False
}]
}
}
assert_equal(record, expect_result)
def test_load_from_config_deprecate_normalized_types(self):
"""Normalizer - Load normalization config and deprecate conf/normalized_types.json
"""
config = {
'logs': {
'cloudwatch:events': {
'schema': {
'account': 'string',
'source': 'string',
'key': 'string'
},
'parser': 'json',
'configuration': {
'normalization': {
'ip_address': [{
'path': ['path', 'to', 'sourceIPAddress'],
'function':
'source ip address'
}]
}
}
},
'other_log_type': {}
},
'normalized_types': {
'cloudwatch': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
}
expected_config = {
'cloudwatch:events': {
'ip_address':
NormalizedType('cloudwatch:events', 'ip_address',
[{
'path': ['path', 'to', 'sourceIPAddress'],
'function': 'source ip address'
}])
}
}
normalizer = Normalizer.load_from_config(config)
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_load_from_config(self):
"""Normalizer - Load From Config"""
config = {
'normalized_types': {
'cloudtrail': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
}
normalizer = Normalizer.load_from_config(config)
expected_config = {
'cloudtrail': {
'region': ['region', 'awsRegion'],
'sourceAccount': ['account', 'accountId']
}
}
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, expected_config)
def test_load_from_config_empty(self):
"""Normalizer - Load From Config, Empty"""
normalizer = Normalizer.load_from_config({})
assert_equal(normalizer, Normalizer)
assert_equal(normalizer._types_config, None)
def test_load_from_config_exist_types_config(self):
"""Normalizer - Load normalized_types from conf when it was loaded previously"""
Normalizer._types_config = {'normalized_type1': {}}
assert_equal(Normalizer.load_from_config({'foo': 'bar'}), Normalizer)