def test_normalize_none_defined(self, log_mock):
"""Normalizer - Normalize, No Types Defined"""
log_type = 'cloudtrail'
Normalizer._types_config = {}
Normalizer.normalize(self._test_record(), log_type)
log_mock.assert_called_with(
'No normalized types defined for log type: %s', log_type)
def test_normalize_corner_case(self):
"""Normalizer - Normalize - Corner Case"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'normalized_key':
NormalizedType(log_type, 'normalized_key',
['original_key', 'original_key']),
'account':
self._normalized_type_account()
}
}
record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
}
}
Normalizer.normalize(record, log_type)
expected_record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
},
'streamalert_normalization': {
'streamalert_record_id': MOCK_RECORD_ID,
'normalized_key': [{
'values': ['fizzbuzz'],
'function': None
}]
}
}
assert_equal(record, expected_record)
def test_normalize(self):
"""Normalizer - Normalize"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'region': {'region', 'awsRegion'},
'sourceAccount': {'account', 'accountId'}
}
}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert:normalization': {
'region': ['region_name'],
'sourceAccount': [123456]
}
}
assert_equal(record, expected_record)
def test_normalize_corner_case(self):
"""Normalizer - Normalize - Corner Case"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'normalized_key': {'normalized_key', 'original_key'},
'sourceAccount': {'account', 'accountId'}
}
}
record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
}
}
Normalizer.normalize(record, log_type)
expected_record = {
'unrelated_key': 'foobar',
'original_key': {
'original_key': 'fizzbuzz',
},
'streamalert:normalization': {
'normalized_key': ['fizzbuzz']
}
}
assert_equal(record, expected_record)
def _classify_payload(self, payload):
"""Run the payload through the classification logic to determine the data type
Args:
payload (StreamPayload): StreamAlert payload object being processed
"""
# Get logs defined for the service/entity in the config
logs_config = self._load_logs_for_resource(payload.service(), payload.resource)
if not logs_config:
LOGGER.error(
'No log types defined for resource [%s] in sources configuration for service [%s]',
payload.resource,
payload.service()
)
return
for record in payload.pre_parse():
# Increment the processed size using the length of this record
self._processed_size += len(record)
# Get the parser for this data
self._process_log_schemas(record, logs_config)
LOGGER.debug('Parsed and classified payload: %s', bool(record))
payload.fully_classified = payload.fully_classified and record
if not record:
self._log_bad_records(record, 1)
continue
LOGGER.debug(
'Classified %d record(s) with schema: %s',
len(record.parsed_records),
record.log_schema_type
)
# Even if the parser was successful, there's a chance it
# could not parse all records, so log them here as invalid
self._log_bad_records(record, len(record.invalid_records))
for parsed_rec in record.parsed_records:
#
# In Normalization v1, the normalized types are defined based on log source
# (e.g. osquery, cloudwatch etc) and this will be deprecated.
# In Normalization v2, the normalized types are defined based on log type
# (e.g. osquery:differential, cloudwatch:cloudtrail, cloudwatch:events etc)
#
Normalizer.normalize(parsed_rec, record.log_schema_type)
self._payloads.append(record)
def test_normalize(self):
"""Normalizer - Normalize"""
log_type = 'cloudtrail'
Normalizer._types_config = {
log_type: {
'region': self._normalized_type_region(),
'ipv4': self._normalized_type_ip()
}
}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert_normalization': {
'streamalert_record_id':
MOCK_RECORD_ID,
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}, {
'values': ['region_name'],
'function': 'AWS region'
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}, {
'values': ['1.1.1.2'],
'function': 'source ip address'
}]
}
}
assert_equal(record, expected_record)
def test_normalize_condition(self):
"""Normalizer - Test normalization when condition applied"""
log_type = 'cloudtrail'
region = NormalizedType(
'test_log_type', 'region', [{
'path': ['region'],
'function': 'AWS region'
}, {
'path': ['detail', 'awsRegion'],
'function': 'AWS region',
'condition': {
'path': ['detail', 'userIdentity', 'userName'],
'not_in': ['alice', 'bob']
}
}])
ipv4 = NormalizedType('test_log_type', 'ip_address',
[{
'path': ['sourceIPAddress'],
'function': 'source ip address',
'condition': {
'path': ['account'],
'is': '123456'
}
}, {
'path': ['detail', 'source'],
'function': 'source ip address',
'condition': {
'path': ['account'],
'is_not': '123456'
}
}])
Normalizer._types_config = {log_type: {'region': region, 'ipv4': ipv4}}
record = self._test_record()
Normalizer.normalize(record, log_type)
expected_record = {
'account': 123456,
'region': 'region_name',
'detail': {
'awsRegion': 'region_name',
'source': '1.1.1.2',
'userIdentity': {
"userName": "******",
"invokedBy": "signin.amazonaws.com"
}
},
'sourceIPAddress': '1.1.1.3',
'streamalert_normalization': {
'streamalert_record_id': MOCK_RECORD_ID,
'region': [{
'values': ['region_name'],
'function': 'AWS region'
}],
'ipv4': [{
'values': ['1.1.1.3'],
'function': 'source ip address'
}]
}
}
assert_equal(record, expected_record)