def test_process_types_config(self):
"""Threat Intel - Test process_types_config method"""
test_config = {
'types': {
'log_src1': {
'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'],
'normalizedTypeBar:ioc_bar': ['bar1', 'bar2'],
'normalizedTypePan': ['pan1']
},
'log_src2': {
'normalizedTypePing:ioc_ping': ['ping1', 'ping2'],
'normalizedTypePong:ioc_pong': ['pong1', 'pong2']
}
}
}
expected_result = {
'log_src1': {
'normalizedTypeBar': ['bar1', 'bar2'],
'normalizedTypeFoo': ['foo1', 'foo2'],
'normalizedTypePan': ['pan1']
},
'log_src2': {
'normalizedTypePing': ['ping1', 'ping2'],
'normalizedTypePong': ['pong1', 'pong2']
}
}
StreamThreatIntel._process_types_config(test_config['types'])
assert_equal(StreamThreatIntel.normalized_type_mapping(),
expected_result)
def test_from_config(self):
"""Threat Intel - Test load_config method"""
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': True
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_true(isinstance(threat_intel, StreamThreatIntel))
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': False
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_false(threat_intel)
test_config = {
'types': {
'log_src1': {
'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'],
'normalizedTypeBar:ioc_bar': ['bar1', 'bar2']
},
'log_src2': {
'normalizedTypePing:ioc_ping': ['ping1', 'ping2'],
'normalizedTypePong:ioc_pong': ['pong1', 'pong2']
}
}
}
StreamThreatIntel.load_from_config(test_config)
expected_result = {
'log_src1': {
'normalizedTypeBar': ['bar1', 'bar2'],
'normalizedTypeFoo': ['foo1', 'foo2']
},
'log_src2': {
'normalizedTypePing': ['ping1', 'ping2'],
'normalizedTypePong': ['pong1', 'pong2']
}
}
assert_equal(StreamThreatIntel.normalized_type_mapping(),
expected_result)
def _parse(self, payload):
"""Parse a record into a declared type.
Args:
payload: A StreamAlert payload object
Sets:
payload.log_source: The detected log name from the data_sources config.
payload.type: The record's type.
payload.records: The parsed records as a list.
Returns:
bool: the success of the parse.
"""
schema_matches = self._process_log_schemas(payload)
if not schema_matches:
return False
if LOGGER_DEBUG_ENABLED:
LOGGER.debug(
'Schema Matched Records:\n%s',
json.dumps([
schema_match.parsed_data for schema_match in schema_matches
],
indent=2))
schema_match = self._check_schema_match(schema_matches)
if LOGGER_DEBUG_ENABLED:
LOGGER.debug('Log name: %s', schema_match.log_name)
LOGGER.debug('Parsed data:\n%s',
json.dumps(schema_match.parsed_data, indent=2))
for parsed_data_value in schema_match.parsed_data:
# Convert data types per the schema
# Use the root schema for the parser due to updates caused by
# configuration settings such as envelope_keys and optional_keys
try:
if not self._convert_type(parsed_data_value,
schema_match.root_schema):
return False
except KeyError:
LOGGER.error('The payload is mis-classified. Payload [%s]',
parsed_data_value)
return False
normalized_types = StreamThreatIntel.normalized_type_mapping()
payload.log_source = schema_match.log_name
payload.type = schema_match.parser.type()
payload.records = schema_match.parsed_data
payload.normalized_types = normalized_types.get(
payload.log_source.split(':')[0])
return True
def test_load_from_config(self):
"""Threat Intel - Test load_config method"""
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': True,
"excluded_iocs": {
"domain": {
"not-evil.com": {
"comment": "This domain is not evil"
}
},
"ip": {
"10.0.0.0/8": {
"comment": "RFC1918"
},
"127.0.0.0/8": {
"comment": "localhost"
},
"172.16.0.0/12": {
"comment": "RFC1918"
},
"192.168.0.0/16": {
"comment": "RFC1918"
},
"52.52.52.52/32": {
"comment": "Test IP"
}
},
"md5": {
"feca1deadbeefcafebeadbeefcafebee": {
"comment": "not malicious, but delicious"
}
}
}
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_true(isinstance(threat_intel, StreamThreatIntel))
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': False
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_false(threat_intel)
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': False
}
},
'types': {
'log_src1': {
'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'],
'normalizedTypeBar:ioc_bar': ['bar1', 'bar2']
},
'log_src2': {
'normalizedTypePing:ioc_ping': ['ping1', 'ping2'],
'normalizedTypePong:ioc_pong': ['pong1', 'pong2']
}
}
}
StreamThreatIntel.load_from_config(test_config)
expected_result = {
'log_src1': {
'normalizedTypeBar': ['bar1', 'bar2'],
'normalizedTypeFoo': ['foo1', 'foo2']
},
'log_src2': {
'normalizedTypePing': ['ping1', 'ping2'],
'normalizedTypePong': ['pong1', 'pong2']
}
}
assert_equal(StreamThreatIntel.normalized_type_mapping(),
expected_result)
