def test_from_config(self):
"""Threat Intel - Test load_config method"""
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': True
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_true(isinstance(threat_intel, StreamThreatIntel))
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': False
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_false(threat_intel)
test_config = {
'types': {
'log_src1': {
'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'],
'normalizedTypeBar:ioc_bar': ['bar1', 'bar2']
},
'log_src2': {
'normalizedTypePing:ioc_ping': ['ping1', 'ping2'],
'normalizedTypePong:ioc_pong': ['pong1', 'pong2']
}
}
}
StreamThreatIntel.load_from_config(test_config)
expected_result = {
'log_src1': {
'normalizedTypeBar': ['bar1', 'bar2'],
'normalizedTypeFoo': ['foo1', 'foo2']
},
'log_src2': {
'normalizedTypePing': ['ping1', 'ping2'],
'normalizedTypePong': ['pong1', 'pong2']
}
}
assert_equal(StreamThreatIntel.normalized_type_mapping(),
expected_result)
def test_load_from_config_with_cluster_env(self):
"""Threat Intel - Test load_from_config to read cluster"""
config = load_config('tests/unit/conf')
config['global']['threat_intel']['enabled'] = True
threat_intel = StreamThreatIntel.load_from_config(config)
assert_is_instance(threat_intel, StreamThreatIntel)
assert_true('advanced' in config['clusters'].keys())
def test_short_circuit_without_exclude_list(self, is_excluded_ioc):
"""Threat Intel - ensure we skip threat intel exclusion if there are no excluded_iocs"""
self.config = load_config('tests/unit/conf')
self.config['global']['threat_intel']['enabled'] = True
del self.config['global']['threat_intel']['excluded_iocs']
self.threat_intel = StreamThreatIntel.load_from_config(self.config)
records = [{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': '8.8.8.8',
'recipientAccountId': '12345'
},
'source': '8.8.8.8',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 1)
assert_equal(result[0].value, '8.8.8.8')
assert not is_excluded_ioc.called
def test_query_with_duplicated_value(self, mock_client):
"""Threat Intel - Test query value includes dumplicated value"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
test_values = ['1.1.1.2', 'EVIL.com', 'evil.com', 'abcdef0123456789']
threat_intel._query(test_values)
def test_threat_detection(self, mock_client):
"""Threat Intel - Test threat_detection method"""
mock_client.return_value = MockDynamoDBClient()
records = mock_normalized_records()
threat_intel = StreamThreatIntel.load_from_config(self.config)
assert_equal(len(threat_intel.threat_detection(records)), 2)
def test_process_ioc_with_clienterror(self, mock_client):
"""Threat Intel - Test private method process_ioc"""
mock_client.return_value = MockDynamoDBClient(exception=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [StreamIoc(value='1.1.1.2', ioc_type='ip')]
threat_intel._process_ioc(ioc_collections)
def test_query_with_empty_value(self, mock_client):
"""Threat Intel - Test query value includes empty value"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
test_values = ['1.1.1.2', '', 'evil.com', 'abcdef0123456789']
result, _ = threat_intel._query(test_values)
assert_equal(len(result), 3)
def test_load_from_config_with_cluster_env_2(self):
"""Threat Intel - Test load_from_config with threat intel disabled in cluster"""
with patch.dict('os.environ', {'CLUSTER': 'test'}):
config = load_config('tests/unit/conf')
config['global']['threat_intel']['enabled'] = True
threat_intel = StreamThreatIntel.load_from_config(config)
assert_false(isinstance(threat_intel, StreamThreatIntel))
assert_equal(config['clusters'].keys(), ['test'])
def test_load_from_config_with_cluster_env(self):
"""Threat Intel - Test load_from_config to read cluster env variable"""
with patch.dict('os.environ', {'CLUSTER': 'advanced'}):
config = load_config('tests/unit/conf')
config['global']['threat_intel']['enabled'] = True
threat_intel = StreamThreatIntel.load_from_config(config)
assert_is_instance(threat_intel, StreamThreatIntel)
assert_equal(config['clusters'].keys(), ['advanced'])
def __init__(self, config, *rule_paths):
"""Initialize a RulesEngine instance to cache a StreamThreatIntel instance."""
self._threat_intel = StreamThreatIntel.load_from_config(config)
self._required_outputs_set = resources.get_required_outputs()
import_folders(*rule_paths)
self._load_rule_table(config)
lookup_tables = LookupTables.load_lookup_tables(config)
if lookup_tables:
RulesEngine._LOOKUP_TABLES = lookup_tables.download_s3_objects()
def test_query(self, mock_client):
"""Threat Intel - Test DynamoDB query method with batch_get_item"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
test_values = ['1.1.1.2', '2.2.2.2', 'evil.com', 'abcdef0123456789']
result, unprocessed_keys = threat_intel._query(test_values)
assert_equal(len(result), 2)
assert_false(unprocessed_keys)
assert_equal(result[0], {'ioc_value': '1.1.1.2', 'sub_type': 'mal_ip'})
assert_equal(result[1], {'ioc_value': 'evil.com', 'sub_type': 'c2_domain'})
def test_process_ioc_with_clienterror(self, log_mock, mock_client):
"""Threat Intel - Test private method process_ioc with Error"""
mock_client.return_value = MockDynamoDBClient(exception=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [StreamIoc(value='1.1.1.2', ioc_type='ip')]
threat_intel._process_ioc(ioc_collections)
log_mock.assert_called_with(
'An error occurred while quering dynamodb table. Error is: %s',
{'Error': {
'Code': 400,
'Message': 'raising test exception'
}})
def test_process_ioc(self, mock_client):
"""Threat Intel - Test private method process_ioc"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [
StreamIoc(value='1.1.1.2', ioc_type='ip'),
StreamIoc(value='2.2.2.2', ioc_type='ip'),
StreamIoc(value='evil.com', ioc_type='domain')
]
threat_intel._process_ioc(ioc_collections)
assert_true(ioc_collections[0].is_ioc)
assert_false(ioc_collections[1].is_ioc)
assert_true(ioc_collections[2].is_ioc)
def test_process_ioc_with_unprocessed_keys(self, mock_client):
"""Threat Intel - Test private method process_ioc when response has UnprocessedKeys"""
mock_client.return_value = MockDynamoDBClient(unprocesed_keys=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [
StreamIoc(value='1.1.1.2', ioc_type='ip'),
StreamIoc(value='foo', ioc_type='domain'),
StreamIoc(value='bar', ioc_type='domain')
]
threat_intel._process_ioc(ioc_collections)
assert_true(ioc_collections[0].is_ioc)
assert_false(ioc_collections[1].is_ioc)
assert_false(ioc_collections[2].is_ioc)
def test_threat_detection_with_empty_ioc_value(self, mock_client):
"""Threat Intel - Test threat_detection with record contains empty/duplicated value"""
records = [
{
'account': 12345,
'region': '123456123456',
'detail': {
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'accountId': '12345'
},
'sourceIPAddress': None,
'recipientAccountId': '12345'
},
'source': '1.1.1.2',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'],
['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
}
},
{
'domain': 'evil.com',
'pc_name': 'test-pc',
'date': 'Dec 1st, 2016',
'data': 'ABCDEF',
'streamalert:normalization': {
'destinationDomain': [['domain']]
}
},
{
'domain': 'EVIL.com',
'pc_name': 'test-pc',
'date': 'Dec 1st, 2016',
'data': 'ABCDEF',
'streamalert:normalization': {
'destinationDomain': [['domain']]
}
},
]
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
records = mock_normalized_records(records)
assert_equal(len(threat_intel.threat_detection(records)), 3)
def test_load_from_config(self):
"""Threat Intel - Test load_config method"""
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': True,
"excluded_iocs": {
"domain": {
"not-evil.com": {
"comment": "This domain is not evil"
}
},
"ip": {
"10.0.0.0/8": {
"comment": "RFC1918"
},
"127.0.0.0/8": {
"comment": "localhost"
},
"172.16.0.0/12": {
"comment": "RFC1918"
},
"192.168.0.0/16": {
"comment": "RFC1918"
},
"52.52.52.52/32": {
"comment": "Test IP"
}
},
"md5": {
"feca1deadbeefcafebeadbeefcafebee": {
"comment": "not malicious, but delicious"
}
}
}
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_true(isinstance(threat_intel, StreamThreatIntel))
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': False
}
}
}
threat_intel = StreamThreatIntel.load_from_config(test_config)
assert_false(threat_intel)
test_config = {
'global': {
'account': {
'region': 'us-east-1'
},
'threat_intel': {
'dynamodb_table': 'test_table_name',
'enabled': False
}
},
'types': {
'log_src1': {
'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'],
'normalizedTypeBar:ioc_bar': ['bar1', 'bar2']
},
'log_src2': {
'normalizedTypePing:ioc_ping': ['ping1', 'ping2'],
'normalizedTypePong:ioc_pong': ['pong1', 'pong2']
}
}
}
StreamThreatIntel.load_from_config(test_config)
expected_result = {
'log_src1': {
'normalizedTypeBar': ['bar1', 'bar2'],
'normalizedTypeFoo': ['foo1', 'foo2']
},
'log_src2': {
'normalizedTypePing': ['ping1', 'ping2'],
'normalizedTypePong': ['pong1', 'pong2']
}
}
assert_equal(StreamThreatIntel.normalized_type_mapping(),
expected_result)
def __init__(self, config):
"""Initialize a StreamRules instance to cache a StreamThreatIntel instance."""
self._threat_intel = StreamThreatIntel.load_from_config(config)
self._required_outputs_set = resources.get_required_outputs()
def setup(self):
"""Setup before each method"""
# Clear out the cached matchers and rules to avoid conflicts with production code
self.config = load_config('tests/unit/conf')
self.config['global']['threat_intel']['enabled'] = True
self.threat_intel = StreamThreatIntel.load_from_config(self.config)
def test_query_with_exception(self, mock_client):
"""Threat Intel - Test DynamoDB query method with exception"""
mock_client.return_value = MockDynamoDBClient(exception=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
threat_intel._query(['1.1.1.2'])
def __init__(self, config):
"""Initialize a StreamRules instance to cache a StreamThreatIntel instance."""
self._threat_intel = StreamThreatIntel.load_from_config(config)
def __init__(self, config, *rule_paths):
"""Initialize a RulesEngine instance to cache a StreamThreatIntel instance."""
self._threat_intel = StreamThreatIntel.load_from_config(config)
self._required_outputs_set = resources.get_required_outputs()
import_folders(*rule_paths)
self._load_rule_table(config)
