def test_from_config(self): """Threat Intel - Test load_config method""" test_config = { 'global': { 'account': { 'region': 'us-east-1' }, 'threat_intel': { 'dynamodb_table': 'test_table_name', 'enabled': True } } } threat_intel = StreamThreatIntel.load_from_config(test_config) assert_true(isinstance(threat_intel, StreamThreatIntel)) test_config = { 'global': { 'account': { 'region': 'us-east-1' }, 'threat_intel': { 'dynamodb_table': 'test_table_name', 'enabled': False } } } threat_intel = StreamThreatIntel.load_from_config(test_config) assert_false(threat_intel) test_config = { 'types': { 'log_src1': { 'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'], 'normalizedTypeBar:ioc_bar': ['bar1', 'bar2'] }, 'log_src2': { 'normalizedTypePing:ioc_ping': ['ping1', 'ping2'], 'normalizedTypePong:ioc_pong': ['pong1', 'pong2'] } } } StreamThreatIntel.load_from_config(test_config) expected_result = { 'log_src1': { 'normalizedTypeBar': ['bar1', 'bar2'], 'normalizedTypeFoo': ['foo1', 'foo2'] }, 'log_src2': { 'normalizedTypePing': ['ping1', 'ping2'], 'normalizedTypePong': ['pong1', 'pong2'] } } assert_equal(StreamThreatIntel.normalized_type_mapping(), expected_result)
def test_load_from_config_with_cluster_env(self): """Threat Intel - Test load_from_config to read cluster""" config = load_config('tests/unit/conf') config['global']['threat_intel']['enabled'] = True threat_intel = StreamThreatIntel.load_from_config(config) assert_is_instance(threat_intel, StreamThreatIntel) assert_true('advanced' in config['clusters'].keys())
def test_short_circuit_without_exclude_list(self, is_excluded_ioc): """Threat Intel - ensure we skip threat intel exclusion if there are no excluded_iocs""" self.config = load_config('tests/unit/conf') self.config['global']['threat_intel']['enabled'] = True del self.config['global']['threat_intel']['excluded_iocs'] self.threat_intel = StreamThreatIntel.load_from_config(self.config) records = [{ 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': '8.8.8.8', 'recipientAccountId': '12345' }, 'source': '8.8.8.8', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 1) assert_equal(result[0].value, '8.8.8.8') assert not is_excluded_ioc.called
def test_query_with_duplicated_value(self, mock_client): """Threat Intel - Test query value includes dumplicated value""" mock_client.return_value = MockDynamoDBClient() threat_intel = StreamThreatIntel.load_from_config(self.config) test_values = ['1.1.1.2', 'EVIL.com', 'evil.com', 'abcdef0123456789'] threat_intel._query(test_values)
def test_threat_detection(self, mock_client): """Threat Intel - Test threat_detection method""" mock_client.return_value = MockDynamoDBClient() records = mock_normalized_records() threat_intel = StreamThreatIntel.load_from_config(self.config) assert_equal(len(threat_intel.threat_detection(records)), 2)
def test_process_ioc_with_clienterror(self, mock_client): """Threat Intel - Test private method process_ioc""" mock_client.return_value = MockDynamoDBClient(exception=True) threat_intel = StreamThreatIntel.load_from_config(self.config) ioc_collections = [StreamIoc(value='1.1.1.2', ioc_type='ip')] threat_intel._process_ioc(ioc_collections)
def test_query_with_empty_value(self, mock_client): """Threat Intel - Test query value includes empty value""" mock_client.return_value = MockDynamoDBClient() threat_intel = StreamThreatIntel.load_from_config(self.config) test_values = ['1.1.1.2', '', 'evil.com', 'abcdef0123456789'] result, _ = threat_intel._query(test_values) assert_equal(len(result), 3)
def test_load_from_config_with_cluster_env_2(self): """Threat Intel - Test load_from_config with threat intel disabled in cluster""" with patch.dict('os.environ', {'CLUSTER': 'test'}): config = load_config('tests/unit/conf') config['global']['threat_intel']['enabled'] = True threat_intel = StreamThreatIntel.load_from_config(config) assert_false(isinstance(threat_intel, StreamThreatIntel)) assert_equal(config['clusters'].keys(), ['test'])
def test_load_from_config_with_cluster_env(self): """Threat Intel - Test load_from_config to read cluster env variable""" with patch.dict('os.environ', {'CLUSTER': 'advanced'}): config = load_config('tests/unit/conf') config['global']['threat_intel']['enabled'] = True threat_intel = StreamThreatIntel.load_from_config(config) assert_is_instance(threat_intel, StreamThreatIntel) assert_equal(config['clusters'].keys(), ['advanced'])
def __init__(self, config, *rule_paths): """Initialize a RulesEngine instance to cache a StreamThreatIntel instance.""" self._threat_intel = StreamThreatIntel.load_from_config(config) self._required_outputs_set = resources.get_required_outputs() import_folders(*rule_paths) self._load_rule_table(config) lookup_tables = LookupTables.load_lookup_tables(config) if lookup_tables: RulesEngine._LOOKUP_TABLES = lookup_tables.download_s3_objects()
def test_query(self, mock_client): """Threat Intel - Test DynamoDB query method with batch_get_item""" mock_client.return_value = MockDynamoDBClient() threat_intel = StreamThreatIntel.load_from_config(self.config) test_values = ['1.1.1.2', '2.2.2.2', 'evil.com', 'abcdef0123456789'] result, unprocessed_keys = threat_intel._query(test_values) assert_equal(len(result), 2) assert_false(unprocessed_keys) assert_equal(result[0], {'ioc_value': '1.1.1.2', 'sub_type': 'mal_ip'}) assert_equal(result[1], {'ioc_value': 'evil.com', 'sub_type': 'c2_domain'})
def test_process_ioc_with_clienterror(self, log_mock, mock_client): """Threat Intel - Test private method process_ioc with Error""" mock_client.return_value = MockDynamoDBClient(exception=True) threat_intel = StreamThreatIntel.load_from_config(self.config) ioc_collections = [StreamIoc(value='1.1.1.2', ioc_type='ip')] threat_intel._process_ioc(ioc_collections) log_mock.assert_called_with( 'An error occurred while quering dynamodb table. Error is: %s', {'Error': { 'Code': 400, 'Message': 'raising test exception' }})
def test_process_ioc(self, mock_client): """Threat Intel - Test private method process_ioc""" mock_client.return_value = MockDynamoDBClient() threat_intel = StreamThreatIntel.load_from_config(self.config) ioc_collections = [ StreamIoc(value='1.1.1.2', ioc_type='ip'), StreamIoc(value='2.2.2.2', ioc_type='ip'), StreamIoc(value='evil.com', ioc_type='domain') ] threat_intel._process_ioc(ioc_collections) assert_true(ioc_collections[0].is_ioc) assert_false(ioc_collections[1].is_ioc) assert_true(ioc_collections[2].is_ioc)
def test_process_ioc_with_unprocessed_keys(self, mock_client): """Threat Intel - Test private method process_ioc when response has UnprocessedKeys""" mock_client.return_value = MockDynamoDBClient(unprocesed_keys=True) threat_intel = StreamThreatIntel.load_from_config(self.config) ioc_collections = [ StreamIoc(value='1.1.1.2', ioc_type='ip'), StreamIoc(value='foo', ioc_type='domain'), StreamIoc(value='bar', ioc_type='domain') ] threat_intel._process_ioc(ioc_collections) assert_true(ioc_collections[0].is_ioc) assert_false(ioc_collections[1].is_ioc) assert_false(ioc_collections[2].is_ioc)
def test_threat_detection_with_empty_ioc_value(self, mock_client): """Threat Intel - Test threat_detection with record contains empty/duplicated value""" records = [ { 'account': 12345, 'region': '123456123456', 'detail': { 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'accountId': '12345' }, 'sourceIPAddress': None, 'recipientAccountId': '12345' }, 'source': '1.1.1.2', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] } }, { 'domain': 'evil.com', 'pc_name': 'test-pc', 'date': 'Dec 1st, 2016', 'data': 'ABCDEF', 'streamalert:normalization': { 'destinationDomain': [['domain']] } }, { 'domain': 'EVIL.com', 'pc_name': 'test-pc', 'date': 'Dec 1st, 2016', 'data': 'ABCDEF', 'streamalert:normalization': { 'destinationDomain': [['domain']] } }, ] mock_client.return_value = MockDynamoDBClient() threat_intel = StreamThreatIntel.load_from_config(self.config) records = mock_normalized_records(records) assert_equal(len(threat_intel.threat_detection(records)), 3)
def test_load_from_config(self): """Threat Intel - Test load_config method""" test_config = { 'global': { 'account': { 'region': 'us-east-1' }, 'threat_intel': { 'dynamodb_table': 'test_table_name', 'enabled': True, "excluded_iocs": { "domain": { "not-evil.com": { "comment": "This domain is not evil" } }, "ip": { "10.0.0.0/8": { "comment": "RFC1918" }, "127.0.0.0/8": { "comment": "localhost" }, "172.16.0.0/12": { "comment": "RFC1918" }, "192.168.0.0/16": { "comment": "RFC1918" }, "52.52.52.52/32": { "comment": "Test IP" } }, "md5": { "feca1deadbeefcafebeadbeefcafebee": { "comment": "not malicious, but delicious" } } } } } } threat_intel = StreamThreatIntel.load_from_config(test_config) assert_true(isinstance(threat_intel, StreamThreatIntel)) test_config = { 'global': { 'account': { 'region': 'us-east-1' }, 'threat_intel': { 'dynamodb_table': 'test_table_name', 'enabled': False } } } threat_intel = StreamThreatIntel.load_from_config(test_config) assert_false(threat_intel) test_config = { 'global': { 'account': { 'region': 'us-east-1' }, 'threat_intel': { 'dynamodb_table': 'test_table_name', 'enabled': False } }, 'types': { 'log_src1': { 'normalizedTypeFoo:ioc_foo': ['foo1', 'foo2'], 'normalizedTypeBar:ioc_bar': ['bar1', 'bar2'] }, 'log_src2': { 'normalizedTypePing:ioc_ping': ['ping1', 'ping2'], 'normalizedTypePong:ioc_pong': ['pong1', 'pong2'] } } } StreamThreatIntel.load_from_config(test_config) expected_result = { 'log_src1': { 'normalizedTypeBar': ['bar1', 'bar2'], 'normalizedTypeFoo': ['foo1', 'foo2'] }, 'log_src2': { 'normalizedTypePing': ['ping1', 'ping2'], 'normalizedTypePong': ['pong1', 'pong2'] } } assert_equal(StreamThreatIntel.normalized_type_mapping(), expected_result)
def __init__(self, config): """Initialize a StreamRules instance to cache a StreamThreatIntel instance.""" self._threat_intel = StreamThreatIntel.load_from_config(config) self._required_outputs_set = resources.get_required_outputs()
def setup(self): """Setup before each method""" # Clear out the cached matchers and rules to avoid conflicts with production code self.config = load_config('tests/unit/conf') self.config['global']['threat_intel']['enabled'] = True self.threat_intel = StreamThreatIntel.load_from_config(self.config)
def test_query_with_exception(self, mock_client): """Threat Intel - Test DynamoDB query method with exception""" mock_client.return_value = MockDynamoDBClient(exception=True) threat_intel = StreamThreatIntel.load_from_config(self.config) threat_intel._query(['1.1.1.2'])
def __init__(self, config): """Initialize a StreamRules instance to cache a StreamThreatIntel instance.""" self._threat_intel = StreamThreatIntel.load_from_config(config)
def __init__(self, config, *rule_paths): """Initialize a RulesEngine instance to cache a StreamThreatIntel instance.""" self._threat_intel = StreamThreatIntel.load_from_config(config) self._required_outputs_set = resources.get_required_outputs() import_folders(*rule_paths) self._load_rule_table(config)