def test_short_circuit_without_exclude_list(self, is_excluded_ioc): """Threat Intel - ensure we skip threat intel exclusion if there are no excluded_iocs""" self.config = load_config('tests/unit/conf') self.config['global']['threat_intel']['enabled'] = True del self.config['global']['threat_intel']['excluded_iocs'] self.threat_intel = StreamThreatIntel.load_from_config(self.config) records = [{ 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': '8.8.8.8', 'recipientAccountId': '12345' }, 'source': '8.8.8.8', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 1) assert_equal(result[0].value, '8.8.8.8') assert not is_excluded_ioc.called
def test_threat_detection(self, mock_client): """Threat Intel - Test threat_detection method""" mock_client.return_value = MockDynamoDBClient() records = mock_normalized_records() threat_intel = StreamThreatIntel.load_from_config(self.config) assert_equal(len(threat_intel.threat_detection(records)), 2)
def test_extract_ioc_from_records_with_amazon_domains(self): """ Threat Intel - Test extracting IOCs where the "IP" is actually an AWS domain In cloudtrail_api events, many "ipaddress" values could be s3.amazonaws.com, ec2.amazonaws.com, cloudtrail.amazonaws.com, etc. expected output: an empty array of records """ records = [{ 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': 'ec2.amazon.com', 'recipientAccountId': '12345' }, 'source': 'ec2.amazon.com', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 0)
def test_extract_ioc_from_record_not_excluded(self): """Threat Intel - Test we don't exclude IOCs unintentionally""" records = [{ 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': '8.8.8.8', 'recipientAccountId': '12345' }, 'source': '8.8.8.8', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 1) assert_equal(result[0].value, '8.8.8.8')
def test_extract_ioc_from_record_without_excluded_ip(self): """ Threat Intel - Test extracting values from records with excluded IPs Expected output: an array with a single record for 6.6.6.6 """ records = [{ 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': '6.6.6.6', 'recipientAccountId': '12345' }, 'source': '6.6.6.6', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 1)
def test_extract_ioc_from_record_corner_cases(self): """ Threat Intel - Test extracting values with corner cases """ records = [{ 'account': 54321, 'region': '654321654321', 'detail': { 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'accountId': '54321' }, 'sourceIPAddress': '::1', 'recipientAccountId': '54321' }, 'source': '1.1.1.2/24', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] } }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 0)
def test_extract_ioc_from_record_with_private_ip(self): """Threat Intel - Test extracting values from a record based on normalized keys""" records = [ { 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': 'ec2.amazon.com', 'recipientAccountId': '12345' }, 'source': 'ec2.amazon.com', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }, { 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': '192.168.1.2', 'recipientAccountId': '12345' }, 'source': '192.168.1.2', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' } ] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 0)
def test_threat_intel_match(self, mock_client): """Rules Engine - Threat Intel is enabled when threat_intel_match is called""" @rule(datatypes=['sourceAddress', 'destinationDomain', 'fileHash'], outputs=['s3:sample_bucket']) def match_rule(_): # pylint: disable=unused-variable """Testing dummy rule""" return True mock_client.return_value = MockDynamoDBClient() toggled_config = self.config toggled_config['global']['threat_intel']['enabled'] = True toggled_config['global']['threat_intel']['dynamodb_table'] = 'test_table_name' new_rules_engine = RulesEngine(toggled_config) records = mock_normalized_records() alerts = new_rules_engine.threat_intel_match(records) assert_equal(len(alerts), 2)
def test_threat_detection_with_empty_ioc_value(self, mock_client): """Threat Intel - Test threat_detection with record contains empty/duplicated value""" records = [ { 'account': 12345, 'region': '123456123456', 'detail': { 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'accountId': '12345' }, 'sourceIPAddress': None, 'recipientAccountId': '12345' }, 'source': '1.1.1.2', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] } }, { 'domain': 'evil.com', 'pc_name': 'test-pc', 'date': 'Dec 1st, 2016', 'data': 'ABCDEF', 'streamalert:normalization': { 'destinationDomain': [['domain']] } }, { 'domain': 'EVIL.com', 'pc_name': 'test-pc', 'date': 'Dec 1st, 2016', 'data': 'ABCDEF', 'streamalert:normalization': { 'destinationDomain': [['domain']] } }, ] mock_client.return_value = MockDynamoDBClient() threat_intel = StreamThreatIntel.load_from_config(self.config) records = mock_normalized_records(records) assert_equal(len(threat_intel.threat_detection(records)), 3)
def test_extract_ioc_from_record(self): """Threat Intel - Test extracting values from a record based on normalized keys""" records = [{ 'account': 12345, 'region': '123456123456', 'detail': { 'eventType': 'AwsConsoleSignIn', 'eventName': 'ConsoleLogin', 'userIdentity': { 'userName': '******', 'type': 'Root', 'principalId': '12345', }, 'sourceIPAddress': '1.1.1.2', 'recipientAccountId': '12345' }, 'source': '1.1.1.2', 'streamalert:normalization': { 'sourceAddress': [['detail', 'sourceIPAddress'], ['source']], 'usernNme': [['detail', 'userIdentity', 'userName']] }, 'id': '12345' }] records = mock_normalized_records(records) for record in records: result = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(result), 1) assert_equal(result[0].value, '1.1.1.2') records = [{ 'cb_server': 'cbserver', 'computer_name': 'DESKTOP-TESTING', 'direction': 'outbound', 'domain': 'evil.com', 'event_type': 'netconn', 'ipv4': '1.1.1.2', 'local_ip': '1.1.1.2', 'local_port': '57347', 'md5': 'ABCDEF0123456789ABCDEF0123456789', 'pid': '268', 'port': '50002', 'process_guid': '00003a07-0000-010c-01d3-766fd6eee995', 'process_path': 'bad_actor.exe', 'protocol': '6', 'remote_ip': '82.82.82.82', 'remote_port': '50002', 'sensor_id': '14855', 'timestamp': 1515151515, 'type': 'ingress.event.netconn', 'streamalert:normalization': { 'destinationAddress': [['remote_ip']], 'destinationDomain': [['domain']], 'fileHash': [['md5']], 'sourceAddress': [['ipv4'], ['local_ip']] } }] records = mock_normalized_records(records) for record in records: results = self.threat_intel._extract_ioc_from_record(record) assert_equal(len(results), 4) assert_equal((results[0].value, results[0].ioc_type), ('1.1.1.2', 'ip')) assert_equal((results[1].value, results[1].ioc_type), ('82.82.82.82', 'ip')) assert_equal((results[2].value, results[2].ioc_type), ('evil.com', 'domain')) assert_equal((results[3].value, results[3].ioc_type), ('abcdef0123456789abcdef0123456789', 'md5'))