def test_short_circuit_without_exclude_list(self, is_excluded_ioc):
"""Threat Intel - ensure we skip threat intel exclusion if there are no excluded_iocs"""
self.config = load_config('tests/unit/conf')
self.config['global']['threat_intel']['enabled'] = True
del self.config['global']['threat_intel']['excluded_iocs']
self.threat_intel = StreamThreatIntel.load_from_config(self.config)
records = [{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': '8.8.8.8',
'recipientAccountId': '12345'
},
'source': '8.8.8.8',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 1)
assert_equal(result[0].value, '8.8.8.8')
assert not is_excluded_ioc.called
def test_threat_detection(self, mock_client):
"""Threat Intel - Test threat_detection method"""
mock_client.return_value = MockDynamoDBClient()
records = mock_normalized_records()
threat_intel = StreamThreatIntel.load_from_config(self.config)
assert_equal(len(threat_intel.threat_detection(records)), 2)
def test_extract_ioc_from_records_with_amazon_domains(self):
"""
Threat Intel - Test extracting IOCs where the "IP" is actually an AWS domain
In cloudtrail_api events, many "ipaddress" values could be
s3.amazonaws.com, ec2.amazonaws.com, cloudtrail.amazonaws.com, etc.
expected output: an empty array of records
"""
records = [{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': 'ec2.amazon.com',
'recipientAccountId': '12345'
},
'source': 'ec2.amazon.com',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 0)
def test_extract_ioc_from_record_not_excluded(self):
"""Threat Intel - Test we don't exclude IOCs unintentionally"""
records = [{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': '8.8.8.8',
'recipientAccountId': '12345'
},
'source': '8.8.8.8',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 1)
assert_equal(result[0].value, '8.8.8.8')
def test_extract_ioc_from_record_without_excluded_ip(self):
"""
Threat Intel - Test extracting values from records with excluded IPs
Expected output: an array with a single record for 6.6.6.6
"""
records = [{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': '6.6.6.6',
'recipientAccountId': '12345'
},
'source': '6.6.6.6',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 1)
def test_extract_ioc_from_record_corner_cases(self):
"""
Threat Intel - Test extracting values with corner cases
"""
records = [{
'account': 54321,
'region': '654321654321',
'detail': {
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'accountId': '54321'
},
'sourceIPAddress': '::1',
'recipientAccountId': '54321'
},
'source': '1.1.1.2/24',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
}
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 0)
def test_extract_ioc_from_record_with_private_ip(self):
"""Threat Intel - Test extracting values from a record based on normalized keys"""
records = [
{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': 'ec2.amazon.com',
'recipientAccountId': '12345'
},
'source': 'ec2.amazon.com',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
},
{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': '192.168.1.2',
'recipientAccountId': '12345'
},
'source': '192.168.1.2',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}
]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 0)
def test_threat_intel_match(self, mock_client):
"""Rules Engine - Threat Intel is enabled when threat_intel_match is called"""
@rule(datatypes=['sourceAddress', 'destinationDomain', 'fileHash'],
outputs=['s3:sample_bucket'])
def match_rule(_): # pylint: disable=unused-variable
"""Testing dummy rule"""
return True
mock_client.return_value = MockDynamoDBClient()
toggled_config = self.config
toggled_config['global']['threat_intel']['enabled'] = True
toggled_config['global']['threat_intel']['dynamodb_table'] = 'test_table_name'
new_rules_engine = RulesEngine(toggled_config)
records = mock_normalized_records()
alerts = new_rules_engine.threat_intel_match(records)
assert_equal(len(alerts), 2)
def test_threat_detection_with_empty_ioc_value(self, mock_client):
"""Threat Intel - Test threat_detection with record contains empty/duplicated value"""
records = [
{
'account': 12345,
'region': '123456123456',
'detail': {
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'accountId': '12345'
},
'sourceIPAddress': None,
'recipientAccountId': '12345'
},
'source': '1.1.1.2',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'],
['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
}
},
{
'domain': 'evil.com',
'pc_name': 'test-pc',
'date': 'Dec 1st, 2016',
'data': 'ABCDEF',
'streamalert:normalization': {
'destinationDomain': [['domain']]
}
},
{
'domain': 'EVIL.com',
'pc_name': 'test-pc',
'date': 'Dec 1st, 2016',
'data': 'ABCDEF',
'streamalert:normalization': {
'destinationDomain': [['domain']]
}
},
]
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
records = mock_normalized_records(records)
assert_equal(len(threat_intel.threat_detection(records)), 3)
def test_extract_ioc_from_record(self):
"""Threat Intel - Test extracting values from a record based on normalized keys"""
records = [{
'account': 12345,
'region': '123456123456',
'detail': {
'eventType': 'AwsConsoleSignIn',
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'type': 'Root',
'principalId': '12345',
},
'sourceIPAddress': '1.1.1.2',
'recipientAccountId': '12345'
},
'source': '1.1.1.2',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'], ['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
},
'id': '12345'
}]
records = mock_normalized_records(records)
for record in records:
result = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(result), 1)
assert_equal(result[0].value, '1.1.1.2')
records = [{
'cb_server': 'cbserver',
'computer_name': 'DESKTOP-TESTING',
'direction': 'outbound',
'domain': 'evil.com',
'event_type': 'netconn',
'ipv4': '1.1.1.2',
'local_ip': '1.1.1.2',
'local_port': '57347',
'md5': 'ABCDEF0123456789ABCDEF0123456789',
'pid': '268',
'port': '50002',
'process_guid': '00003a07-0000-010c-01d3-766fd6eee995',
'process_path': 'bad_actor.exe',
'protocol': '6',
'remote_ip': '82.82.82.82',
'remote_port': '50002',
'sensor_id': '14855',
'timestamp': 1515151515,
'type': 'ingress.event.netconn',
'streamalert:normalization': {
'destinationAddress': [['remote_ip']],
'destinationDomain': [['domain']],
'fileHash': [['md5']],
'sourceAddress': [['ipv4'], ['local_ip']]
}
}]
records = mock_normalized_records(records)
for record in records:
results = self.threat_intel._extract_ioc_from_record(record)
assert_equal(len(results), 4)
assert_equal((results[0].value, results[0].ioc_type),
('1.1.1.2', 'ip'))
assert_equal((results[1].value, results[1].ioc_type),
('82.82.82.82', 'ip'))
assert_equal((results[2].value, results[2].ioc_type),
('evil.com', 'domain'))
assert_equal((results[3].value, results[3].ioc_type),
('abcdef0123456789abcdef0123456789', 'md5'))
