def test_threat_detection(self, mock_client):
"""Threat Intel - Test threat_detection method"""
mock_client.return_value = MockDynamoDBClient()
records = mock_normalized_records()
threat_intel = StreamThreatIntel.load_from_config(self.config)
assert_equal(len(threat_intel.threat_detection(records)), 2)
def test_query_with_duplicated_value(self, mock_client):
"""Threat Intel - Test query value includes dumplicated value"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
test_values = ['1.1.1.2', 'EVIL.com', 'evil.com', 'abcdef0123456789']
threat_intel._query(test_values)
def test_process_with_threat_intel_enabled(self, mock_client):
"""Rules Engine - Threat Intel is enabled when process method is called"""
@rule(datatypes=['sourceAddress'], outputs=['s3:sample_bucket'])
def match_ipaddress(_): # pylint: disable=unused-variable
"""Testing dummy rule"""
return True
mock_client.return_value = MockDynamoDBClient()
toggled_config = self.config
toggled_config['global']['threat_intel']['enabled'] = True
toggled_config['global']['threat_intel'][
'dynamodb_table'] = 'test_table_name'
new_rules_engine = RulesEngine(toggled_config)
kinesis_data_items = [{
'account': 123456,
'region': '123456123456',
'source': '1.1.1.2',
'detail': {
'eventName': 'ConsoleLogin',
'sourceIPAddress': '1.1.1.2',
'recipientAccountId': '654321'
}
}]
for data in kinesis_data_items:
kinesis_data = json.dumps(data)
service, entity = 'kinesis', 'test_kinesis_stream'
raw_record = make_kinesis_raw_record(entity, kinesis_data)
payload = load_and_classify_payload(toggled_config, service,
entity, raw_record)
assert_equal(len(new_rules_engine.run(payload)[0]), 1)
def test_process_ioc_with_clienterror(self, mock_client):
"""Threat Intel - Test private method process_ioc"""
mock_client.return_value = MockDynamoDBClient(exception=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [StreamIoc(value='1.1.1.2', ioc_type='ip')]
threat_intel._process_ioc(ioc_collections)
def test_query_with_empty_value(self, mock_client):
"""Threat Intel - Test query value includes empty value"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
test_values = ['1.1.1.2', '', 'evil.com', 'abcdef0123456789']
result, _ = threat_intel._query(test_values)
assert_equal(len(result), 3)
def test_query(self, mock_client):
"""Threat Intel - Test DynamoDB query method with batch_get_item"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
test_values = ['1.1.1.2', '2.2.2.2', 'evil.com', 'abcdef0123456789']
result, unprocessed_keys = threat_intel._query(test_values)
assert_equal(len(result), 2)
assert_false(unprocessed_keys)
assert_equal(result[0], {'ioc_value': '1.1.1.2', 'sub_type': 'mal_ip'})
assert_equal(result[1], {'ioc_value': 'evil.com', 'sub_type': 'c2_domain'})
def test_process_ioc_with_clienterror(self, log_mock, mock_client):
"""Threat Intel - Test private method process_ioc with Error"""
mock_client.return_value = MockDynamoDBClient(exception=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [StreamIoc(value='1.1.1.2', ioc_type='ip')]
threat_intel._process_ioc(ioc_collections)
log_mock.assert_called_with(
'An error occurred while quering dynamodb table. Error is: %s',
{'Error': {
'Code': 400,
'Message': 'raising test exception'
}})
def test_process_ioc_with_unprocessed_keys(self, mock_client):
"""Threat Intel - Test private method process_ioc when response has UnprocessedKeys"""
mock_client.return_value = MockDynamoDBClient(unprocesed_keys=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [
StreamIoc(value='1.1.1.2', ioc_type='ip'),
StreamIoc(value='foo', ioc_type='domain'),
StreamIoc(value='bar', ioc_type='domain')
]
threat_intel._process_ioc(ioc_collections)
assert_true(ioc_collections[0].is_ioc)
assert_false(ioc_collections[1].is_ioc)
assert_false(ioc_collections[2].is_ioc)
def test_process_ioc(self, mock_client):
"""Threat Intel - Test private method process_ioc"""
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
ioc_collections = [
StreamIoc(value='1.1.1.2', ioc_type='ip'),
StreamIoc(value='2.2.2.2', ioc_type='ip'),
StreamIoc(value='evil.com', ioc_type='domain')
]
threat_intel._process_ioc(ioc_collections)
assert_true(ioc_collections[0].is_ioc)
assert_false(ioc_collections[1].is_ioc)
assert_true(ioc_collections[2].is_ioc)
def test_threat_intel_match(self, mock_client):
"""Rules Engine - Threat Intel is enabled when threat_intel_match is called"""
@rule(datatypes=['sourceAddress', 'destinationDomain', 'fileHash'],
outputs=['s3:sample_bucket'])
def match_rule(_): # pylint: disable=unused-variable
"""Testing dummy rule"""
return True
mock_client.return_value = MockDynamoDBClient()
toggled_config = self.config
toggled_config['global']['threat_intel']['enabled'] = True
toggled_config['global']['threat_intel']['dynamodb_table'] = 'test_table_name'
new_rules_engine = RulesEngine(toggled_config)
records = mock_normalized_records()
alerts = new_rules_engine.threat_intel_match(records)
assert_equal(len(alerts), 2)
def test_threat_detection_with_empty_ioc_value(self, mock_client):
"""Threat Intel - Test threat_detection with record contains empty/duplicated value"""
records = [
{
'account': 12345,
'region': '123456123456',
'detail': {
'eventName': 'ConsoleLogin',
'userIdentity': {
'userName': '******',
'accountId': '12345'
},
'sourceIPAddress': None,
'recipientAccountId': '12345'
},
'source': '1.1.1.2',
'streamalert:normalization': {
'sourceAddress': [['detail', 'sourceIPAddress'],
['source']],
'usernNme': [['detail', 'userIdentity', 'userName']]
}
},
{
'domain': 'evil.com',
'pc_name': 'test-pc',
'date': 'Dec 1st, 2016',
'data': 'ABCDEF',
'streamalert:normalization': {
'destinationDomain': [['domain']]
}
},
{
'domain': 'EVIL.com',
'pc_name': 'test-pc',
'date': 'Dec 1st, 2016',
'data': 'ABCDEF',
'streamalert:normalization': {
'destinationDomain': [['domain']]
}
},
]
mock_client.return_value = MockDynamoDBClient()
threat_intel = StreamThreatIntel.load_from_config(self.config)
records = mock_normalized_records(records)
assert_equal(len(threat_intel.threat_detection(records)), 3)
def test_query_with_exception(self, mock_client):
"""Threat Intel - Test DynamoDB query method with exception"""
mock_client.return_value = MockDynamoDBClient(exception=True)
threat_intel = StreamThreatIntel.load_from_config(self.config)
threat_intel._query(['1.1.1.2'])
def test_process_allow_multi_around_normalization(self, mock_client):
"""Rules Engine - Threat Intel is enabled run multi-round_normalization"""
@rule(datatypes=['fileHash'], outputs=['s3:sample_bucket'])
def match_file_hash(rec): # pylint: disable=unused-variable
"""Testing dummy rule to match file hash"""
return 'streamalert:ioc' in rec and 'md5' in rec['streamalert:ioc']
@rule(datatypes=['fileHash'], outputs=['s3:sample_bucket'])
def match_file_hash_again(_): # pylint: disable=unused-variable
"""Testing dummy rule to match file hash again"""
return False
@rule(datatypes=['fileHash', 'sourceDomain'],
outputs=['s3:sample_bucket'])
def match_source_domain(rec): # pylint: disable=unused-variable
"""Testing dummy rule to match source domain and file hash"""
return 'streamalert:ioc' in rec
mock_client.return_value = MockDynamoDBClient()
toggled_config = self.config
toggled_config['global']['threat_intel']['enabled'] = True
toggled_config['global']['threat_intel'][
'dynamodb_table'] = 'test_table_name'
new_rules_engine = RulesEngine(toggled_config)
kinesis_data = {
"Field1": {
"SubField1": {
"key1": 17,
"key2_md5": "md5-of-file",
"key3_source_domain": "evil.com"
},
"SubField2": 1
},
"Field2": {
"Authentication": {}
},
"Field3": {},
"Field4": {}
}
kinesis_data = json.dumps(kinesis_data)
service, entity = 'kinesis', 'test_stream_threat_intel'
raw_record = make_kinesis_raw_record(entity, kinesis_data)
payload = load_and_classify_payload(toggled_config, service, entity,
raw_record)
alerts, normalized_records = new_rules_engine.run(payload)
# Two testing rules are for threat intelligence matching. So no alert will be
# generated before threat intel takes effect.
assert_equal(len(alerts), 0)
# One record will be normalized once by two different rules with different
# normalization keys.
assert_equal(len(normalized_records), 1)
assert_equal(
normalized_records[0].
pre_parsed_record['streamalert:normalization'].keys(),
['fileHash', 'sourceDomain'])
# Pass normalized records to threat intel engine.
alerts_from_threat_intel = new_rules_engine.threat_intel_match(
normalized_records)
assert_equal(len(alerts_from_threat_intel), 2)
assert_equal(alerts_from_threat_intel[0].rule_name, 'match_file_hash')
assert_equal(alerts_from_threat_intel[1].rule_name,
'match_source_domain')
