def test_remove_processed_keys(self):
"""ThreatIntel - Remove Unprocessed Keys"""
query_values = {
'1.1.1.1',
'2.2.2.2',
'09bb8985ca0a702907dbcfad511d138e',
'02907dbcfad38e511d109bb8985ca0a7'
}
unprocesed_keys = [
{
'ioc_value': {
'S': '2.2.2.2'
},
'sub_type': {
'S': 'mal_ip'
}
},
{
'ioc_value': {
'S': '09bb8985ca0a702907dbcfad511d138e'
},
'sub_type': {
'S': 'mal_md5'
}
}
]
expected_result = {
'2.2.2.2',
'09bb8985ca0a702907dbcfad511d138e'
}
ThreatIntel._remove_processed_keys(query_values, unprocesed_keys)
assert_equal(query_values, expected_result)
def __init__(self, *rule_paths):
RulesEngine._config = RulesEngine._config or load_config()
RulesEngine._threat_intel = (
RulesEngine._threat_intel or ThreatIntel.load_from_config(self.config)
)
# Instantiate the alert forwarder to handle sending alerts to the alert processor
RulesEngine._alert_forwarder = RulesEngine._alert_forwarder or AlertForwarder()
# Load the lookup tables
RulesEngine._lookup_tables = LookupTables.get_instance(config=self.config)
# If no rule import paths are specified, default to the config
rule_paths = rule_paths or [
item for location in {'rule_locations', 'matcher_locations'}
for item in self.config['global']['general'][location]
]
import_folders(*rule_paths)
self._rule_stat_tracker = RuleStatisticTracker(
'STREAMALERT_TRACK_RULE_STATS' in env,
'LAMBDA_RUNTIME_DIR' in env
)
self._required_outputs_set = resources.get_required_outputs()
self._load_rule_table(self.config)
def test_exceptions_to_giveup(self):
"""ThreatIntel - Exceptions to Giveup"""
err = Mock(
response={'Error': {'Code': 'ResourceNotFoundException'}}
)
result = ThreatIntel._exceptions_to_giveup(err)
assert_equal(result, True)
def test_load_from_config_disabled(self):
"""ThreatIntel - Load From Config, Disabled"""
config = {
'threat_intel': {
'enabled': False
}
}
assert_equal(ThreatIntel.load_from_config(config), None)
def test_insert_ioc_info(self):
"""ThreatIntel - Insert IOC Info"""
record = {
'key': 'value'
}
ioc_type = 'ip'
ioc_value = 'ioc_value'
expected_result = {
'key': 'value',
'streamalert:ioc': {
ioc_type: {ioc_value}
}
}
ThreatIntel._insert_ioc_info(record, ioc_type, ioc_value)
assert_equal(record, expected_result)
def test_segment(self):
"""ThreatIntel - Segment"""
expected_result = [
set(range(100)),
set(range(100, 120))
]
result = list(ThreatIntel._segment(list(range(120))))
assert_equal(result, expected_result)
def test_load_from_config_no_clusters(self):
"""ThreatIntel - Load From Config, Clusters Disabled"""
config = {
'threat_intel': {
'enabled': True
},
'clusters': {
'prod': {
'enable_threat_intel': False
}
}
}
assert_equal(ThreatIntel.load_from_config(config), None)
def test_insert_ioc_info_existing(self):
"""ThreatIntel - Insert IOC Info, With Existing"""
ioc_type = 'ip'
existing_value = 'existing_value'
record = {
'key': 'value',
'streamalert:ioc': {
ioc_type: {existing_value}
}
}
new_value = 'new_value'
expected_result = {
'key': 'value',
'streamalert:ioc': {
ioc_type: {existing_value, new_value}
}
}
ThreatIntel._insert_ioc_info(record, ioc_type, new_value)
assert_equal(record, expected_result)
def test_load_from_config(self):
"""ThreatIntel - Load From Config"""
ti_client = ThreatIntel.load_from_config(self._default_config)
assert_equal(isinstance(ti_client, ThreatIntel), True)
assert_equal(ti_client._table, 'table_name')
assert_equal(ti_client._enabled_clusters, {'prod'})
expected_config = {
'destinationDomain': 'domain',
'sourceAddress': 'ip',
'destinationAddress': 'ip',
'fileHash': 'md5'
}
assert_equal(ti_client._ioc_config, expected_config)
assert_equal(ti_client._excluded_iocs, {'domain': {'not.evil.com'}})
def test_setup_excluded_iocs(self):
"""ThreatIntel - Setup Excluded IOCs"""
excluded_iocs = {
'md5': {
'feca1deadbeefcafebeadbeefcafebee': {
'comment': 'not malicious'
}
}
}
expected_result = {
'md5': {
'feca1deadbeefcafebeadbeefcafebee'
}
}
result = ThreatIntel._setup_excluded_iocs(excluded_iocs)
assert_equal(result, expected_result)
def test_deserialize(self):
"""ThreatIntel - Deserialize"""
data = [
{
'ioc_value': {
'S': '09bb8985ca0a702907dbcfad511d138e'
},
'sub_type': {
'S': 'mal_md5'
}
}
]
expected_result = [
{
'ioc_value': '09bb8985ca0a702907dbcfad511d138e',
'sub_type': 'mal_md5'
}
]
result = list(ThreatIntel._deserialize(data))
assert_equal(result, expected_result)
def test_setup_excluded_iocs_ip(self):
"""ThreatIntel - Setup Excluded IOCs, With IPs"""
excluded_iocs = {
'ip': {
'10.0.0.0/8': {
'comment': 'RFC1918'
}
},
'md5': {
'feca1deadbeefcafebeadbeefcafebee': {
'comment': 'not malicious'
}
}
}
expected_result = {
'ip': {
IPNetwork('10.0.0.0/8')
},
'md5': {
'feca1deadbeefcafebeadbeefcafebee'
}
}
result = ThreatIntel._setup_excluded_iocs(excluded_iocs)
assert_equal(result, expected_result)
def test_load_from_config_empty(self):
"""ThreatIntel - Load From Config, Empty"""
assert_equal(ThreatIntel.load_from_config({}), None)
def setup(self):
"""ThreatIntel - Setup"""
with patch('boto3.client'):
self._threat_intel = ThreatIntel.load_from_config(self._default_config)