def __init__(self,
initial_domains=None,
initial_ips=None,
generations=DEFAULT_RELATED_DOMAINS_GENERATIONS,
related_when=None,
**kwargs):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__(**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep(
'opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'),
cache_file_name=cache_file_name)
self._domains_to_lookup = set(
initial_domains) if initial_domains else set()
self._ips_to_lookup = set(initial_ips) if initial_ips else set()
self._related_when = related_when
self._generation_count = generations
self._all_blobs = list()
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(ApiCache,
'bulk_lookup',
autospec=True,
return_value={}),
patch.object(MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain][
'suspicious'] = self._is_category_info_suspicious(
categorized[domain])
# Decide which values to lookup security info for
iocs = filter(
lambda domain: self._should_get_security_info(
domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(
security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorized[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = [x for x in all_iocs if not self._whitelist.match_values(x)]
categorization = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain, categorization_info in six.iteritems(categorization):
if categorization_info:
categorization_info['suspicious'] = \
self._is_category_info_suspicious(categorization_info)
else:
logging.warning(
'No categorization for domain {0}'.format(domain), )
categorization[domain] = {'suspicious': False}
# Decide which values to lookup security info for
iocs = [
domain for domain in categorization
if self._should_get_security_info(categorization[domain])
]
security = investigate.security(iocs)
for domain, security_info in six.iteritems(security):
if security_info:
security_info['suspicious'] = \
self._is_security_info_suspicious(security_info)
else:
logging.warning(
'No security information for domain {0}'.format(domain), )
security[domain] = {'suspicious': False}
for domain in security:
if self._should_store_ioc_info(categorization[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorization[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(
domain.encode('utf-8', errors='ignore')
if six.PY2 else domain, ),
}
return threat_info
