def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(ApiCache,
'bulk_lookup',
autospec=True,
return_value={}),
patch.object(MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain][
'suspicious'] = self._is_category_info_suspicious(
categorized[domain])
# Decide which values to lookup security info for
iocs = filter(
lambda domain: self._should_get_security_info(
domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(
security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorized[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(
ApiCache, 'bulk_lookup', autospec=True, return_value={}),
patch.object(
MultiRequest, 'multi_post', autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep('opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key, cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain]['suspicious'] = self._is_category_info_suspicious(categorized[domain])
# Decide which values to lookup security info for
iocs = filter(lambda domain: self._should_get_security_info(domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain], security[domain]):
threat_info[domain] = {
'domain': domain,
'categorization': categorized[domain],
'security': self._trim_security_result(security[domain]),
'link': 'https://investigate.opendns.com/domain-view/name/{0}/view'.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
class InvestigateApiTest(T.TestCase):
"""Tests requesting reports from OpenDNS."""
@T.setup
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _patch_and_assert_categorization(self, all_responses,
expected_responses, domains,
expected_url, expected_data):
with patch.object(MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses) as patched_multi_post:
actual_responses = self.opendns.categorization(domains)
patched_multi_post.assert_called_with(ANY,
expected_url,
data=expected_data)
assert expected_responses == actual_responses
def test_categorization(self):
domains = [
'yellowstone.org', 'zion.org', 'sequoia.org', 'greatsanddunes.org'
]
all_responses = [{
u'yellowstone.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'zion.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'sequoia.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'greatsanddunes.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}]
expected_url = u'https://investigate.api.opendns.com/domains/categorization/?showLabels'
expected_data = [
'["yellowstone.org", "zion.org", "sequoia.org", "greatsanddunes.org"]'
]
expected_responses = all_responses[0]
self._patch_and_assert_categorization(all_responses,
expected_responses, domains,
expected_url, expected_data)
def test_categorization_domains_limit(self):
self.opendns.MAX_DOMAINS_IN_POST = 2
domains = [
'northyorkmoors.org.uk', 'peakdistrict.org.uk',
'cairngorms.org.uk', 'pembrokeshirecoast.org.uk',
'northumberland.org.uk'
]
all_responses = [{
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
}, {
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
}, {
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}]
expected_data = [
'["northyorkmoors.org.uk", "peakdistrict.org.uk"]',
'["cairngorms.org.uk", "pembrokeshirecoast.org.uk"]',
'["northumberland.org.uk"]'
]
expected_responses = {
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
self._patch_and_assert_categorization(all_responses,
expected_responses, domains, ANY,
expected_data)
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with patch.object(builtins, 'open', mock_read,
create=True), patch.object(
ApiCache,
'bulk_lookup',
autospec=True,
return_value={}), patch.object(
MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _test_api_call_get(self,
call,
endpoint,
request,
expected_url_params,
api_response,
expected_result,
expected_query_params=None):
"""
Tests a OpenDNS call by mocking out the HTTP GET request.
Args:
call: function in OpenDNSApi to call.
endpoint: endpoint of OpenDNS API that is hit (appended to base url)
request: call arguments
expected_url_params: URL parameters that should be passed to API
api_response: the expected response by the API
expected_result: what call should return (given the api response provided)
expected_query_params: query parameters that should be passed to API
"""
with patch.object(self.opendns, '_requests') as request_mock:
request_mock.multi_get.return_value = api_response
result = call(request)
url = self.opendns._to_url(endpoint.format(expected_url_params))
request_mock.multi_get.assert_called_with([url],
expected_query_params)
T.assert_equal(result, expected_result)
def test_security(self):
self._test_api_call_get(call=self.opendns.security,
endpoint=u'security/name/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_whois_emails(self):
self._test_api_call_get(call=self.opendns.whois_emails,
endpoint=u'whois/emails/{0}',
request=['*****@*****.**'],
expected_url_params='*****@*****.**',
api_response={},
expected_result={})
def test_whois_nameservers(self):
self._test_api_call_get(call=self.opendns.whois_nameservers,
endpoint=u'whois/nameservers/{0}',
request=['ns.dns.com'],
expected_url_params='ns.dns.com',
api_response={},
expected_result={})
def test_whois_domains(self):
self._test_api_call_get(call=self.opendns.whois_domains,
endpoint=u'whois/{0}',
request=['google.com'],
expected_url_params='google.com',
api_response={},
expected_result={})
def test_whois_domains_history(self):
self._test_api_call_get(call=self.opendns.whois_domains_history,
endpoint=u'whois/{0}/history',
request=['5esb.biz'],
expected_url_params='5esb.biz',
api_response={},
expected_result={})
def test_coocurrences(self):
self._test_api_call_get(call=self.opendns.cooccurrences,
endpoint=u'recommendations/name/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_rr_history(self):
self._test_api_call_get(call=self.opendns.rr_history,
endpoint=u'dnsdb/ip/a/{0}.json',
request=['8.8.8.8'],
expected_url_params='8.8.8.8',
api_response={},
expected_result={})
def test_latest_malicious(self):
self._test_api_call_get(call=self.opendns.latest_malicious,
endpoint=u'ips/{0}/latest_domains',
request=['8.8.8.8'],
expected_url_params='8.8.8.8',
api_response={},
expected_result={})
def test_domain_tag(self):
self._test_api_call_get(call=self.opendns.domain_tag,
endpoint=u'domains/{0}/latest_tags',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_dns_rr(self):
self._test_api_call_get(call=self.opendns.dns_rr,
endpoint=u'dnsdb/name/a/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_related_domains(self):
self._test_api_call_get(call=self.opendns.related_domains,
endpoint=u'links/name/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_sample(self):
self._test_api_call_get(
call=self.opendns.sample,
endpoint=u'sample/{0}',
request=['0492d93195451e41f568f68e7704eb0812bc2b19'],
expected_url_params='0492d93195451e41f568f68e7704eb0812bc2b19',
api_response={},
expected_result={})
def test_search(self):
self._test_api_call_get(call=self.opendns.search,
endpoint=u'search/{}',
request=['pattern'],
expected_url_params='pattern',
api_response={},
expected_result={},
expected_query_params={
'start': '-30days',
'includecategory': 'false',
'limit': 1000
})
class InvestigateApiTest(T.TestCase):
"""Tests requesting reports from OpenDNS."""
@T.setup
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _patch_and_assert_categorization(self, all_responses, expected_responses, domains, expected_url, expected_data):
with patch.object(MultiRequest, 'multi_post', autospec=True, return_value=all_responses) as patched_multi_post:
actual_responses = self.opendns.categorization(domains)
patched_multi_post.assert_called_with(ANY, expected_url, data=expected_data)
assert expected_responses == actual_responses
def test_categorization(self):
domains = ['yellowstone.org', 'zion.org', 'sequoia.org', 'greatsanddunes.org']
all_responses = [
{
u'yellowstone.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'zion.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'sequoia.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'greatsanddunes.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
]
expected_url = u'https://investigate.api.opendns.com/domains/categorization/?showLabels'
expected_data = ['["yellowstone.org", "zion.org", "sequoia.org", "greatsanddunes.org"]']
expected_responses = all_responses[0]
self._patch_and_assert_categorization(all_responses, expected_responses, domains, expected_url, expected_data)
def test_categorization_domains_limit(self):
self.opendns.MAX_DOMAINS_IN_POST = 2
domains = [
'northyorkmoors.org.uk', 'peakdistrict.org.uk',
'cairngorms.org.uk', 'pembrokeshirecoast.org.uk',
'northumberland.org.uk']
all_responses = [
{
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
},
{
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
},
{
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
]
expected_data = [
'["northyorkmoors.org.uk", "peakdistrict.org.uk"]',
'["cairngorms.org.uk", "pembrokeshirecoast.org.uk"]',
'["northumberland.org.uk"]']
expected_responses = {
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
self._patch_and_assert_categorization(all_responses, expected_responses, domains, ANY, expected_data)
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(
ApiCache, 'bulk_lookup', autospec=True, return_value={}),
patch.object(
MultiRequest, 'multi_post', autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _test_api_call_get(self, call, endpoint, request, expected_query_params, api_response, expected_result):
"""
Tests a OpenDNS call by mocking out the HTTP GET request.
Args:
call: function in OpenDNSApi to call.
endpoint: endpoint of OpenDNS API that is hit (appended to base url)
request: call arguments
expected_query_params: query parameters that should be passed to API
api_response: the expected response by the API
expected_result: what call should return (given the api response provided)
"""
with patch.object(self.opendns, '_requests') as request_mock:
request_mock.multi_get.return_value = api_response
result = call(request)
url = self.opendns._to_url(endpoint.format(expected_query_params))
request_mock.multi_get.assert_called_with([url])
T.assert_equal(result, expected_result)
def test_security(self):
self._test_api_call_get(call=self.opendns.security,
endpoint=u'security/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_whois_emails(self):
self._test_api_call_get(call=self.opendns.whois_emails,
endpoint=u'whois/emails/{0}',
request=['*****@*****.**'],
expected_query_params='*****@*****.**',
api_response={},
expected_result={})
def test_whois_nameservers(self):
self._test_api_call_get(call=self.opendns.whois_nameservers,
endpoint=u'whois/nameservers/{0}',
request=['ns.dns.com'],
expected_query_params='ns.dns.com',
api_response={},
expected_result={})
def test_whois_domains(self):
self._test_api_call_get(call=self.opendns.whois_domains,
endpoint=u'whois/{0}',
request=['google.com'],
expected_query_params='google.com',
api_response={},
expected_result={})
def test_whois_domains_history(self):
self._test_api_call_get(call=self.opendns.whois_domains_history,
endpoint=u'whois/{0}/history',
request=['5esb.biz'],
expected_query_params='5esb.biz',
api_response={},
expected_result={})
def test_coocurrences(self):
self._test_api_call_get(call=self.opendns.cooccurrences,
endpoint=u'recommendations/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_rr_history(self):
self._test_api_call_get(call=self.opendns.rr_history,
endpoint=u'dnsdb/ip/a/{0}.json',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_latest_malicious(self):
self._test_api_call_get(call=self.opendns.latest_malicious,
endpoint=u'ips/{0}/latest_domains',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_domain_tag(self):
self._test_api_call_get(call=self.opendns.domain_tag,
endpoint=u'domains/{0}/latest_tags',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_dns_rr(self):
self._test_api_call_get(call=self.opendns.dns_rr,
endpoint=u'dnsdb/name/a/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_related_domains(self):
self._test_api_call_get(call=self.opendns.related_domains,
endpoint=u'links/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = [x for x in all_iocs if not self._whitelist.match_values(x)]
categorization = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain, categorization_info in six.iteritems(categorization):
if categorization_info:
categorization_info['suspicious'] = \
self._is_category_info_suspicious(categorization_info)
else:
logging.warning(
'No categorization for domain {0}'.format(domain), )
categorization[domain] = {'suspicious': False}
# Decide which values to lookup security info for
iocs = [
domain for domain in categorization
if self._should_get_security_info(categorization[domain])
]
security = investigate.security(iocs)
for domain, security_info in six.iteritems(security):
if security_info:
security_info['suspicious'] = \
self._is_security_info_suspicious(security_info)
else:
logging.warning(
'No security information for domain {0}'.format(domain), )
security[domain] = {'suspicious': False}
for domain in security:
if self._should_store_ioc_info(categorization[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorization[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(
domain.encode('utf-8', errors='ignore')
if six.PY2 else domain, ),
}
return threat_info
