def __init__(self,
initial_domains=None,
initial_ips=None,
generations=DEFAULT_RELATED_DOMAINS_GENERATIONS,
related_when=None,
**kwargs):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__(**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep(
'opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'),
cache_file_name=cache_file_name)
self._domains_to_lookup = set(
initial_domains) if initial_domains else set()
self._ips_to_lookup = set(initial_ips) if initial_ips else set()
self._related_when = related_when
self._generation_count = generations
self._all_blobs = list()
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(ApiCache,
'bulk_lookup',
autospec=True,
return_value={}),
patch.object(MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain][
'suspicious'] = self._is_category_info_suspicious(
categorized[domain])
# Decide which values to lookup security info for
iocs = filter(
lambda domain: self._should_get_security_info(
domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(
security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorized[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
def __init__(self,
initial_domains=None,
initial_ips=None,
generations=DEFAULT_RELATED_DOMAINS_GENERATIONS,
related_when=None,
**kwargs):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__(**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._domains_to_lookup = set(initial_domains) if initial_domains else set()
self._ips_to_lookup = set(initial_ips) if initial_ips else set()
self._related_when = related_when
self._generation_count = generations
self._all_blobs = list()
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep('opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key, cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain]['suspicious'] = self._is_category_info_suspicious(categorized[domain])
# Decide which values to lookup security info for
iocs = filter(lambda domain: self._should_get_security_info(domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain], security[domain]):
threat_info[domain] = {
'domain': domain,
'categorization': categorized[domain],
'security': self._trim_security_result(security[domain]),
'link': 'https://investigate.opendns.com/domain-view/name/{0}/view'.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(
ApiCache, 'bulk_lookup', autospec=True, return_value={}),
patch.object(
MultiRequest, 'multi_post', autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
class InvestigateApiTest(T.TestCase):
"""Tests requesting reports from OpenDNS."""
@T.setup
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _test_api_call_get(self, call, endpoint, request, expected_query_params, api_response, expected_result):
"""
Tests a OpenDNS call by mocking out the HTTP GET request.
Args:
call: function in OpenDNSApi to call.
endpoint: endpoint of OpenDNS API that is hit (appended to base url)
request: call arguments
expected_query_params: query parameters that should be passed to API
api_response: the expected response by the API
expected_result: what call should return (given the api response provided)
"""
with patch.object(self.opendns, '_requests') as request_mock:
request_mock.multi_get.return_value = api_response
result = call(request)
url = self.opendns._to_url(endpoint.format(expected_query_params))
request_mock.multi_get.assert_called_with([url])
T.assert_equal(result, expected_result)
def test_security(self):
self._test_api_call_get(call=self.opendns.security,
endpoint=u'security/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_whois_emails(self):
self._test_api_call_get(call=self.opendns.whois_emails,
endpoint=u'whois/emails/{0}',
request=['*****@*****.**'],
expected_query_params='*****@*****.**',
api_response={},
expected_result={})
def test_whois_nameservers(self):
self._test_api_call_get(call=self.opendns.whois_nameservers,
endpoint=u'whois/nameservers/{0}',
request=['ns.dns.com'],
expected_query_params='ns.dns.com',
api_response={},
expected_result={})
def test_whois_domains(self):
self._test_api_call_get(call=self.opendns.whois_domains,
endpoint=u'whois/{0}',
request=['google.com'],
expected_query_params='google.com',
api_response={},
expected_result={})
def test_whois_domains_history(self):
self._test_api_call_get(call=self.opendns.whois_domains_history,
endpoint=u'whois/{0}/history',
request=['5esb.biz'],
expected_query_params='5esb.biz',
api_response={},
expected_result={})
def test_coocurrences(self):
self._test_api_call_get(call=self.opendns.cooccurrences,
endpoint=u'recommendations/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_rr_history(self):
self._test_api_call_get(call=self.opendns.rr_history,
endpoint=u'dnsdb/ip/a/{0}.json',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_latest_malicious(self):
self._test_api_call_get(call=self.opendns.latest_malicious,
endpoint=u'ips/{0}/latest_domains',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_domain_tag(self):
self._test_api_call_get(call=self.opendns.domain_tag,
endpoint=u'domains/{0}/latest_tags',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_dns_rr(self):
self._test_api_call_get(call=self.opendns.dns_rr,
endpoint=u'dnsdb/name/a/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_related_domains(self):
self._test_api_call_get(call=self.opendns.related_domains,
endpoint=u'links/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
class InvestigateApiTest(T.TestCase):
"""Tests requesting reports from OpenDNS."""
@T.setup
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _patch_and_assert_categorization(self, all_responses,
expected_responses, domains,
expected_url, expected_data):
with patch.object(MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses) as patched_multi_post:
actual_responses = self.opendns.categorization(domains)
patched_multi_post.assert_called_with(ANY,
expected_url,
data=expected_data)
assert expected_responses == actual_responses
def test_categorization(self):
domains = [
'yellowstone.org', 'zion.org', 'sequoia.org', 'greatsanddunes.org'
]
all_responses = [{
u'yellowstone.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'zion.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'sequoia.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'greatsanddunes.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}]
expected_url = u'https://investigate.api.opendns.com/domains/categorization/?showLabels'
expected_data = [
'["yellowstone.org", "zion.org", "sequoia.org", "greatsanddunes.org"]'
]
expected_responses = all_responses[0]
self._patch_and_assert_categorization(all_responses,
expected_responses, domains,
expected_url, expected_data)
def test_categorization_domains_limit(self):
self.opendns.MAX_DOMAINS_IN_POST = 2
domains = [
'northyorkmoors.org.uk', 'peakdistrict.org.uk',
'cairngorms.org.uk', 'pembrokeshirecoast.org.uk',
'northumberland.org.uk'
]
all_responses = [{
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
}, {
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
}, {
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}]
expected_data = [
'["northyorkmoors.org.uk", "peakdistrict.org.uk"]',
'["cairngorms.org.uk", "pembrokeshirecoast.org.uk"]',
'["northumberland.org.uk"]'
]
expected_responses = {
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
self._patch_and_assert_categorization(all_responses,
expected_responses, domains, ANY,
expected_data)
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with patch.object(builtins, 'open', mock_read,
create=True), patch.object(
ApiCache,
'bulk_lookup',
autospec=True,
return_value={}), patch.object(
MultiRequest,
'multi_post',
autospec=True,
return_value=all_responses):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _test_api_call_get(self,
call,
endpoint,
request,
expected_url_params,
api_response,
expected_result,
expected_query_params=None):
"""
Tests a OpenDNS call by mocking out the HTTP GET request.
Args:
call: function in OpenDNSApi to call.
endpoint: endpoint of OpenDNS API that is hit (appended to base url)
request: call arguments
expected_url_params: URL parameters that should be passed to API
api_response: the expected response by the API
expected_result: what call should return (given the api response provided)
expected_query_params: query parameters that should be passed to API
"""
with patch.object(self.opendns, '_requests') as request_mock:
request_mock.multi_get.return_value = api_response
result = call(request)
url = self.opendns._to_url(endpoint.format(expected_url_params))
request_mock.multi_get.assert_called_with([url],
expected_query_params)
T.assert_equal(result, expected_result)
def test_security(self):
self._test_api_call_get(call=self.opendns.security,
endpoint=u'security/name/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_whois_emails(self):
self._test_api_call_get(call=self.opendns.whois_emails,
endpoint=u'whois/emails/{0}',
request=['*****@*****.**'],
expected_url_params='*****@*****.**',
api_response={},
expected_result={})
def test_whois_nameservers(self):
self._test_api_call_get(call=self.opendns.whois_nameservers,
endpoint=u'whois/nameservers/{0}',
request=['ns.dns.com'],
expected_url_params='ns.dns.com',
api_response={},
expected_result={})
def test_whois_domains(self):
self._test_api_call_get(call=self.opendns.whois_domains,
endpoint=u'whois/{0}',
request=['google.com'],
expected_url_params='google.com',
api_response={},
expected_result={})
def test_whois_domains_history(self):
self._test_api_call_get(call=self.opendns.whois_domains_history,
endpoint=u'whois/{0}/history',
request=['5esb.biz'],
expected_url_params='5esb.biz',
api_response={},
expected_result={})
def test_coocurrences(self):
self._test_api_call_get(call=self.opendns.cooccurrences,
endpoint=u'recommendations/name/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_rr_history(self):
self._test_api_call_get(call=self.opendns.rr_history,
endpoint=u'dnsdb/ip/a/{0}.json',
request=['8.8.8.8'],
expected_url_params='8.8.8.8',
api_response={},
expected_result={})
def test_latest_malicious(self):
self._test_api_call_get(call=self.opendns.latest_malicious,
endpoint=u'ips/{0}/latest_domains',
request=['8.8.8.8'],
expected_url_params='8.8.8.8',
api_response={},
expected_result={})
def test_domain_tag(self):
self._test_api_call_get(call=self.opendns.domain_tag,
endpoint=u'domains/{0}/latest_tags',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_dns_rr(self):
self._test_api_call_get(call=self.opendns.dns_rr,
endpoint=u'dnsdb/name/a/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_related_domains(self):
self._test_api_call_get(call=self.opendns.related_domains,
endpoint=u'links/name/{0}.json',
request=['domain'],
expected_url_params='domain',
api_response={},
expected_result={})
def test_sample(self):
self._test_api_call_get(
call=self.opendns.sample,
endpoint=u'sample/{0}',
request=['0492d93195451e41f568f68e7704eb0812bc2b19'],
expected_url_params='0492d93195451e41f568f68e7704eb0812bc2b19',
api_response={},
expected_result={})
def test_search(self):
self._test_api_call_get(call=self.opendns.search,
endpoint=u'search/{}',
request=['pattern'],
expected_url_params='pattern',
api_response={},
expected_result={},
expected_query_params={
'start': '-30days',
'includecategory': 'false',
'limit': 1000
})
class InvestigateApiTest(T.TestCase):
"""Tests requesting reports from OpenDNS."""
@T.setup
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _patch_and_assert_categorization(self, all_responses, expected_responses, domains, expected_url, expected_data):
with patch.object(MultiRequest, 'multi_post', autospec=True, return_value=all_responses) as patched_multi_post:
actual_responses = self.opendns.categorization(domains)
patched_multi_post.assert_called_with(ANY, expected_url, data=expected_data)
assert expected_responses == actual_responses
def test_categorization(self):
domains = ['yellowstone.org', 'zion.org', 'sequoia.org', 'greatsanddunes.org']
all_responses = [
{
u'yellowstone.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'zion.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'sequoia.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'greatsanddunes.org': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
]
expected_url = u'https://investigate.api.opendns.com/domains/categorization/?showLabels'
expected_data = ['["yellowstone.org", "zion.org", "sequoia.org", "greatsanddunes.org"]']
expected_responses = all_responses[0]
self._patch_and_assert_categorization(all_responses, expected_responses, domains, expected_url, expected_data)
def test_categorization_domains_limit(self):
self.opendns.MAX_DOMAINS_IN_POST = 2
domains = [
'northyorkmoors.org.uk', 'peakdistrict.org.uk',
'cairngorms.org.uk', 'pembrokeshirecoast.org.uk',
'northumberland.org.uk']
all_responses = [
{
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
},
{
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
},
{
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
]
expected_data = [
'["northyorkmoors.org.uk", "peakdistrict.org.uk"]',
'["cairngorms.org.uk", "pembrokeshirecoast.org.uk"]',
'["northumberland.org.uk"]']
expected_responses = {
u'northyorkmoors.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'peakdistrict.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'cairngorms.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'pembrokeshirecoast.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
},
u'northumberland.org.uk': {
u'content_categories': [u'National Parks'],
u'security_categories': [],
u'status': 1
}
}
self._patch_and_assert_categorization(all_responses, expected_responses, domains, ANY, expected_data)
def test_categorization_response_error(self):
"""Tests whether the ResponseError is raised when the response
returned from the actual API call is empty.
"""
domains = ['yosemite.gov', 'joushuatree.gov', 'deathvalley.gov']
# empty responses should raise an error
all_responses = [{}]
# mock cache file
mock_read = mock_open(read_data="{}")
with nested(
patch('__builtin__.open', mock_read, create=True),
patch.object(
ApiCache, 'bulk_lookup', autospec=True, return_value={}),
patch.object(
MultiRequest, 'multi_post', autospec=True,
return_value=all_responses),
) as (__, __, patched_multi_post):
i = InvestigateApi('hocus pocus', 'cache.json')
with T.assert_raises(ResponseError):
i.categorization(domains)
def _test_api_call_get(self, call, endpoint, request, expected_query_params, api_response, expected_result):
"""
Tests a OpenDNS call by mocking out the HTTP GET request.
Args:
call: function in OpenDNSApi to call.
endpoint: endpoint of OpenDNS API that is hit (appended to base url)
request: call arguments
expected_query_params: query parameters that should be passed to API
api_response: the expected response by the API
expected_result: what call should return (given the api response provided)
"""
with patch.object(self.opendns, '_requests') as request_mock:
request_mock.multi_get.return_value = api_response
result = call(request)
url = self.opendns._to_url(endpoint.format(expected_query_params))
request_mock.multi_get.assert_called_with([url])
T.assert_equal(result, expected_result)
def test_security(self):
self._test_api_call_get(call=self.opendns.security,
endpoint=u'security/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_whois_emails(self):
self._test_api_call_get(call=self.opendns.whois_emails,
endpoint=u'whois/emails/{0}',
request=['*****@*****.**'],
expected_query_params='*****@*****.**',
api_response={},
expected_result={})
def test_whois_nameservers(self):
self._test_api_call_get(call=self.opendns.whois_nameservers,
endpoint=u'whois/nameservers/{0}',
request=['ns.dns.com'],
expected_query_params='ns.dns.com',
api_response={},
expected_result={})
def test_whois_domains(self):
self._test_api_call_get(call=self.opendns.whois_domains,
endpoint=u'whois/{0}',
request=['google.com'],
expected_query_params='google.com',
api_response={},
expected_result={})
def test_whois_domains_history(self):
self._test_api_call_get(call=self.opendns.whois_domains_history,
endpoint=u'whois/{0}/history',
request=['5esb.biz'],
expected_query_params='5esb.biz',
api_response={},
expected_result={})
def test_coocurrences(self):
self._test_api_call_get(call=self.opendns.cooccurrences,
endpoint=u'recommendations/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_rr_history(self):
self._test_api_call_get(call=self.opendns.rr_history,
endpoint=u'dnsdb/ip/a/{0}.json',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_latest_malicious(self):
self._test_api_call_get(call=self.opendns.latest_malicious,
endpoint=u'ips/{0}/latest_domains',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_domain_tag(self):
self._test_api_call_get(call=self.opendns.domain_tag,
endpoint=u'domains/{0}/latest_tags',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_dns_rr(self):
self._test_api_call_get(call=self.opendns.dns_rr,
endpoint=u'dnsdb/name/a/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_related_domains(self):
self._test_api_call_get(call=self.opendns.related_domains,
endpoint=u'links/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
class InvestigateApiTest(T.TestCase):
"""Tests requesting reports from OpenDNS."""
@T.setup
def setup_opendns(self):
self.opendns = InvestigateApi('test_key')
def _test_api_call_get(self, call, endpoint, request,
expected_query_params, api_response,
expected_result):
"""
Tests a OpenDNS call by mocking out the HTTP GET request.
Args:
call: function in OpenDNSApi to call.
endpoint: endpoint of OpenDNS API that is hit (appended to base url)
request: call arguments
expected_query_params: query parameters that should be passed to API
api_response: the expected response by the API
expected_result: what call should return (given the api response provided)
"""
with patch.object(self.opendns, '_requests') as request_mock:
request_mock.multi_get.return_value = api_response
result = call(request)
url = self.opendns._to_url(endpoint.format(expected_query_params))
request_mock.multi_get.assert_called_with([url])
T.assert_equal(result, expected_result)
def test_security(self):
self._test_api_call_get(call=self.opendns.security,
endpoint=u'security/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_whois_emails(self):
self._test_api_call_get(call=self.opendns.whois_emails,
endpoint=u'whois/emails/{0}',
request=['*****@*****.**'],
expected_query_params='*****@*****.**',
api_response={},
expected_result={})
def test_whois_nameservers(self):
self._test_api_call_get(call=self.opendns.whois_nameservers,
endpoint=u'whois/nameservers/{0}',
request=['ns.dns.com'],
expected_query_params='ns.dns.com',
api_response={},
expected_result={})
def test_whois_domains(self):
self._test_api_call_get(call=self.opendns.whois_domains,
endpoint=u'whois/{0}',
request=['google.com'],
expected_query_params='google.com',
api_response={},
expected_result={})
def test_whois_domains_history(self):
self._test_api_call_get(call=self.opendns.whois_domains_history,
endpoint=u'whois/{0}/history',
request=['5esb.biz'],
expected_query_params='5esb.biz',
api_response={},
expected_result={})
def test_coocurrences(self):
self._test_api_call_get(call=self.opendns.cooccurrences,
endpoint=u'recommendations/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_rr_history(self):
self._test_api_call_get(call=self.opendns.rr_history,
endpoint=u'dnsdb/ip/a/{0}.json',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_latest_malicious(self):
self._test_api_call_get(call=self.opendns.latest_malicious,
endpoint=u'ips/{0}/latest_domains',
request=['8.8.8.8'],
expected_query_params='8.8.8.8',
api_response={},
expected_result={})
def test_domain_tag(self):
self._test_api_call_get(call=self.opendns.domain_tag,
endpoint=u'domains/{0}/latest_tags',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_dns_rr(self):
self._test_api_call_get(call=self.opendns.dns_rr,
endpoint=u'dnsdb/name/a/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def test_related_domains(self):
self._test_api_call_get(call=self.opendns.related_domains,
endpoint=u'links/name/{0}.json',
request=['domain'],
expected_query_params='domain',
api_response={},
expected_result={})
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = [x for x in all_iocs if not self._whitelist.match_values(x)]
categorization = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain, categorization_info in six.iteritems(categorization):
if categorization_info:
categorization_info['suspicious'] = \
self._is_category_info_suspicious(categorization_info)
else:
logging.warning(
'No categorization for domain {0}'.format(domain), )
categorization[domain] = {'suspicious': False}
# Decide which values to lookup security info for
iocs = [
domain for domain in categorization
if self._should_get_security_info(categorization[domain])
]
security = investigate.security(iocs)
for domain, security_info in six.iteritems(security):
if security_info:
security_info['suspicious'] = \
self._is_security_info_suspicious(security_info)
else:
logging.warning(
'No security information for domain {0}'.format(domain), )
security[domain] = {'suspicious': False}
for domain in security:
if self._should_store_ioc_info(categorization[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorization[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(
domain.encode('utf-8', errors='ignore')
if six.PY2 else domain, ),
}
return threat_info
class RelatedDomainsFilter(OutputFilter):
"""Uses OpenDNS to find domains related to input domains or IPs.
A whitelist of domains to ignore is read during initialization.
Adds 'osxcollector_related' key to the output:
```python
{
'osxcollector_related': {
'domains': {
'domain_in_line.com': ['related_domain.com'],
'another.com': ['1.2.3.4']
}
}
}
```
"""
def __init__(self,
initial_domains=None,
initial_ips=None,
generations=DEFAULT_RELATED_DOMAINS_GENERATIONS,
related_when=None,
**kwargs):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__(**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep(
'opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'),
cache_file_name=cache_file_name)
self._domains_to_lookup = set(
initial_domains) if initial_domains else set()
self._ips_to_lookup = set(initial_ips) if initial_ips else set()
self._related_when = related_when
self._generation_count = generations
self._all_blobs = list()
def filter_line(self, blob):
"""Accumulate a set of all domains.
Args:
blob: A dict representing one line of output from OSXCollector.
Returns:
A dict or None
"""
self._all_blobs.append(blob)
if 'osxcollector_domains' in blob and self._related_when and self._related_when(
blob):
for domain in blob.get('osxcollector_domains'):
self._domains_to_lookup.add(domain)
return None
def end_of_lines(self):
"""Called after all lines have been fed to filter_output_line.
The OutputFilter performs any processing that requires the complete input to have already been fed.
Returns:
An enumerable of dicts
"""
domains_to_related = self._perform_lookup_for_all_domains(
self._domains_to_lookup, self._ips_to_lookup)
if domains_to_related:
for blob in self._all_blobs:
for domain in blob.get('osxcollector_domains', []):
add_related_domains = False
if domain in domains_to_related:
blob.setdefault('osxcollector_related', {})
blob['osxcollector_related'].setdefault('domains', {})
blob['osxcollector_related']['domains'].setdefault(
domain, [])
blob['osxcollector_related']['domains'][
domain] += domains_to_related[domain]
add_related_domains = True
# Unique the related domains
if add_related_domains:
blob['osxcollector_related']['domains'][domain] = list(
set(blob['osxcollector_related']['domains']
[domain]))
return self._all_blobs
def get_argument_parser(self):
parser = ArgumentParser()
group = parser.add_argument_group('opendns.RelatedDomainsFilter')
group.add_argument(
'-d',
'--domain',
dest='initial_domains',
default=[],
action='append',
help=
'[OPTIONAL] Suspicious domains to use in pivoting. May be specified more than once.'
)
group.add_argument(
'-i',
'--ip',
dest='initial_ips',
default=[],
action='append',
help=
'[OPTIONAL] Suspicious IP to use in pivoting. May be specified more than once.'
)
group.add_argument(
'--related-domains-generations',
dest='generations',
default=DEFAULT_RELATED_DOMAINS_GENERATIONS,
help=
'[OPTIONAL] How many generations of related domains to lookup with OpenDNS'
)
return parser
def _filter_domains_by_whitelist(self, domains):
"""Remove all domains that are on the whitelist.
Args:
domains: An enumerable of domains
Returns:
An enumerable of domains
"""
return filter(lambda x: not self._whitelist.match_values(x),
list(domains))
def _perform_lookup_for_all_domains(self, domains_to_lookup,
ips_to_lookup):
"""Lookup all the domains related to the input domains or IPs.
Args:
domains_to_lookup: Enumerable of domains
ips_to_lookup: Enumerable of IPs
Returns:
A dict mapping {'related_domain': ['initial_domainA', 'initial_domainB']}
"""
self._domains_to_lookup = self._filter_domains_by_whitelist(
self._domains_to_lookup)
domains_to_related = {}
what_to_lookup = [(domain, True) for domain in domains_to_lookup
] + [(ip, False) for ip in ips_to_lookup]
for domain_or_ip, is_domain in what_to_lookup:
related_domains = self._perform_lookup_for_single_domain(
domain_or_ip, is_domain, self._generation_count)
related_domains = self._filter_domains_by_whitelist(
related_domains)
for related_domain in related_domains:
domains_to_related.setdefault(related_domain, set())
domains_to_related[related_domain].add(domain_or_ip)
return domains_to_related
def _perform_lookup_for_single_domain(self, domain_or_ip, is_domain,
generation_count):
"""Given a domain or IP, lookup the Nth related domains.
Args:
domain_or_ip: A string domain name or IP
is_domain: A boolean of whether the previous arg is a domain or IP
generation_count: A count of generations to lookup
Returns:
set of related domains
"""
domains_found = set([domain_or_ip]) if is_domain else set()
generation_results = set([domain_or_ip])
# For IPs, do one IP specific lookup then switch to domain lookups
if not is_domain:
generation_results = self._find_related_domains(
None, generation_results)
domains_found |= generation_results
generation_count -= 1
while generation_count > 0:
if len(generation_results):
generation_results = self._find_related_domains(
generation_results, None)
domains_found |= generation_results
generation_count -= 1
return domains_found
def _find_related_domains(self, domains, ips):
"""Calls OpenDNS to find related domains and normalizes the responses.
Args:
domains: An enumerable of domains
ips: An enumerable of IPs
Returns:
An enumerable of domains
"""
related_domains = set()
if domains:
domains = self._filter_domains_by_whitelist(domains)
cooccurrence_info = self._investigate.cooccurrences(domains)
cooccurrence_domains = self._cooccurrences_to_domains(
cooccurrence_info)
related_domains.update(cooccurrence_domains)
if ips:
rr_history_info = self._investigate.rr_history(ips)
related_domains.update(
self._rr_history_to_domains(rr_history_info))
return related_domains
def _cooccurrences_to_domains(self, cooccurrence_info):
"""Parse the results of a call to the OpenDNS cooccurrences endpoint.
Args:
cooccurrence_info: Result of a call to cooccurrences
Returns:
An enumerable of domains
"""
domains = set()
for domain, cooccurence in cooccurrence_info.iteritems():
for occur_domain in cooccurence.get('pfs2', []):
for elem in expand_domain(occur_domain[0]):
domains.add(elem)
return domains
def _rr_history_to_domains(self, rr_history_info):
"""Parse the results of a call to the OpenDNS rr_history endpoint.
Args:
rr_history_info: Result of a call to rr_history
Returns:
An enumerable of domains
"""
domains = set()
for ip, rr_history in rr_history_info.iteritems():
for rr_domain in rr_history.get('rrs', []):
for elem in expand_domain(rr_domain['rr']):
domains.add(elem)
return domains
class RelatedDomainsFilter(OutputFilter):
"""Uses OpenDNS to find domains related to input domains or IPs.
A whitelist of domains to ignore is read during initialization.
Adds 'osxcollector_related' key to the output:
```python
{
'osxcollector_related': {
'domains': {
'domain_in_line.com': ['related_domain.com'],
'another.com': ['1.2.3.4']
}
}
}
```
"""
def __init__(self,
initial_domains=None,
initial_ips=None,
generations=DEFAULT_RELATED_DOMAINS_GENERATIONS,
related_when=None,
**kwargs):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__(**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._domains_to_lookup = set(initial_domains) if initial_domains else set()
self._ips_to_lookup = set(initial_ips) if initial_ips else set()
self._related_when = related_when
self._generation_count = generations
self._all_blobs = list()
def filter_line(self, blob):
"""Accumulate a set of all domains.
Args:
blob: A dict representing one line of output from OSXCollector.
Returns:
A dict or None
"""
self._all_blobs.append(blob)
if 'osxcollector_domains' in blob and self._related_when and self._related_when(blob):
for domain in blob.get('osxcollector_domains'):
self._domains_to_lookup.add(domain)
return None
def end_of_lines(self):
"""Called after all lines have been fed to filter_output_line.
The OutputFilter performs any processing that requires the complete input to have already been fed.
Returns:
An enumerable of dicts
"""
domains_to_related = self._perform_lookup_for_all_domains(self._domains_to_lookup, self._ips_to_lookup)
if domains_to_related:
for blob in self._all_blobs:
for domain in blob.get('osxcollector_domains', []):
add_related_domains = False
if domain in domains_to_related:
blob.setdefault('osxcollector_related', {})
blob['osxcollector_related'].setdefault('domains', {})
blob['osxcollector_related']['domains'].setdefault(domain, [])
blob['osxcollector_related']['domains'][domain] += domains_to_related[domain]
add_related_domains = True
# Unique the related domains
if add_related_domains:
blob['osxcollector_related']['domains'][domain] = list(set(blob['osxcollector_related']['domains'][domain]))
return self._all_blobs
def get_argument_parser(self):
parser = ArgumentParser()
group = parser.add_argument_group('opendns.RelatedDomainsFilter')
group.add_argument('-d', '--domain', dest='initial_domains', default=[], action='append',
help='[OPTIONAL] Suspicious domains to use in pivoting. May be specified more than once.')
group.add_argument('-i', '--ip', dest='initial_ips', default=[], action='append',
help='[OPTIONAL] Suspicious IP to use in pivoting. May be specified more than once.')
group.add_argument('--related-domains-generations', dest='generations', default=DEFAULT_RELATED_DOMAINS_GENERATIONS,
help='[OPTIONAL] How many generations of related domains to lookup with OpenDNS')
return parser
def _filter_domains_by_whitelist(self, domains):
"""Remove all domains that are on the whitelist.
Args:
domains: An enumerable of domains
Returns:
An enumerable of domains
"""
return filter(lambda x: not self._whitelist.match_values(x), list(domains))
def _perform_lookup_for_all_domains(self, domains_to_lookup, ips_to_lookup):
"""Lookup all the domains related to the input domains or IPs.
Args:
domains_to_lookup: Enumerable of domains
ips_to_lookup: Enumerable of IPs
Returns:
A dict mapping {'related_domain': ['initial_domainA', 'initial_domainB']}
"""
self._domains_to_lookup = self._filter_domains_by_whitelist(self._domains_to_lookup)
domains_to_related = {}
what_to_lookup = [(domain, True) for domain in domains_to_lookup] + [(ip, False) for ip in ips_to_lookup]
for domain_or_ip, is_domain in what_to_lookup:
related_domains = self._perform_lookup_for_single_domain(domain_or_ip, is_domain, self._generation_count)
related_domains = self._filter_domains_by_whitelist(related_domains)
for related_domain in related_domains:
domains_to_related.setdefault(related_domain, set())
domains_to_related[related_domain].add(domain_or_ip)
return domains_to_related
def _perform_lookup_for_single_domain(self, domain_or_ip, is_domain, generation_count):
"""Given a domain or IP, lookup the Nth related domains.
Args:
domain_or_ip: A string domain name or IP
is_domain: A boolean of whether the previous arg is a domain or IP
generation_count: A count of generations to lookup
Returns:
set of related domains
"""
domains_found = set([domain_or_ip]) if is_domain else set()
generation_results = set([domain_or_ip])
# For IPs, do one IP specific lookup then switch to domain lookups
if not is_domain:
generation_results = self._find_related_domains(None, generation_results)
domains_found |= generation_results
generation_count -= 1
while generation_count > 0:
if len(generation_results):
generation_results = self._find_related_domains(generation_results, None)
domains_found |= generation_results
generation_count -= 1
return domains_found
def _find_related_domains(self, domains, ips):
"""Calls OpenDNS to find related domains and normalizes the responses.
Args:
domains: An enumerable of domains
ips: An enumerable of IPs
Returns:
An enumerable of domains
"""
related_domains = set()
if domains:
domains = self._filter_domains_by_whitelist(domains)
cooccurrence_info = self._investigate.cooccurrences(domains)
cooccurrence_domains = self._cooccurrences_to_domains(cooccurrence_info)
related_domains.update(cooccurrence_domains)
if ips:
rr_history_info = self._investigate.rr_history(ips)
related_domains.update(self._rr_history_to_domains(rr_history_info))
return related_domains
def _cooccurrences_to_domains(self, cooccurrence_info):
"""Parse the results of a call to the OpenDNS cooccurrences endpoint.
Args:
cooccurrence_info: Result of a call to cooccurrences
Returns:
An enumerable of domains
"""
domains = set()
for domain, cooccurence in cooccurrence_info.iteritems():
for occur_domain in cooccurence.get('pfs2', []):
for elem in expand_domain(occur_domain[0]):
domains.add(elem)
return domains
def _rr_history_to_domains(self, rr_history_info):
"""Parse the results of a call to the OpenDNS rr_history endpoint.
Args:
rr_history_info: Result of a call to rr_history
Returns:
An enumerable of domains
"""
domains = set()
for ip, rr_history in rr_history_info.iteritems():
for rr_domain in rr_history.get('rrs', []):
for elem in expand_domain(rr_domain['rr']):
domains.add(elem)
return domains