def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain][
'suspicious'] = self._is_category_info_suspicious(
categorized[domain])
# Decide which values to lookup security info for
iocs = filter(
lambda domain: self._should_get_security_info(
domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(
security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorized[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
def _lookup_iocs(self):
"""Caches the OpenDNS info for a set of domains"""
investigate = InvestigateApi(self._api_key)
categorized = investigate.categorization(list(self._all_iocs))
for domain in categorized.keys():
categorized_info = categorized[domain]
if self._should_get_security_info(domain, categorized_info):
security = investigate.security(domain)
if self._should_store_ioc_info(categorized_info, security):
self._threat_info_by_iocs[domain] = {
'domain': domain,
'categorization': categorized_info,
'security': security,
'link': 'https://investigate.opendns.com/domain-view/name/{0}/view'.format(domain)
}
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep('opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key, cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain]['suspicious'] = self._is_category_info_suspicious(categorized[domain])
# Decide which values to lookup security info for
iocs = filter(lambda domain: self._should_get_security_info(domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain], security[domain]):
threat_info[domain] = {
'domain': domain,
'categorization': categorized[domain],
'security': self._trim_security_result(security[domain]),
'link': 'https://investigate.opendns.com/domain-view/name/{0}/view'.format(domain)
}
return threat_info
class RelatedDomainsFilter(OutputFilter):
"""Uses OpenDNS to find domains related to input domains or ips."""
def __init__(self, initial_domains=None, initial_ips=None, depth=2):
super(RelatedDomainsFilter, self).__init__()
self._investigate = InvestigateApi(self.config.get_config('api_key.opendns'))
initial_domains = initial_domains or []
initial_ips = initial_ips or []
self._related_domains = set(initial_domains)
domains = initial_domains
ips = initial_ips
while depth > 0:
domains = self._find_related_domains(domains, ips)
ips = None
self._related_domains |= domains
depth -= 1
def _find_related_domains(self, domains, ips):
related_domains = set()
if domains:
for domain in self._investigate.cooccurrences(domains):
related_domains.add(domain)
if ips:
for domain in self._investigate.rr_history(ips):
related_domains.add(domain)
return related_domains
def filter_line(self, blob):
if self._related_domains and 'osxcollector_domains' in blob:
if any([domain in self._related_domains for domain in blob.get('osxcollector_domains')]):
blob.setdefault('osxcollector_related', [])
blob['osxcollector_related'].append('domains')
return blob
def __init__(self, initial_domains=None, initial_ips=None, depth=2):
super(RelatedDomainsFilter, self).__init__()
self._investigate = InvestigateApi(self.config.get_config('api_key.opendns'))
initial_domains = initial_domains or []
initial_ips = initial_ips or []
self._related_domains = set(initial_domains)
domains = initial_domains
ips = initial_ips
while depth > 0:
domains = self._find_related_domains(domains, ips)
ips = None
self._related_domains |= domains
depth -= 1
class RelatedDomainsFilter(OutputFilter):
"""Uses OpenDNS to find domains related to input domains or ips.
A whitelist of domains to ignore is read during initialization.
"""
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def filter_line(self, blob):
"""Accumulate a set of all domains.
Args:
blob: A dict representing one line of output from OSXCollector.
Returns:
A dict or None
"""
self._all_blobs.append(blob)
if 'osxcollector_domains' in blob and self._related_when and self._related_when(blob):
for domain in blob.get('osxcollector_domains'):
self._related_domains.add(domain)
return None
def end_of_lines(self):
"""Called after all lines have been fed to filter_output_line.
The OutputFilter performs any processing that requires the complete input to have already been fed.
Returns:
An enumerable of dicts
"""
domains = self._initial_domains
ips = self._initial_ips
generations = self._generations
while generations > 0:
domains = self._find_related_domains(domains, ips)
ips = None
self._related_domains |= domains
generations -= 1
self._related_domains = filter(lambda x: not self._whitelist.match_values(x), list(self._related_domains))
for blob in self._all_blobs:
if self._related_domains and 'osxcollector_domains' in blob:
for domain in blob.get('osxcollector_domains'):
if domain in self._related_domains:
blob.setdefault('osxcollector_related', {})
blob['osxcollector_related'].setdefault('domains', [])
blob['osxcollector_related']['domains'].append(domain)
return self._all_blobs
def _find_related_domains(self, domains, ips):
related_domains = set()
if domains:
cooccurrence_info = self._investigate.cooccurrences(domains)
related_domains.update(self._cooccurrences_to_domains(cooccurrence_info))
if ips:
rr_history_info = self._investigate.rr_history(ips)
related_domains.update(self._rr_history_to_domains(rr_history_info))
return related_domains
def _cooccurrences_to_domains(self, cooccurrence_info):
domains = []
for cooccurence in cooccurrence_info:
for occur_domain in cooccurence.get('pfs2', []):
for elem in expand_domain(occur_domain[0]):
domains.add(elem)
return domains
def _rr_history_to_domains(self, rr_history_info):
domains = []
for rr_history in rr_history_info:
for rr_domain in rr_history.get('rrs', []):
for elem in expand_domain(rr_domain['rr']):
domains.add(elem)
return domains
class RelatedDomainsFilter(OutputFilter):
"""Uses OpenDNS to find domains related to input domains or ips.
A whitelist of domains to ignore is read during initialization.
"""
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def filter_line(self, blob):
"""Accumulate a set of all domains.
Args:
blob: A dict representing one line of output from OSXCollector.
Returns:
A dict or None
"""
self._all_blobs.append(blob)
if 'osxcollector_domains' in blob and self._related_when and self._related_when(blob):
for domain in blob.get('osxcollector_domains'):
self._related_domains.add(domain)
return None
def end_of_lines(self):
"""Called after all lines have been fed to filter_output_line.
The OutputFilter performs any processing that requires the complete input to have already been fed.
Returns:
An enumerable of dicts
"""
domains = self._initial_domains
ips = self._initial_ips
generations = self._generations
while generations > 0:
domains = self._find_related_domains(domains, ips)
ips = None
self._related_domains |= domains
generations -= 1
self._related_domains = filter(lambda x: not self._whitelist.match_values(x), list(self._related_domains))
for blob in self._all_blobs:
if self._related_domains and 'osxcollector_domains' in blob:
for domain in blob.get('osxcollector_domains'):
if domain in self._related_domains:
blob.setdefault('osxcollector_related', {})
blob['osxcollector_related'].setdefault('domains', [])
blob['osxcollector_related']['domains'].append(domain)
return self._all_blobs
def _find_related_domains(self, domains, ips):
related_domains = set()
if domains:
cooccurrence_info = self._investigate.cooccurrences(domains)
related_domains.update(self.__cooccurrences_to_domains(cooccurrence_info))
if ips:
rr_history_info = self._investigate.rr_history(ips)
related_domains.update(self._rr_history_to_domains(rr_history_info))
return related_domains
def _cooccurrences_to_domains(self, cooccurrence_info):
domains = []
for cooccurence in cooccurrence_info:
for occur_domain in cooccurence.get('pfs2', []):
for elem in expand_domain(occur_domain[0]):
domains.add(elem)
return domains
def _rr_history_to_domains(self, rr_history_info):
domains = []
for rr_history in rr_history_info:
for rr_domain in rr_history.get('rrs', []):
for elem in expand_domain(rr_domain['rr']):
domains.add(elem)
return domains
