def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def _lookup_iocs(self, all_iocs):
"""Caches the VirusTotal info for a set of domains.
Domains on a whitelist will be ignored.
Args:
all_iocs - a list of domains.
Returns:
A dict with domain as key and threat info as value
"""
threat_info = {}
cache_file_name = config_get_deep('virustotal.LookupDomainsFilter.cache_file_name', None)
vt = VirusTotalApi(self._api_key, cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
reports = vt.get_domain_reports(iocs)
for domain in reports.keys():
if not reports[domain]:
continue
trimmed_report = self._trim_domain_report(domain, reports[domain])
if self._should_store_ioc_info(trimmed_report):
threat_info[domain] = trimmed_report
return threat_info
def _lookup_iocs(self, all_iocs, resource_per_req=25):
"""Caches the VirusTotal info for a set of hashes.
Args:
all_iocs - a list of hashes.
Returns:
A dict with hash as key and threat info as value
"""
threat_info = {}
cache_file_name = config_get_deep(
'virustotal.LookupHashesFilter.cache_file_name', None)
vt = VirusTotalApi(self._api_key,
resource_per_req,
cache_file_name=cache_file_name)
reports = vt.get_file_reports(all_iocs)
for hash_val in reports:
report = reports[hash_val]
if not report:
continue
if self._should_store_ioc_info(report):
threat_info[hash_val] = self._trim_hash_report(report)
return threat_info
def __init__(self, lookup_when=None, **kwargs):
super(LookupDomainsFilter, self).__init__('osxcollector_domains',
'osxcollector_opendns',
lookup_when=lookup_when,
name_of_api_key='opendns',
**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def _lookup_iocs(self, all_iocs):
"""Caches the VirusTotal info for a set of domains.
Domains on a whitelist will be ignored.
Args:
all_iocs - a list of domains.
Returns:
A dict with domain as key and threat info as value
"""
threat_info = {}
cache_file_name = config_get_deep(
'virustotal.LookupDomainsFilter.cache_file_name', None)
vt = VirusTotalApi(self._api_key, cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
reports = vt.get_domain_reports(iocs)
for domain in reports.keys():
if not reports[domain]:
continue
trimmed_report = self._trim_domain_report(domain, reports[domain])
if self._should_store_ioc_info(trimmed_report):
threat_info[domain] = trimmed_report
return threat_info
def __init__(self,
ioc_key,
output_key,
lookup_when=None,
name_of_api_key=None,
**kwargs):
"""Configure the ThreatFeedFilter.
Args:
ioc_key: A string key to look for in each line of OSXCollector output.
The value of this key is the potential IOC to lookup in a threat feed.
output_key: A string key which is added to output lines and contains the result of threat feed lookups.
lookup_when: A boolean function to call to decide whether to perform a lookup on a line.
Use lookup_when to limit which IOCs are looked up.
name_of_api_key: A string name of the key in the 'api_key' section of config.
"""
super(ThreatFeedFilter, self).__init__(**kwargs)
if name_of_api_key:
self._api_key = config_get_deep(
'api_key.{0}'.format(name_of_api_key))
self._lookup_when = lookup_when
self._blobs_with_iocs = list()
self.ioc_set = set()
self._ioc_key = ioc_key
self._output_key = output_key
def __init__(self, lookup_when=None, **kwargs):
super(LookupDomainsFilter, self).__init__(
'osxcollector_domains', 'osxcollector_vtdomain',
lookup_when=lookup_when, name_of_api_key='virustotal', **kwargs
)
self._whitelist = create_blacklist(
config_get_deep('domain_whitelist'), kwargs.get('data_feeds', {}),
)
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain][
'suspicious'] = self._is_category_info_suspicious(
categorized[domain])
# Decide which values to lookup security info for
iocs = filter(
lambda domain: self._should_get_security_info(
domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(
security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorized[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(domain.encode('utf-8', errors='ignore'))
}
return threat_info
def _lookup_iocs(self, all_iocs):
"""Looks up the ShadowServer info for a set of hashes.
Args:
all_iocs - a list of hashes.
Returns:
A dict with hash as key and threat info as value
"""
cache_file_name = config_get_deep('shadowserver.LookupHashesFilter.cache_file_name', None)
ss = ShadowServerApi(cache_file_name=cache_file_name)
return ss.get_bin_test(all_iocs)
def _lookup_iocs(self, all_iocs):
"""Looks up the ShadowServer info for a set of hashes.
Args:
all_iocs - a list of hashes.
Returns:
A dict with hash as key and threat info as value
"""
cache_file_name = config_get_deep('shadowserver.LookupHashesFilter.cache_file_name', None)
ss = ShadowServerApi(cache_file_name=cache_file_name)
return ss.get_bin_test(all_iocs)
def _prepare_resource_chunks(self, resources, resource_delim=','):
"""As in some VirusTotal API methods the call can be made for multiple
resources at once this method prepares a list of concatenated resources
according to the maximum number of resources per requests.
Args:
resources: a list of the resources.
resource_delim: a string used to separate the resources.
Default value is a comma.
Returns:
A list of the concatenated resources.
"""
resources_per_req = config_get_deep('virustotal.resources_per_req', 25)
return [resource_delim.join(resources[pos:pos + resources_per_req]) for pos in xrange(0, len(resources), resources_per_req)]
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep('opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key, cache_file_name=cache_file_name)
iocs = filter(lambda x: not self._whitelist.match_values(x), all_iocs)
categorized = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain in categorized.keys():
categorized[domain]['suspicious'] = self._is_category_info_suspicious(categorized[domain])
# Decide which values to lookup security info for
iocs = filter(lambda domain: self._should_get_security_info(domain, categorized[domain]), categorized.keys())
security = investigate.security(iocs)
for domain in security.keys():
security[domain]['suspicious'] = self._is_security_info_suspicious(security[domain])
for domain in security.keys():
if self._should_store_ioc_info(categorized[domain], security[domain]):
threat_info[domain] = {
'domain': domain,
'categorization': categorized[domain],
'security': self._trim_security_result(security[domain]),
'link': 'https://investigate.opendns.com/domain-view/name/{0}/view'.format(domain)
}
return threat_info
def _lookup_iocs(self, domains, resource_per_req=25):
"""Caches the Alexa ranking info for a set of domains.
Args:
domains - a list of domains.
Returns:
A dict with domain as key and threat info as value
"""
traffic_info = {}
cache_file_name = config_get_deep('alexa.LookupRankingsFilter.cache_file_name', None)
ar = AlexaRankingApi(resource_per_req, cache_file_name=cache_file_name)
iocs = domains
reports = ar.get_alexa_rankings(iocs)
for domain in reports:
report = reports[domain]
if report and self._should_store_ioc_info(report):
traffic_info[domain] = report
return traffic_info
def __init__(self, ioc_key, output_key, lookup_when=None, name_of_api_key=None):
"""Configure the ThreatFeedFilter.
Args:
ioc_key: A string key to look for in each line of OSXCollector output.
The value of this key is the potential IOC to lookup in a threat feed.
output_key: A string key which is added to output lines and contains the result of threat feed lookups.
lookup_when: A boolean function to call to decide whether to perform a lookup on a line.
Use lookup_when to limit which IOCs are looked up.
name_of_api_key: A string name of the key in the 'api_key' section of config.
"""
super(ThreatFeedFilter, self).__init__()
if name_of_api_key:
self._api_key = config_get_deep('api_key.{0}'.format(name_of_api_key))
self._lookup_when = lookup_when
self._blobs_with_iocs = list()
self.ioc_set = set()
self._ioc_key = ioc_key
self._output_key = output_key
def _lookup_iocs(self, all_iocs):
"""Caches the VirusTotal info for a set of hashes.
Args:
all_iocs - a list of hashes.
Returns:
A dict with hash as key and threat info as value
"""
threat_info = {}
cache_file_name = config_get_deep('virustotal.LookupHashesFilter.cache_file_name', None)
vt = VirusTotalApi(self._api_key, cache_file_name=cache_file_name)
reports = vt.get_file_reports(all_iocs)
for hash_val in reports.keys():
report = reports[hash_val]
if not report:
continue
if self._should_store_ioc_info(report):
threat_info[hash_val] = self._trim_hash_report(report)
return threat_info
def _lookup_iocs(self, all_iocs, resource_per_req=25):
"""Caches the VirusTotal info for a set of URLs.
Args:
all_iocs - a list of URLs.
Returns:
A dict with URL as key and threat info as value
"""
threat_info = {}
cache_file_name = config_get_deep('virustotal.LookupURLsFilter.cache_file_name', None)
vt = VirusTotalApi(self._api_key, resource_per_req, cache_file_name=cache_file_name)
reports = vt.get_url_reports(all_iocs)
for url in reports.keys():
report = reports[url]
if not report:
continue
if self._should_store_ioc_info(report):
threat_info[url] = self._trim_url_report(report)
return threat_info
def _lookup_iocs(self, all_iocs):
"""Caches the VirusTotal info for a set of URLs.
Args:
all_iocs - a list of URLs.
Returns:
A dict with URL as key and threat info as value
"""
threat_info = {}
cache_file_name = config_get_deep(
'virustotal.LookupURLsFilter.cache_file_name', None)
vt = VirusTotalApi(self._api_key, cache_file_name=cache_file_name)
reports = vt.get_url_reports(all_iocs)
for url in reports.keys():
report = reports[url]
if not report:
continue
if self._should_store_ioc_info(report):
threat_info[url] = self._trim_url_report(report)
return threat_info
def test_read_top_level_key(self):
T.assert_equal('b', config_get_deep('a'))
def test_numeric_val(self):
assert config_get_deep('f') == 1
def _init_blacklists(self):
"""Reads the config and builds a list of blacklists."""
return [
create_blacklist(config_chunk)
for config_chunk in config_get_deep('blacklists')
]
def test_list_val(self):
assert config_get_deep('g') == ['apple', 'banana']
def test_list_val(self):
T.assert_equal(['apple', 'banana'], config_get_deep('g'))
def _init_blacklists(self):
"""Reads the config and builds a list of blacklists."""
return [create_blacklist(config_chunk) for config_chunk in config_get_deep('blacklists')]
def test_numeric_val(self):
T.assert_equal(1, config_get_deep('f'))
def test_read_multi_level_key(self):
T.assert_equal('e', config_get_deep('c.d'))
def test_list_val(self):
T.assert_equal(['apple', 'banana'], config_get_deep('g'))
def __init__(self, lookup_when=None, **kwargs):
super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_vtdomain',
lookup_when=lookup_when, name_of_api_key='virustotal', **kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def test_read_multi_level_key(self):
assert config_get_deep('c.d') == 'e'
def test_read_multi_level_key(self):
T.assert_equal('e', config_get_deep('c.d'))
def test_read_top_level_key(self):
T.assert_equal('b', config_get_deep('a'))
def test_numeric_val(self):
T.assert_equal(1, config_get_deep('f'))
def _lookup_iocs(self, all_iocs):
"""Caches the OpenDNS info for a set of domains.
Domains on a whitelist will be ignored.
First, lookup the categorization details for each domain.
Next, if the categorization seems suspicious or unknown, lookup detailed security info.
Finally, if the categorization or security info is suspicious, save the threat info.
Args:
all_iocs: an enumerable of string domain names.
Returns:
A dict {domain: opendns_info}
"""
threat_info = {}
cache_file_name = config_get_deep(
'opendns.LookupDomainsFilter.cache_file_name', None)
investigate = InvestigateApi(self._api_key,
cache_file_name=cache_file_name)
iocs = [x for x in all_iocs if not self._whitelist.match_values(x)]
categorization = investigate.categorization(iocs)
# Mark the categorization as suspicious
for domain, categorization_info in six.iteritems(categorization):
if categorization_info:
categorization_info['suspicious'] = \
self._is_category_info_suspicious(categorization_info)
else:
logging.warning(
'No categorization for domain {0}'.format(domain), )
categorization[domain] = {'suspicious': False}
# Decide which values to lookup security info for
iocs = [
domain for domain in categorization
if self._should_get_security_info(categorization[domain])
]
security = investigate.security(iocs)
for domain, security_info in six.iteritems(security):
if security_info:
security_info['suspicious'] = \
self._is_security_info_suspicious(security_info)
else:
logging.warning(
'No security information for domain {0}'.format(domain), )
security[domain] = {'suspicious': False}
for domain in security:
if self._should_store_ioc_info(categorization[domain],
security[domain]):
threat_info[domain] = {
'domain':
domain,
'categorization':
categorization[domain],
'security':
self._trim_security_result(security[domain]),
'link':
'https://investigate.opendns.com/domain-view/name/{0}/view'
.format(
domain.encode('utf-8', errors='ignore')
if six.PY2 else domain, ),
}
return threat_info
def __init__(self, lookup_when=None):
super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_opendns',
lookup_when=lookup_when, name_of_api_key='opendns')
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def test_read_top_level_key(self):
assert config_get_deep('a') == 'b'
