def get_kwargs():
"""Get kwargs for the analyzer.
Returns:
List of features to search for.
"""
features_config = interface.get_yaml_config('features.yaml')
if not features_config:
return 'Unable to parse the config features file.'
features_kwargs = [
{'feature': feature, 'feature_config': config}
for feature, config in features_config.items()
]
return features_kwargs
def get_kwargs():
"""Get kwargs for the analyzer.
Returns:
List of searches to tag results for.
"""
tags_config = interface.get_yaml_config('tags.yaml')
if not tags_config:
return 'Unable to parse the tags config file.'
tags_kwargs = [{
'tag': tag,
'tag_config': config
} for tag, config in tags_config.items()]
return tags_kwargs
def get_kwargs():
"""Get kwargs for the analyzer.
Returns:
List of matchers.
"""
bq_config = interface.get_yaml_config('bigquery_matcher.yaml')
if not bq_config:
logger.error('BigQuery Matcher could not load configuration file.')
return []
matcher_kwargs = [{
'matcher_name': matcher_name,
'matcher_config': matcher_config
} for matcher_name, matcher_config in bq_config.items()]
return matcher_kwargs
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result.
"""
config = self._config or interface.get_yaml_config(self.CONFIG_FILE)
if not config:
return 'Unable to parse the config file.'
tag_results = []
for name, tag_config in iter(config.items()):
tag_result = self.tagger(name, tag_config)
if tag_result:
tag_results.append(tag_result)
return ', '.join(tag_results)
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result.
"""
config = self._config or interface.get_yaml_config(self.CONFIG_FILE)
if not config:
return 'Unable to parse the config file.'
return_strings = []
for name, feature_config in iter(config.items()):
feature_string = self.extract_feature(name, feature_config)
if feature_string:
return_strings.append(feature_string)
return ', '.join(return_strings)
def ontology():
"""Return a dict with the ontology definitions."""
return interface.get_yaml_config("ontology.yaml")
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result
"""
# Exit ASAP if the API key is missing.
if not self._safebrowsing_api_key:
return 'Safe Browsing API requires an API key!'
query = ('{"query": { "bool": { "should": [ '
'{ "exists" : { "field" : "url" }} ] } } }')
return_fields = ['url']
events = self.event_stream(
query_dsl=query,
return_fields=return_fields,
)
urls = {}
for event in events:
url = self._sanitize_url(event.source.get('url'))
if not url:
continue
urls.setdefault(url, []).append(event)
# Exit early if there are no URLs in the data set to analyze.
if not urls:
return 'No URLs to analyze.'
url_allowlisted = 0
url_allowlist = set(
interface.get_yaml_config(self._URL_ALLOW_LIST_CONFIG, ), )
if not url_allowlist:
domain_analyzer_allowlisted = current_app.config.get(
'DOMAIN_ANALYZER_EXCLUDE_DOMAINS',
[],
)
for domain in domain_analyzer_allowlisted:
url_allowlist.add('*.%s/*' % domain)
logger.info(
'{0:d} entries on the allowlist.'.format(len(url_allowlist)), )
safebrowsing_platforms = current_app.config.get(
'SAFEBROWSING_PLATFORMS',
['ANY_PLATFORM'],
)
safebrowsing_types = current_app.config.get(
'SAFEBROWSING_THREATTYPES',
['MALWARE'],
)
lookup_urls = []
for url in urls:
if self._is_url_allowlisted(url, url_allowlist):
url_allowlisted += 1
continue
lookup_urls.append(url)
try:
safebrowsing_results = self._do_safebrowsing_lookup(
lookup_urls,
safebrowsing_platforms,
safebrowsing_types,
)
except requests.HTTPError:
return 'Couldn\'t reach the Safe Browsing API.'
for url in lookup_urls:
safebrowsing_result = safebrowsing_results.get(url)
if not safebrowsing_result:
continue
for event in urls[url]:
tags = ['google-safebrowsing-url']
threat_type = safebrowsing_result.get('threatType')
if threat_type:
tags.append('google-safebrowsing-%s' %
threat_type.lower(), )
event.add_tags(tags)
threat_attributes = []
for item in safebrowsing_result.items():
threat_attributes.append('%s: %s' % item)
event.add_attributes(
{
'google-safebrowsing-threat':
', '.join(threat_attributes),
}, )
event.commit()
return ('{0:d} Safe Browsing result(s) on {1:d} URL(s), '
'{2:d} on the allow list.').format(
len(safebrowsing_results),
len(urls),
url_allowlisted,
)
