def get_kwargs(): """Get kwargs for the analyzer. Returns: List of features to search for. """ features_config = interface.get_yaml_config('features.yaml') if not features_config: return 'Unable to parse the config features file.' features_kwargs = [ {'feature': feature, 'feature_config': config} for feature, config in features_config.items() ] return features_kwargs
def get_kwargs(): """Get kwargs for the analyzer. Returns: List of searches to tag results for. """ tags_config = interface.get_yaml_config('tags.yaml') if not tags_config: return 'Unable to parse the tags config file.' tags_kwargs = [{ 'tag': tag, 'tag_config': config } for tag, config in tags_config.items()] return tags_kwargs
def get_kwargs(): """Get kwargs for the analyzer. Returns: List of matchers. """ bq_config = interface.get_yaml_config('bigquery_matcher.yaml') if not bq_config: logger.error('BigQuery Matcher could not load configuration file.') return [] matcher_kwargs = [{ 'matcher_name': matcher_name, 'matcher_config': matcher_config } for matcher_name, matcher_config in bq_config.items()] return matcher_kwargs
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result. """ config = self._config or interface.get_yaml_config(self.CONFIG_FILE) if not config: return 'Unable to parse the config file.' tag_results = [] for name, tag_config in iter(config.items()): tag_result = self.tagger(name, tag_config) if tag_result: tag_results.append(tag_result) return ', '.join(tag_results)
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result. """ config = self._config or interface.get_yaml_config(self.CONFIG_FILE) if not config: return 'Unable to parse the config file.' return_strings = [] for name, feature_config in iter(config.items()): feature_string = self.extract_feature(name, feature_config) if feature_string: return_strings.append(feature_string) return ', '.join(return_strings)
def ontology(): """Return a dict with the ontology definitions.""" return interface.get_yaml_config("ontology.yaml")
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ # Exit ASAP if the API key is missing. if not self._safebrowsing_api_key: return 'Safe Browsing API requires an API key!' query = ('{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }} ] } } }') return_fields = ['url'] events = self.event_stream( query_dsl=query, return_fields=return_fields, ) urls = {} for event in events: url = self._sanitize_url(event.source.get('url')) if not url: continue urls.setdefault(url, []).append(event) # Exit early if there are no URLs in the data set to analyze. if not urls: return 'No URLs to analyze.' url_allowlisted = 0 url_allowlist = set( interface.get_yaml_config(self._URL_ALLOW_LIST_CONFIG, ), ) if not url_allowlist: domain_analyzer_allowlisted = current_app.config.get( 'DOMAIN_ANALYZER_EXCLUDE_DOMAINS', [], ) for domain in domain_analyzer_allowlisted: url_allowlist.add('*.%s/*' % domain) logger.info( '{0:d} entries on the allowlist.'.format(len(url_allowlist)), ) safebrowsing_platforms = current_app.config.get( 'SAFEBROWSING_PLATFORMS', ['ANY_PLATFORM'], ) safebrowsing_types = current_app.config.get( 'SAFEBROWSING_THREATTYPES', ['MALWARE'], ) lookup_urls = [] for url in urls: if self._is_url_allowlisted(url, url_allowlist): url_allowlisted += 1 continue lookup_urls.append(url) try: safebrowsing_results = self._do_safebrowsing_lookup( lookup_urls, safebrowsing_platforms, safebrowsing_types, ) except requests.HTTPError: return 'Couldn\'t reach the Safe Browsing API.' for url in lookup_urls: safebrowsing_result = safebrowsing_results.get(url) if not safebrowsing_result: continue for event in urls[url]: tags = ['google-safebrowsing-url'] threat_type = safebrowsing_result.get('threatType') if threat_type: tags.append('google-safebrowsing-%s' % threat_type.lower(), ) event.add_tags(tags) threat_attributes = [] for item in safebrowsing_result.items(): threat_attributes.append('%s: %s' % item) event.add_attributes( { 'google-safebrowsing-threat': ', '.join(threat_attributes), }, ) event.commit() return ('{0:d} Safe Browsing result(s) on {1:d} URL(s), ' '{2:d} on the allow list.').format( len(safebrowsing_results), len(urls), url_allowlisted, )