def test_get_tld_from_domain(self): """Test get_tld_from_domain function.""" domain = 'this.is.a.subdomain.example.com' tld = utils.get_tld_from_domain(domain) self.assertEqual(tld, 'example.com') domain = 'a' tld = utils.get_tld_from_domain(domain) self.assertEqual(tld, 'a') domain = 'example.com' tld = utils.get_tld_from_domain(domain) self.assertEqual(tld, 'example.com')
def test_get_tld_from_domain(self): """Test get_tld_from_domain function.""" domain = 'this.is.a.subdomain.example.com' tld = utils.get_tld_from_domain(domain) self.assertEquals(tld, 'example.com') domain = 'a' tld = utils.get_tld_from_domain(domain) self.assertEquals(tld, 'a') domain = 'example.com' tld = utils.get_tld_from_domain(domain) self.assertEquals(tld, 'example.com')
def test_get_tld_from_domain(self): """Test get_tld_from_domain function.""" domain = "this.is.a.subdomain.example.com" tld = utils.get_tld_from_domain(domain) self.assertEqual(tld, "example.com") domain = "a" tld = utils.get_tld_from_domain(domain) self.assertEqual(tld, "a") domain = "example.com" tld = utils.get_tld_from_domain(domain) self.assertEqual(tld, "example.com")
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ query = ('{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }') return_fields = ['domain', 'url', 'message', 'human_readable'] events = self.event_stream('', query_dsl=query, return_fields=return_fields) domains = {} domain_counter = collections.Counter() tld_counter = collections.Counter() for event in events: domain = event.source.get('domain') if not domain: continue domain_counter[domain] += 1 domains.setdefault(domain, []) domains[domain].append(event) tld = utils.get_tld_from_domain(domain) tld_counter[tld] += 1 watched_domains_list = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS', []) domain_threshold = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10) watched_domains_list.extend([ utils.strip_www_from_domain(x) for x, _ in domain_counter.most_common(domain_threshold) ]) watched_domains_list.extend( [x for x, _ in tld_counter.most_common(domain_threshold)]) watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST) watched_domains_list_temp = set(watched_domains_list) watched_domains_list = [] for domain in watched_domains_list_temp: if domain in self.domain_scoring_whitelist: continue if any(domain.endswith(x) for x in self.domain_scoring_whitelist): continue if '.' not in domain: continue watched_domains_list.append(domain) watched_domains = {} for domain in watched_domains_list: minhash = self._get_minhash_from_domain(domain) watched_domains[domain] = minhash similar_domain_counter = 0 evil_emoji = emojis.get_emoji('SKULL_CROSSBONE') phishing_emoji = emojis.get_emoji('FISHING_POLE') for domain, _ in domain_counter.iteritems(): emojis_to_add = [] tags_to_add = [] text = None similar_domains = self._get_similar_domains( domain, watched_domains) if similar_domains: similar_domain_counter += 1 emojis_to_add.append(evil_emoji) emojis_to_add.append(phishing_emoji) tags_to_add.append('phishy-domain') similar_text_list = [ '{0:s} [score: {1:.2f}]'.format(phishy_domain, score) for phishy_domain, score in similar_domains ] text = 'Domain {0:s} is similar to {1:s}'.format( domain, ', '.join(similar_text_list)) if any( domain.endswith(x) for x in self.domain_scoring_whitelist): tags_to_add.append('known-network') for event in domains.get(domain, []): event.add_emojis(emojis_to_add) event.add_tags(tags_to_add) if text: event.add_human_readable(text, self.NAME, append=False) if similar_domain_counter: self.sketch.add_view(view_name='Phishy Domains', analyzer_name=self.NAME, query_string='tag:"phishy-domain"') return ('{0:d} potentially phishy domains discovered.' ).format(similar_domain_counter)
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ query = ( '{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }' ) return_fields = ["domain", "url", "message", "human_readable"] events = self.event_stream("", query_dsl=query, return_fields=return_fields) domains = {} domain_counter = collections.Counter() tld_counter = collections.Counter() for event in events: domain = event.source.get("domain") if not domain: continue domain_counter[domain] += 1 domains.setdefault(domain, []) domains[domain].append(event) tld = utils.get_tld_from_domain(domain) tld_counter[tld] += 1 if not domain_counter: return "No domains discovered, so no phishy domains." watched_domains_list = current_app.config.get( "DOMAIN_ANALYZER_WATCHED_DOMAINS", [] ) domain_threshold = current_app.config.get( "DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD", 10 ) watched_domains_list.extend( [ utils.strip_www_from_domain(x) for x, _ in domain_counter.most_common(domain_threshold) ] ) watched_domains_list.extend( [x for x, _ in tld_counter.most_common(domain_threshold)] ) watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST) watched_domains_list_temp = set(watched_domains_list) watched_domains_list = [] for domain in watched_domains_list_temp: if domain in self.domain_scoring_exclude_domains: continue if any(domain.endswith(x) for x in self.domain_scoring_exclude_domains): continue if "." not in domain: continue watched_domains_list.append(domain) watched_domains = {} for domain in watched_domains_list: minhash = self._get_minhash_from_domain(domain) watched_domains[domain] = {"hash": minhash, "depth": len(domain.split("."))} similar_domain_counter = 0 allowlist_encountered = False evil_emoji = emojis.get_emoji("SKULL_CROSSBONE") phishing_emoji = emojis.get_emoji("FISHING_POLE") for domain, _ in iter(domain_counter.items()): emojis_to_add = [] tags_to_add = [] text = None similar_domains = self._get_similar_domains(domain, watched_domains) if similar_domains: similar_domain_counter += 1 emojis_to_add.append(evil_emoji) emojis_to_add.append(phishing_emoji) tags_to_add.append("phishy-domain") similar_text_list = [ "{0:s} [score: {1:.2f}]".format(phishy_domain, score) for phishy_domain, score in similar_domains ] text = "Domain {0:s} is similar to {1:s}".format( domain, ", ".join(similar_text_list) ) if any(domain.endswith(x) for x in self.domain_scoring_exclude_domains): tags_to_add.append("known-domain") allowlist_encountered = True for event in domains.get(domain, []): event.add_emojis(emojis_to_add) event.add_tags(tags_to_add) if text: event.add_human_readable(text, self.NAME, append=False) # Commit the event to the datastore. event.commit() if similar_domain_counter: self.sketch.add_view( view_name="Phishy Domains", analyzer_name=self.NAME, query_string='tag:"phishy-domain"', ) if allowlist_encountered: self.sketch.add_view( view_name="Phishy Domains, excl. known domains", analyzer_name=self.NAME, query_string=('tag:"phishy-domain" AND NOT tag:"known-domain"'), ) return ("{0:d} potentially phishy domains discovered.").format( similar_domain_counter )
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ query = ( '{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }') return_fields = ['domain', 'url', 'message', 'human_readable'] events = self.event_stream( '', query_dsl=query, return_fields=return_fields) domains = {} domain_counter = collections.Counter() tld_counter = collections.Counter() for event in events: domain = event.source.get('domain') if not domain: continue domain_counter[domain] += 1 domains.setdefault(domain, []) domains[domain].append(event) tld = utils.get_tld_from_domain(domain) tld_counter[tld] += 1 if not domain_counter: return 'No domains discovered, so no phishy domains.' watched_domains_list = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS', []) domain_threshold = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10) watched_domains_list.extend([ utils.strip_www_from_domain(x) for x, _ in domain_counter.most_common(domain_threshold)]) watched_domains_list.extend([ x for x, _ in tld_counter.most_common(domain_threshold)]) watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST) watched_domains_list_temp = set(watched_domains_list) watched_domains_list = [] for domain in watched_domains_list_temp: if domain in self.domain_scoring_whitelist: continue if any(domain.endswith(x) for x in self.domain_scoring_whitelist): continue if '.' not in domain: continue watched_domains_list.append(domain) watched_domains = {} for domain in watched_domains_list: minhash = self._get_minhash_from_domain(domain) watched_domains[domain] = { 'hash': minhash, 'depth': len(domain.split('.')) } similar_domain_counter = 0 whitelist_encountered = False evil_emoji = emojis.get_emoji('SKULL_CROSSBONE') phishing_emoji = emojis.get_emoji('FISHING_POLE') for domain, _ in iter(domain_counter.items()): emojis_to_add = [] tags_to_add = [] text = None similar_domains = self._get_similar_domains( domain, watched_domains) if similar_domains: similar_domain_counter += 1 emojis_to_add.append(evil_emoji) emojis_to_add.append(phishing_emoji) tags_to_add.append('phishy-domain') similar_text_list = ['{0:s} [score: {1:.2f}]'.format( phishy_domain, score) for phishy_domain, score in similar_domains] text = 'Domain {0:s} is similar to {1:s}'.format( domain, ', '.join(similar_text_list)) if any(domain.endswith( x) for x in self.domain_scoring_whitelist): tags_to_add.append('whitelisted-domain') whitelist_encountered = True for event in domains.get(domain, []): event.add_emojis(emojis_to_add) event.add_tags(tags_to_add) if text: event.add_human_readable(text, self.NAME, append=False) # Commit the event to the datastore. event.commit() if similar_domain_counter: self.sketch.add_view( view_name='Phishy Domains', analyzer_name=self.NAME, query_string='tag:"phishy-domain"') if whitelist_encountered: self.sketch.add_view( view_name='Phishy Domains, excl. whitelist', analyzer_name=self.NAME, query_string=( 'tag:"phishy-domain" AND NOT tag:"whitelisted-domain"')) return ( '{0:d} potentially phishy domains discovered.').format( similar_domain_counter)