def test_strip_www_from_domain(self): """Test strip_www_from_domain function.""" domain = 'www.mbl.is' stripped = utils.strip_www_from_domain(domain) self.assertEqual(stripped, 'mbl.is') domain = 'mbl.is' stripped = utils.strip_www_from_domain(domain) self.assertEqual(stripped, domain)
def test_strip_www_from_domain(self): """Test strip_www_from_domain function.""" domain = 'www.mbl.is' stripped = utils.strip_www_from_domain(domain) self.assertEquals(stripped, 'mbl.is') domain = 'mbl.is' stripped = utils.strip_www_from_domain(domain) self.assertEquals(stripped, domain)
def test_strip_www_from_domain(self): """Test strip_www_from_domain function.""" domain = "www.mbl.is" stripped = utils.strip_www_from_domain(domain) self.assertEqual(stripped, "mbl.is") domain = "mbl.is" stripped = utils.strip_www_from_domain(domain) self.assertEqual(stripped, domain)
def _get_similar_domains(self, domain, domain_dict): """Compare a domain to a list of domains and return similar domains. This function takes a domain and a dict object that contains as key domain names and value the calculated MinHash value for that domain. It will then strip www. if needed from the domain, and compare the Jaccard distance between all domains in the dict and the supplied domain (removing the TLD extension from all domains). If the Jaccard distance between the supplied domain and one or more of the domains in the domain dict is higher than the configured threshold the domain is further tested to see if there are overlapping substrings between the two domains. If there is a common substring that is longer than half the domain name and the Jaccard distance is above the threshold the domain is considered to be similar. Args: domain: string with a full domain, eg. www.google.com domain_dict: dict with domain names (keys) and MinHash objects (values) for all domains to compare against. Returns: a list of tuples (score, similar_domain_name) with the names of the similar domains as well as the Jaccard distance between the supplied domain and the matching one. """ domain = utils.strip_www_from_domain(domain) similar = [] if '.' not in domain: return similar if domain in domain_dict: return similar if any(domain.endswith('.{0:s}'.format(x)) for x in domain_dict): return similar minhash = self._get_minhash_from_domain(domain) # We want to get rid of the TLD extension of the domain. # This is only used in the substring match in case the Jaccard # distance is above the threshold. domain_items = domain.split('.') domain_part = '.'.join(domain_items[:-1]) for watched_domain, watched_hash in domain_dict.iteritems(): score = watched_hash.jaccard(minhash) if score < self.domain_scoring_threshold: continue watched_domain_items = watched_domain.split('.') watched_domain_part = '.'.join(watched_domain_items[:-1]) # Check if there are also any overlapping strings. sequence = difflib.SequenceMatcher(None, domain_part, watched_domain_part) match = sequence.find_longest_match(0, len(domain_part), 0, len(watched_domain_part)) # We want to have at least half of the domain matching. # TODO: This can be improved, this is a value and part that # needs or can be tweaked. Perhaps move this to a config option # that is the min length of strings. match_size = min(int(len(domain_part) / 2), int(len(watched_domain_part) / 2)) if match.size < match_size: continue similar.append((watched_domain, score)) return similar
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ query = ('{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }') return_fields = ['domain', 'url', 'message', 'human_readable'] events = self.event_stream('', query_dsl=query, return_fields=return_fields) domains = {} domain_counter = collections.Counter() tld_counter = collections.Counter() for event in events: domain = event.source.get('domain') if not domain: continue domain_counter[domain] += 1 domains.setdefault(domain, []) domains[domain].append(event) tld = utils.get_tld_from_domain(domain) tld_counter[tld] += 1 watched_domains_list = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS', []) domain_threshold = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10) watched_domains_list.extend([ utils.strip_www_from_domain(x) for x, _ in domain_counter.most_common(domain_threshold) ]) watched_domains_list.extend( [x for x, _ in tld_counter.most_common(domain_threshold)]) watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST) watched_domains_list_temp = set(watched_domains_list) watched_domains_list = [] for domain in watched_domains_list_temp: if domain in self.domain_scoring_whitelist: continue if any(domain.endswith(x) for x in self.domain_scoring_whitelist): continue if '.' not in domain: continue watched_domains_list.append(domain) watched_domains = {} for domain in watched_domains_list: minhash = self._get_minhash_from_domain(domain) watched_domains[domain] = minhash similar_domain_counter = 0 evil_emoji = emojis.get_emoji('SKULL_CROSSBONE') phishing_emoji = emojis.get_emoji('FISHING_POLE') for domain, _ in domain_counter.iteritems(): emojis_to_add = [] tags_to_add = [] text = None similar_domains = self._get_similar_domains( domain, watched_domains) if similar_domains: similar_domain_counter += 1 emojis_to_add.append(evil_emoji) emojis_to_add.append(phishing_emoji) tags_to_add.append('phishy-domain') similar_text_list = [ '{0:s} [score: {1:.2f}]'.format(phishy_domain, score) for phishy_domain, score in similar_domains ] text = 'Domain {0:s} is similar to {1:s}'.format( domain, ', '.join(similar_text_list)) if any( domain.endswith(x) for x in self.domain_scoring_whitelist): tags_to_add.append('known-network') for event in domains.get(domain, []): event.add_emojis(emojis_to_add) event.add_tags(tags_to_add) if text: event.add_human_readable(text, self.NAME, append=False) if similar_domain_counter: self.sketch.add_view(view_name='Phishy Domains', analyzer_name=self.NAME, query_string='tag:"phishy-domain"') return ('{0:d} potentially phishy domains discovered.' ).format(similar_domain_counter)
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ query = ( '{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }' ) return_fields = ["domain", "url", "message", "human_readable"] events = self.event_stream("", query_dsl=query, return_fields=return_fields) domains = {} domain_counter = collections.Counter() tld_counter = collections.Counter() for event in events: domain = event.source.get("domain") if not domain: continue domain_counter[domain] += 1 domains.setdefault(domain, []) domains[domain].append(event) tld = utils.get_tld_from_domain(domain) tld_counter[tld] += 1 if not domain_counter: return "No domains discovered, so no phishy domains." watched_domains_list = current_app.config.get( "DOMAIN_ANALYZER_WATCHED_DOMAINS", [] ) domain_threshold = current_app.config.get( "DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD", 10 ) watched_domains_list.extend( [ utils.strip_www_from_domain(x) for x, _ in domain_counter.most_common(domain_threshold) ] ) watched_domains_list.extend( [x for x, _ in tld_counter.most_common(domain_threshold)] ) watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST) watched_domains_list_temp = set(watched_domains_list) watched_domains_list = [] for domain in watched_domains_list_temp: if domain in self.domain_scoring_exclude_domains: continue if any(domain.endswith(x) for x in self.domain_scoring_exclude_domains): continue if "." not in domain: continue watched_domains_list.append(domain) watched_domains = {} for domain in watched_domains_list: minhash = self._get_minhash_from_domain(domain) watched_domains[domain] = {"hash": minhash, "depth": len(domain.split("."))} similar_domain_counter = 0 allowlist_encountered = False evil_emoji = emojis.get_emoji("SKULL_CROSSBONE") phishing_emoji = emojis.get_emoji("FISHING_POLE") for domain, _ in iter(domain_counter.items()): emojis_to_add = [] tags_to_add = [] text = None similar_domains = self._get_similar_domains(domain, watched_domains) if similar_domains: similar_domain_counter += 1 emojis_to_add.append(evil_emoji) emojis_to_add.append(phishing_emoji) tags_to_add.append("phishy-domain") similar_text_list = [ "{0:s} [score: {1:.2f}]".format(phishy_domain, score) for phishy_domain, score in similar_domains ] text = "Domain {0:s} is similar to {1:s}".format( domain, ", ".join(similar_text_list) ) if any(domain.endswith(x) for x in self.domain_scoring_exclude_domains): tags_to_add.append("known-domain") allowlist_encountered = True for event in domains.get(domain, []): event.add_emojis(emojis_to_add) event.add_tags(tags_to_add) if text: event.add_human_readable(text, self.NAME, append=False) # Commit the event to the datastore. event.commit() if similar_domain_counter: self.sketch.add_view( view_name="Phishy Domains", analyzer_name=self.NAME, query_string='tag:"phishy-domain"', ) if allowlist_encountered: self.sketch.add_view( view_name="Phishy Domains, excl. known domains", analyzer_name=self.NAME, query_string=('tag:"phishy-domain" AND NOT tag:"known-domain"'), ) return ("{0:d} potentially phishy domains discovered.").format( similar_domain_counter )
def _get_similar_domains(self, domain, domain_dict): """Compare a domain to a list of domains and return similar domains. This function takes a domain and a dict object that contains as key domain names and value the calculated MinHash value for that domain as well as the domains depth (mbl.is is 2, foobar.evil.com would be 3). It will then strip www. if needed from the domain, and compare the Jaccard distance between all domains in the dict and the supplied domain (removing the TLD extension from all domains). If the Jaccard distance between the supplied domain and one or more of the domains in the domain dict is higher than the configured threshold the domain is further tested to see if there are overlapping substrings between the two domains. If there is a common substring that is longer than half the domain name and the Jaccard distance is above the threshold the domain is considered to be similar. Args: domain: string with a full domain, eg. www.google.com domain_dict: dict with domain names (keys) and MinHash objects (values) for all domains to compare against. Returns: a list of tuples (score, similar_domain_name) with the names of the similar domains as well as the Jaccard distance between the supplied domain and the matching one. """ domain = utils.strip_www_from_domain(domain) similar = [] if '.' not in domain: return similar if domain in domain_dict: return similar if any(domain.endswith('.{0:s}'.format(x)) for x in domain_dict): return similar # We want to get rid of the TLD extension of the domain. # This is only used in the substring match in case the Jaccard # distance is above the threshold. domain_items = domain.split('.') domain_depth = len(domain_items) domain_part = '.'.join(domain_items[:-1]) minhashes = {} for index in range(0, domain_depth - 1): minhashes[domain_depth - index] = self._get_minhash_from_domain( '.'.join(domain_items[index:])) for watched_domain, watched_item in iter(domain_dict.items()): watched_hash = watched_item.get('hash') watched_depth = watched_item.get('depth') minhash = minhashes.get(watched_depth) if not minhash: # The supplied domains length does not match this watched # domain. continue score = watched_hash.jaccard(minhash) if score < self.domain_scoring_threshold: continue watched_domain_items = watched_domain.split('.') watched_domain_part = '.'.join(watched_domain_items[:-1]) # Check if there are also any overlapping strings. sequence = difflib.SequenceMatcher( None, domain_part, watched_domain_part) match = sequence.find_longest_match( 0, len(domain_part), 0, len(watched_domain_part)) # We want to have at least half of the domain matching. # TODO: This can be improved, this is a value and part that # needs or can be tweaked. Perhaps move this to a config option # that is the min length of strings. match_size = min( int(len(domain_part)/2), int(len(watched_domain_part)/2)) if match.size < match_size: continue similar.append((watched_domain, score)) return similar
def run(self): """Entry point for the analyzer. Returns: String with summary of the analyzer result """ query = ( '{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }') return_fields = ['domain', 'url', 'message', 'human_readable'] events = self.event_stream( '', query_dsl=query, return_fields=return_fields) domains = {} domain_counter = collections.Counter() tld_counter = collections.Counter() for event in events: domain = event.source.get('domain') if not domain: continue domain_counter[domain] += 1 domains.setdefault(domain, []) domains[domain].append(event) tld = utils.get_tld_from_domain(domain) tld_counter[tld] += 1 if not domain_counter: return 'No domains discovered, so no phishy domains.' watched_domains_list = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS', []) domain_threshold = current_app.config.get( 'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10) watched_domains_list.extend([ utils.strip_www_from_domain(x) for x, _ in domain_counter.most_common(domain_threshold)]) watched_domains_list.extend([ x for x, _ in tld_counter.most_common(domain_threshold)]) watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST) watched_domains_list_temp = set(watched_domains_list) watched_domains_list = [] for domain in watched_domains_list_temp: if domain in self.domain_scoring_whitelist: continue if any(domain.endswith(x) for x in self.domain_scoring_whitelist): continue if '.' not in domain: continue watched_domains_list.append(domain) watched_domains = {} for domain in watched_domains_list: minhash = self._get_minhash_from_domain(domain) watched_domains[domain] = { 'hash': minhash, 'depth': len(domain.split('.')) } similar_domain_counter = 0 whitelist_encountered = False evil_emoji = emojis.get_emoji('SKULL_CROSSBONE') phishing_emoji = emojis.get_emoji('FISHING_POLE') for domain, _ in iter(domain_counter.items()): emojis_to_add = [] tags_to_add = [] text = None similar_domains = self._get_similar_domains( domain, watched_domains) if similar_domains: similar_domain_counter += 1 emojis_to_add.append(evil_emoji) emojis_to_add.append(phishing_emoji) tags_to_add.append('phishy-domain') similar_text_list = ['{0:s} [score: {1:.2f}]'.format( phishy_domain, score) for phishy_domain, score in similar_domains] text = 'Domain {0:s} is similar to {1:s}'.format( domain, ', '.join(similar_text_list)) if any(domain.endswith( x) for x in self.domain_scoring_whitelist): tags_to_add.append('whitelisted-domain') whitelist_encountered = True for event in domains.get(domain, []): event.add_emojis(emojis_to_add) event.add_tags(tags_to_add) if text: event.add_human_readable(text, self.NAME, append=False) # Commit the event to the datastore. event.commit() if similar_domain_counter: self.sketch.add_view( view_name='Phishy Domains', analyzer_name=self.NAME, query_string='tag:"phishy-domain"') if whitelist_encountered: self.sketch.add_view( view_name='Phishy Domains, excl. whitelist', analyzer_name=self.NAME, query_string=( 'tag:"phishy-domain" AND NOT tag:"whitelisted-domain"')) return ( '{0:d} potentially phishy domains discovered.').format( similar_domain_counter)