def test_get_tld_from_domain(self):
"""Test get_tld_from_domain function."""
domain = 'this.is.a.subdomain.example.com'
tld = utils.get_tld_from_domain(domain)
self.assertEqual(tld, 'example.com')
domain = 'a'
tld = utils.get_tld_from_domain(domain)
self.assertEqual(tld, 'a')
domain = 'example.com'
tld = utils.get_tld_from_domain(domain)
self.assertEqual(tld, 'example.com')
def test_get_tld_from_domain(self):
"""Test get_tld_from_domain function."""
domain = 'this.is.a.subdomain.example.com'
tld = utils.get_tld_from_domain(domain)
self.assertEquals(tld, 'example.com')
domain = 'a'
tld = utils.get_tld_from_domain(domain)
self.assertEquals(tld, 'a')
domain = 'example.com'
tld = utils.get_tld_from_domain(domain)
self.assertEquals(tld, 'example.com')
def test_get_tld_from_domain(self):
"""Test get_tld_from_domain function."""
domain = "this.is.a.subdomain.example.com"
tld = utils.get_tld_from_domain(domain)
self.assertEqual(tld, "example.com")
domain = "a"
tld = utils.get_tld_from_domain(domain)
self.assertEqual(tld, "a")
domain = "example.com"
tld = utils.get_tld_from_domain(domain)
self.assertEqual(tld, "example.com")
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result
"""
query = ('{"query": { "bool": { "should": [ '
'{ "exists" : { "field" : "url" }}, '
'{ "exists" : { "field" : "domain" }} ] } } }')
return_fields = ['domain', 'url', 'message', 'human_readable']
events = self.event_stream('',
query_dsl=query,
return_fields=return_fields)
domains = {}
domain_counter = collections.Counter()
tld_counter = collections.Counter()
for event in events:
domain = event.source.get('domain')
if not domain:
continue
domain_counter[domain] += 1
domains.setdefault(domain, [])
domains[domain].append(event)
tld = utils.get_tld_from_domain(domain)
tld_counter[tld] += 1
watched_domains_list = current_app.config.get(
'DOMAIN_ANALYZER_WATCHED_DOMAINS', [])
domain_threshold = current_app.config.get(
'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10)
watched_domains_list.extend([
utils.strip_www_from_domain(x)
for x, _ in domain_counter.most_common(domain_threshold)
])
watched_domains_list.extend(
[x for x, _ in tld_counter.most_common(domain_threshold)])
watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST)
watched_domains_list_temp = set(watched_domains_list)
watched_domains_list = []
for domain in watched_domains_list_temp:
if domain in self.domain_scoring_whitelist:
continue
if any(domain.endswith(x) for x in self.domain_scoring_whitelist):
continue
if '.' not in domain:
continue
watched_domains_list.append(domain)
watched_domains = {}
for domain in watched_domains_list:
minhash = self._get_minhash_from_domain(domain)
watched_domains[domain] = minhash
similar_domain_counter = 0
evil_emoji = emojis.get_emoji('SKULL_CROSSBONE')
phishing_emoji = emojis.get_emoji('FISHING_POLE')
for domain, _ in domain_counter.iteritems():
emojis_to_add = []
tags_to_add = []
text = None
similar_domains = self._get_similar_domains(
domain, watched_domains)
if similar_domains:
similar_domain_counter += 1
emojis_to_add.append(evil_emoji)
emojis_to_add.append(phishing_emoji)
tags_to_add.append('phishy-domain')
similar_text_list = [
'{0:s} [score: {1:.2f}]'.format(phishy_domain, score)
for phishy_domain, score in similar_domains
]
text = 'Domain {0:s} is similar to {1:s}'.format(
domain, ', '.join(similar_text_list))
if any(
domain.endswith(x)
for x in self.domain_scoring_whitelist):
tags_to_add.append('known-network')
for event in domains.get(domain, []):
event.add_emojis(emojis_to_add)
event.add_tags(tags_to_add)
if text:
event.add_human_readable(text, self.NAME, append=False)
if similar_domain_counter:
self.sketch.add_view(view_name='Phishy Domains',
analyzer_name=self.NAME,
query_string='tag:"phishy-domain"')
return ('{0:d} potentially phishy domains discovered.'
).format(similar_domain_counter)
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result
"""
query = (
'{"query": { "bool": { "should": [ '
'{ "exists" : { "field" : "url" }}, '
'{ "exists" : { "field" : "domain" }} ] } } }'
)
return_fields = ["domain", "url", "message", "human_readable"]
events = self.event_stream("", query_dsl=query, return_fields=return_fields)
domains = {}
domain_counter = collections.Counter()
tld_counter = collections.Counter()
for event in events:
domain = event.source.get("domain")
if not domain:
continue
domain_counter[domain] += 1
domains.setdefault(domain, [])
domains[domain].append(event)
tld = utils.get_tld_from_domain(domain)
tld_counter[tld] += 1
if not domain_counter:
return "No domains discovered, so no phishy domains."
watched_domains_list = current_app.config.get(
"DOMAIN_ANALYZER_WATCHED_DOMAINS", []
)
domain_threshold = current_app.config.get(
"DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD", 10
)
watched_domains_list.extend(
[
utils.strip_www_from_domain(x)
for x, _ in domain_counter.most_common(domain_threshold)
]
)
watched_domains_list.extend(
[x for x, _ in tld_counter.most_common(domain_threshold)]
)
watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST)
watched_domains_list_temp = set(watched_domains_list)
watched_domains_list = []
for domain in watched_domains_list_temp:
if domain in self.domain_scoring_exclude_domains:
continue
if any(domain.endswith(x) for x in self.domain_scoring_exclude_domains):
continue
if "." not in domain:
continue
watched_domains_list.append(domain)
watched_domains = {}
for domain in watched_domains_list:
minhash = self._get_minhash_from_domain(domain)
watched_domains[domain] = {"hash": minhash, "depth": len(domain.split("."))}
similar_domain_counter = 0
allowlist_encountered = False
evil_emoji = emojis.get_emoji("SKULL_CROSSBONE")
phishing_emoji = emojis.get_emoji("FISHING_POLE")
for domain, _ in iter(domain_counter.items()):
emojis_to_add = []
tags_to_add = []
text = None
similar_domains = self._get_similar_domains(domain, watched_domains)
if similar_domains:
similar_domain_counter += 1
emojis_to_add.append(evil_emoji)
emojis_to_add.append(phishing_emoji)
tags_to_add.append("phishy-domain")
similar_text_list = [
"{0:s} [score: {1:.2f}]".format(phishy_domain, score)
for phishy_domain, score in similar_domains
]
text = "Domain {0:s} is similar to {1:s}".format(
domain, ", ".join(similar_text_list)
)
if any(domain.endswith(x) for x in self.domain_scoring_exclude_domains):
tags_to_add.append("known-domain")
allowlist_encountered = True
for event in domains.get(domain, []):
event.add_emojis(emojis_to_add)
event.add_tags(tags_to_add)
if text:
event.add_human_readable(text, self.NAME, append=False)
# Commit the event to the datastore.
event.commit()
if similar_domain_counter:
self.sketch.add_view(
view_name="Phishy Domains",
analyzer_name=self.NAME,
query_string='tag:"phishy-domain"',
)
if allowlist_encountered:
self.sketch.add_view(
view_name="Phishy Domains, excl. known domains",
analyzer_name=self.NAME,
query_string=('tag:"phishy-domain" AND NOT tag:"known-domain"'),
)
return ("{0:d} potentially phishy domains discovered.").format(
similar_domain_counter
)
def run(self):
"""Entry point for the analyzer.
Returns:
String with summary of the analyzer result
"""
query = (
'{"query": { "bool": { "should": [ '
'{ "exists" : { "field" : "url" }}, '
'{ "exists" : { "field" : "domain" }} ] } } }')
return_fields = ['domain', 'url', 'message', 'human_readable']
events = self.event_stream(
'', query_dsl=query, return_fields=return_fields)
domains = {}
domain_counter = collections.Counter()
tld_counter = collections.Counter()
for event in events:
domain = event.source.get('domain')
if not domain:
continue
domain_counter[domain] += 1
domains.setdefault(domain, [])
domains[domain].append(event)
tld = utils.get_tld_from_domain(domain)
tld_counter[tld] += 1
if not domain_counter:
return 'No domains discovered, so no phishy domains.'
watched_domains_list = current_app.config.get(
'DOMAIN_ANALYZER_WATCHED_DOMAINS', [])
domain_threshold = current_app.config.get(
'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10)
watched_domains_list.extend([
utils.strip_www_from_domain(x)
for x, _ in domain_counter.most_common(domain_threshold)])
watched_domains_list.extend([
x for x, _ in tld_counter.most_common(domain_threshold)])
watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST)
watched_domains_list_temp = set(watched_domains_list)
watched_domains_list = []
for domain in watched_domains_list_temp:
if domain in self.domain_scoring_whitelist:
continue
if any(domain.endswith(x) for x in self.domain_scoring_whitelist):
continue
if '.' not in domain:
continue
watched_domains_list.append(domain)
watched_domains = {}
for domain in watched_domains_list:
minhash = self._get_minhash_from_domain(domain)
watched_domains[domain] = {
'hash': minhash,
'depth': len(domain.split('.'))
}
similar_domain_counter = 0
whitelist_encountered = False
evil_emoji = emojis.get_emoji('SKULL_CROSSBONE')
phishing_emoji = emojis.get_emoji('FISHING_POLE')
for domain, _ in iter(domain_counter.items()):
emojis_to_add = []
tags_to_add = []
text = None
similar_domains = self._get_similar_domains(
domain, watched_domains)
if similar_domains:
similar_domain_counter += 1
emojis_to_add.append(evil_emoji)
emojis_to_add.append(phishing_emoji)
tags_to_add.append('phishy-domain')
similar_text_list = ['{0:s} [score: {1:.2f}]'.format(
phishy_domain,
score) for phishy_domain, score in similar_domains]
text = 'Domain {0:s} is similar to {1:s}'.format(
domain, ', '.join(similar_text_list))
if any(domain.endswith(
x) for x in self.domain_scoring_whitelist):
tags_to_add.append('whitelisted-domain')
whitelist_encountered = True
for event in domains.get(domain, []):
event.add_emojis(emojis_to_add)
event.add_tags(tags_to_add)
if text:
event.add_human_readable(text, self.NAME, append=False)
# Commit the event to the datastore.
event.commit()
if similar_domain_counter:
self.sketch.add_view(
view_name='Phishy Domains', analyzer_name=self.NAME,
query_string='tag:"phishy-domain"')
if whitelist_encountered:
self.sketch.add_view(
view_name='Phishy Domains, excl. whitelist',
analyzer_name=self.NAME,
query_string=(
'tag:"phishy-domain" AND NOT tag:"whitelisted-domain"'))
return (
'{0:d} potentially phishy domains discovered.').format(
similar_domain_counter)