def score(self, domains=None, start=False, end=False):
"""
scoring module for calculating a dedicated score for each domain.
:param domains: list of domains to check for alerts
:param start: Start date for creating alerts (default: last 24h). Format: YYYY-MM-DD
:param end: End date for creating alerts (default: now). Format: YYYY-MM-DD
:return:
"""
db = DatabaseUtils()
if not domains:
if not start:
start = (datetime.datetime.now() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
if not end:
end = datetime.datetime.now().strftime('%Y-%m-%d')
domains = db.get_registered_by_date(start, end)
if not isinstance(domains, list):
domains = [domains]
logger.info("Start Scoring ({} domains)".format(len(domains)))
scoring = Scoring()
s = scoring.score(domains)
db.add_scores(s)
logger.info("Completed Scoring")
def _store(self, data):
self.data.append(data)
if len(self.data) > self.store_threshold:
db = DatabaseUtils()
db.add_certificate_watch_data(self.data)
self.data.clear()
def get_alerts(self, status='open'):
"""
this method returns all alerts (default status='open')
:param status:
:return:
"""
db = DatabaseUtils()
alerts = db.get_alerts(status=status)
return alerts
def _heuristics(self, registered):
matches = list()
db = DatabaseUtils()
score_hits = db.get_score(score=SCORING_CONFIG['threshold'])
sorted_score = self._sorted(score_hits)
for r in registered:
if isinstance(r, dict):
r = r.get('value')
if r in sorted_score:
matches.append({'value': r, 'score': sorted_score[r][-1].get('score')})
return matches
def collect(self, download=False):
"""
calls collector module for collecting all domains from various sources and uploads them into database.
:return:
"""
logger.info("Start collecting new domains")
c = Collector()
c.collect(download=download)
db = DatabaseUtils()
db.add_registered_domains(c.domains)
logger.info("Completed collecting new domains: {}".format(
len(c.domains)))
return c.domains
def create(self, matches):
db = DatabaseUtils()
alert_ids = list()
for match in matches:
parent = match.get('parent')
value = match.get('value')
if isinstance(value, dict):
value = value.get('value')
if value in self.exists(value):
logger.info("Skipping alert creation for: {}".format(value))
continue
a_id = db.create_alerts(parent, value, match.get('source'), match.get('match'))
alert_ids.extend(a_id)
return alert_ids
def check_watchlist(self, domains):
db = DatabaseUtils()
domains_to_watch = db.get_domains()
matches = list()
for domain in domains_to_watch:
watchlist = db.get_watchlist_by_parent(domain.get('id'))
watchlist_matches = self._compare(watchlist, domains)
for match in watchlist_matches:
m = dict()
m['parent'] = domain.get('value')
m['value'] = match
m['match'] = domain.get('value')
m['source'] = 'watchlist'
matches.append(m)
return matches
def exists(self, alerts):
if not isinstance(alerts, list):
alerts = [alerts]
hits = list()
db = DatabaseUtils()
for alert in alerts:
if isinstance(alert, dict):
alert = alert.get('value')
keys = [
{'value': alert},
{'status': 'closed'},
]
hits.extend(db.exists(keys, index='alerts_index'))
return self._sorted(hits)
def check_ip(self):
db = DatabaseUtils()
domains_to_watch = db.get_domains()
matches = list()
for domain in domains_to_watch:
ip_address = self._get_ip_address(domain.get('value'))
if ip_address and not domain.get('ip_address') == ip_address:
m = dict()
m['parent'] = domain.get('value')
m['values'] = [ip_address]
m['match'] = domain.get('ip_address')
m['source'] = 'ip_address'
matches.append(m)
return matches
def check_keywords(self):
db = DatabaseUtils()
domains_to_watch = db.get_domains()
matches = list()
for domain in domains_to_watch:
keywords = domain.get('keywords')
for keyword in keywords:
keyword_matches = db.get_registered_by_keyword(keyword)
for match in keyword_matches:
m = dict()
m['parent'] = domain.get('value')
m['value'] = match
m['match'] = keyword
m['source'] = 'keyword'
matches.append(m)
return matches
def alert(self, domains=None, start=False, end=False):
"""
check recently collected data for matches with watchlist
:param domains: list of domains to check for alerts
:param start: Start date for creating alerts (default: last 24h). Format: YYYY-MM-DD
:param end: End date for creating alerts (default: now). Format: YYYY-MM-DD
:return:
"""
db = DatabaseUtils()
if not domains:
if not start:
start = (datetime.datetime.now() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
if not end:
end = datetime.datetime.now().strftime('%Y-%m-%d')
domains = db.get_registered_by_date(start, end)
logger.info("Start Alerting ({} domains)".format(len(domains)))
alerts = Alerts()
matches = list()
# create alerts on watchlist
matches.extend(alerts.check_watchlist(domains))
# create alerts on keywords
matches.extend(alerts.check_keywords())
# create alerts on heuristics
matches.extend(alerts.check_heuristics(domains))
# create alerts on certificates
matches.extend(alerts.check_certificates(start=start, end=end))
alert_ids = alerts.create(matches)
logger.info("Alerts created: {}".format(len(alert_ids)))
def generate(self, domain, store=True):
"""
this method generates typo domains for watchlist
:param domain: new domain to watch out for
:param store: store generated typo domains in watchlisth
:return:
"""
logger.info("Start generating watchlist for domain: {}".format(
domain.get('value')))
g = Generator()
g.generate(domain.get('value'))
logger.info("Completed generation: {} new watchlist domains".format(
len(g.typo_domains)))
if store:
db = DatabaseUtils()
# check if domain already exits
if db.get_domain(domain.get('value')):
logger.error("Domain {} already exits..skipping.".format(
domain.get('value')))
return g.typo_domains
# make sure ip is set, if not get it
if not domain.get('ip_address', False):
try:
domain['ip_address'] = socket.gethostbyname(
domain['value'])
except Exception as e:
logger.info("Cannot retrieve IP for {}".format(
domain['value']))
# add new domain to monitor for
domain_id = db.add_new_domain(domain)
# add typo domains for this domain
db.add_watchlist_domains(domain_id[0], g.typo_domains)
return g.typo_domains
def enrich(self,
domains=None,
start=False,
end=False,
limit=0,
missing_only=False):
db = DatabaseUtils()
if not domains:
if not start:
start = (datetime.datetime.now() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
if not end:
end = datetime.datetime.now().strftime('%Y-%m-%d')
domains = db.get_registered_by_date(start, end)
if missing_only:
domains = db.get_missing_enrichments(
[x.get('value') for x in domains], start, end)
logger.info("Start Enrichment ({} domains)".format(len(domains)))
e = Enrichment(domains, limit=limit)
e.enrich()
db.add_enrichments(e.enrichments)
logger.info("Completed enrichment.")
def _certificats(self, start=False, end=False):
db = DatabaseUtils()
certificates_hits = db.get_certificates(start=start, end=end)
return certificates_hits
def _close_alert(self, alert):
db = DatabaseUtils()
db.update_alert_status(alert)
def get_alert(self, value):
db = DatabaseUtils()
alerts = db.get_alerts(value=value)
return alerts