def _grade_errors(errors: list, grade_type: str, site_run_id: int):
remark_type = db_session.query(models.RemarkType).filter_by(name=grade_type).one()
for error in errors:
remark = db_session.query(models.Remark).filter_by(remark_type_id=remark_type.id, enum_value=error.value).one()
remark_siterun = models.SiterunRemark(site_run_id=site_run_id, remark_id=remark.id)
db_session.add(remark_siterun)
db_session.commit()
def grade_spf(site_run_id: int):
site, site_run = get_site_and_site_run(site_run_id)
records = site_run.j_txt_records
grade, errors = grade_spf_record(records, site.domain, site_run.has_mx)
site_run.spf_grade = grade
db_session.commit()
_grade_errors(errors, 'spf', site_run_id)
def process_result(result: dict):
logger.debug("Processing site: %", result['site_id'])
processed = dict()
site = db_session.query(models.Site).filter_by(id=result['site_id']).one()
processed.update(dnutils.get_dmarc_stats(result['dmarc']))
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string=processed['dmarc_policy']).scalar()
if dmarc_policy_db is None:
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string='invalid').scalar()
sub_dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string=processed['dmarc_sub_policy']).scalar()
if sub_dmarc_policy_db is None:
sub_dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string='invalid').scalar()
processed.update(dnutils.caa_stats(result['caa']))
processed.update(spfutils.get_spf_stats(result['txt']))
spf_db = db_session.query(models.SpfPolicy).filter_by(qualifier=processed['spf_policy']).scalar()
processed['email_provider_id'] = mxutils.get_provider_from_mx_records(result['mx'], site.domain)
processed['dns_provider_id'] = dnutils.get_provider_from_ns_records(result['ns'], site.domain)
processed.update(parse_ds(result['ds']))
processed['dnssec_dnskey_algorithm'] = parse_dnskey(result['dnskey'])
sr = models.SiteRun(site_id=result['site_id'], run_id=result['run_id'], run_rank=result['rank'], caa_record=result['caa'], has_caa=processed['caa_exists'],
has_caa_reporting=processed['caa_has_reporting'], caa_issue_count=processed['caa_issue_count'], caa_wildcard_count=processed['caa_wildcard_count'],
has_dmarc=processed['dmarc_exists'], dmarc_policy_id=dmarc_policy_db.id,
dmarc_sub_policy_id=sub_dmarc_policy_db.id, has_dmarc_aggregate_reporting=processed['dmarc_has_aggregate'],
has_dmarc_forensic_reporting=processed['dmarc_has_forensic'], dmarc_record=result['dmarc'], has_spf=processed['spf_exists'],
spf_policy_id=spf_db.id, txt_records=result['txt'], ds_records=result['ds'], mx_records=result['mx'],
ns_records=result['ns'], email_provider_id=processed['email_provider_id'], dns_provider_id=processed['dns_provider_id'],
dnssec_ds_algorithm=processed['ds_algorithm'], dnssec_digest_type=processed['ds_digest_type'],
dnssec_dnskey_algorithm=processed['dnssec_dnskey_algorithm'], has_securitytxt=result['has_dnssec'], has_msdc=result['is_msdcs'],
j_caa_records=result['caa'], j_dmarc_record=result['dmarc'], j_txt_records=result['txt'],
j_ns_records=result['ns'], j_mx_records=result['mx'], j_ds_recoreds=result['ds'],
ns_ip_addresses=result['name_server_ips'], ns_server_ns_results=result['ns_server_ns_results'],
j_soa_records=result['soa'], start_time=result['start_time'], end_time=result['end_time'],
j_bimi_records=result['bimi'])
db_session.add(sr)
db_session.commit()
do_grading(sr)
def _process_new_sites_chunked(domains_ranked: dict) -> None:
for domain in domains_ranked.keys():
site = models.Site(domain=str(domain), current_rank = domains_ranked[domain])
db_session.add(site)
logger.debug("Adding site: {}".format(domain))
db_session.commit()
logger.info("New site chunk updated")
def grade_mx(site_run_id: int) -> None:
site, site_run = get_site_and_site_run(site_run_id)
records = site_run.j_mx_records
grade, errors = grade_mx_records(records, site.domain)
_grade_errors(errors, 'mx', site_run_id)
site_run.mx_grade = grade
db_session.commit()
def grade_spf(site_run_id: int):
site_run = db_session.query(models.SiteRun).filter(models.SiteRun.id == site_run_id).one()
site = db_session.query(models.Site).filter(models.Site.id == site_run.site_id).one()
records = site_run.j_txt_records
grade, errors = grade_spf_record(records, site.domain, site_run.has_mx)
site_run.spf_grade = grade
db_session.commit()
_grade_errors(errors, 'spf', site_run_id)
def _update_site_rank_chunked(domains_ranked: dict) -> None:
for domain in domains_ranked.keys():
site = db_session.query(models.Site).filter_by(domain=domain).first()
if site.current_rank != domains_ranked[domain]:
site.current_rank = domains_ranked[domain]
logger.debug("Updating site rank: {}".format(domain))
db_session.commit()
logger.info("Site rank chunk updated")
def grade_bimi(site_run_id: int) -> None:
site, site_run = get_site_and_site_run(site_run_id)
records = site_run.j_bimi_records
grade, errors = grade_bimi_records(records, site_run.j_dmarc_record, site.domain)
_grade_errors(errors, 'bimi', site_run_id)
site_run.bimi_grade = grade
site_run.has_bimi = grade == 100
db_session.commit()
def _process_new_site(domain: bytes, new_rank: int) -> None:
site = db_session.query(models.Site).filter_by(domain=domain).first()
if site:
site.current_rank = new_rank
else:
site = models.Site(domain=str(domain), current_rank=new_rank)
db_session.add(site)
logger.debug("Adding site: {}".format(domain))
db_session.commit()
def _seed_sites(filename):
with open(filename, 'r') as file:
csv_reader = csv.DictReader(file)
for row in csv_reader:
site = models.Site(current_rank=int(row['rank']),
domain=row['site'])
db_session.add(site)
db_session.commit()
def _seed_remark_types():
remark_types = ['spf', 'dmarc', 'caa', 'ns', 'soa', 'mx', 'bimi']
for remark_type in remark_types:
remark_type_s = db_session.query(
models.RemarkType).filter_by(name=remark_type).scalar()
if not remark_type_s:
remark_type_db = models.RemarkType(name=remark_type)
db_session.add(remark_type_db)
db_session.commit()
def _seed_dmarc_policy():
dmarc_policies = [('none', 'None', '#FFBF7F'),
('quarantine', 'Quarantine', '#72e572'),
('reject', 'Reject', '#8080FF'),
('no_policy', 'No Policy', '#FF8080'),
('invalid', 'Invalid', '#FF00FF')]
for dmarc_policy in dmarc_policies:
dmarc_policy = models.DmarcPolicy(policy_string=dmarc_policy[0],
display_name=dmarc_policy[1],
color=dmarc_policy[2])
db_session.add(dmarc_policy)
db_session.commit()
def grade_soa(site_run_id: int) -> None:
site, site_run = get_site_and_site_run(site_run_id)
records = site_run.j_soa_records
grade = 0
if not records:
logger.debug("SOA Grade: {} - {} - {} - NO CAA".format(site.domain, site_run.ns_grade, 0))
grade = 0
else:
logger.debug("SOA Grade: {} - {} - {}".format(site.domain, site_run.ns_grade, grade))
grade, errors = grade_soa_records(records, site.domain)
_grade_errors(errors, 'soa', site_run_id)
site_run.soa_grade = grade
db_session.commit()
def _seed_spf():
spf_policies = [('+all', 'Pass', '#FF00FF'),
('?all', 'Neutral', '#FFBF7F'),
('~all', 'Soft-fail', '#72e572'),
('-all', 'Fail', '#8080FF'),
('no_policy', 'No Policy', '#FF8080')]
for spf_pol in spf_policies:
spf_policy = models.SpfPolicy(qualifier=spf_pol[0],
display_name=spf_pol[1],
color=spf_pol[2])
db_session.add(spf_policy)
db_session.commit()
def do_run():
date = datetime.datetime.now()
if settings.DNSTATS_ENV == 'Development':
run = models.Run(start_time=date, start_rank=1, end_rank=50)
logger.warning("[DO RUN]: Running a Debug top 50 sites runs")
else:
run = models.Run(start_time=date, start_rank=1, end_rank=1000000)
logger.warning("[DO RUN]: Running a normal run of top 1,000,000 sites runs")
db_session.add(run)
db_session.commit()
run = db_session.query(models.Run).filter_by(start_time=date).first()
_send_start_email(date, run.id)
launch_run(run.id)
def grade_soa(site_run_id: int) -> None:
site_run = db_session.query(models.SiteRun).filter(models.SiteRun.id == site_run_id).one()
site = db_session.query(models.Site).filter(models.Site.id == site_run.site_id).one()
records = site_run.j_soa_records
grade = 0
if not records:
logger.debug("SOA Grade: {} - {} - {} - NO CAA".format(site.domain, site_run.ns_grade, 0))
grade = 0
else:
logger.debug("SOA Grade: {} - {} - {}".format(site.domain, site_run.ns_grade, grade))
grade, errors = grade_soa_records(records, site.domain)
_grade_errors(errors, 'soa', site_run_id)
site_run.soa_grade = grade
db_session.commit()
def grade_ns(site_run_id: int) -> None:
site, site_run = get_site_and_site_run(site_run_id)
records = site_run.j_ns_records
ip_addresses = site_run.ns_ip_addresses
record_results = site_run.ns_server_ns_results
grade = 0
if not records:
logger.debug("NS Grade: {} - {} - {} - NO CAA".format(site.domain, site_run.ns_grade, 0))
grade = 0
else:
logger.debug("NS Grade: {} - {} - {}".format(site.domain, site_run.ns_grade, grade))
grade, errors = grade_ns_records(records, ip_addresses, record_results, site.domain)
_grade_errors(errors, 'ns', site_run_id)
site_run.ns_grade = grade
db_session.commit()
def grade_dmarc(site_run_id: int):
site, site_run = get_site_and_site_run(site_run_id)
records = site_run.j_dmarc_record
grade = 0
dmarcs = list()
if records:
for record in records:
record = record.replace('"', '')
if record.startswith('v=DMARC1;'):
logger.debug('DMARC - {} - {}'.format(site_run_id, record))
dmarcs.append(record)
if dmarcs:
logger.debug('DMARC Count - {} - {}'.format(site_run_id, len(dmarcs)))
grade, errors = grade_dmarc_record(dmarcs, site.domain)
_grade_errors(errors, 'dmarc', site_run_id)
site_run.dmarc_grade = grade
db_session.commit()
def grade_ns(site_run_id: int) -> None:
site_run = db_session.query(models.SiteRun).filter(models.SiteRun.id == site_run_id).one()
site = db_session.query(models.Site).filter(models.Site.id == site_run.site_id).one()
records = site_run.j_ns_records
ip_addresses = site_run.ns_ip_addresses
record_results = site_run.ns_server_ns_results
grade = 0
if not records:
logger.debug("NS Grade: {} - {} - {} - NO CAA".format(site.domain, site_run.ns_grade, 0))
grade = 0
else:
logger.debug("NS Grade: {} - {} - {}".format(site.domain, site_run.ns_grade, grade))
grade, errors = grade_ns_records(records, ip_addresses, record_results, site.domain)
_grade_errors(errors, 'ns', site_run_id)
site_run.ns_grade = grade
db_session.commit()
def _seed_remark_arrays(remark_type_db_spf: models.RemarkType,
spf: list) -> None:
for remark in spf:
remark_db = db_session.query(
models.Remark).filter_by(remark_type_id=remark_type_db_spf.id,
enum_value=remark[2]).scalar()
if remark_db:
remark_db.remark_level = remark[0]
remark_db.name = remark[1]
remark_db.enum_value = remark[2]
else:
remark_db = models.Remark(remark_type_id=remark_type_db_spf.id,
name=remark[1],
remark_level=remark[0],
enum_value=remark[2])
db_session.add(remark_db)
db_session.commit()
def grade_dmarc(site_run_id: int):
site_run = db_session.query(models.SiteRun).filter(models.SiteRun.id == site_run_id).one()
site = db_session.query(models.Site).filter(models.Site.id == site_run.site_id).one()
records = site_run.j_dmarc_record
grade = 0
dmarcs = list()
if records:
for record in records:
record = record.replace('"', '')
if record.startswith('v=DMARC1;'):
logger.debug('DMARC - {} - {}'.format(site_run_id, record))
dmarcs.append(record)
if dmarcs:
logger.debug('DMARC Count - {} - {}'.format(site_run_id, len(dmarcs)))
grade, errors = grade_dmarc_record(dmarcs, site.domain)
_grade_errors(errors, 'dmarc', site_run_id)
site_run.dmarc_grade = grade
db_session.commit()
def _seed_dmarc_policy():
dmarc_policies = [('none', 'None', '#FFBF7F'),
('quarantine', 'Quarantine', '#72e572'),
('reject', 'Reject', '#8080FF'),
('no_policy', 'No Policy', '#FF8080'),
('invalid', 'Invalid', '#FF00FF')]
for dmarc_policy in dmarc_policies:
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(
policy_string=dmarc_policy[0]).scalar()
if not dmarc_policy_db:
dmarc_policy = models.DmarcPolicy(policy_string=dmarc_policy[0],
display_name=dmarc_policy[1],
color=dmarc_policy[2])
db_session.add(dmarc_policy)
else:
dmarc_policy_db.display_name = dmarc_policy[1]
dmarc_policy_db.color = dmarc_policy[2]
db_session.commit()
def _seed_spf():
spf_policies = [('+all', 'Pass', '#FF00FF'),
('?all', 'Neutral', '#FFBF7F'),
('~all', 'Soft-fail', '#72e572'),
('-all', 'Fail', '#8080FF'),
('no_policy', 'No Policy', '#FF8080')]
for spf_pol in spf_policies:
spf_policy_db = db_session.query(
models.SpfPolicy).filter_by(qualifier=spf_pol[0]).scalar()
if not spf_policy_db:
spf_policy = models.SpfPolicy(qualifier=spf_pol[0],
display_name=spf_pol[1],
color=spf_pol[2])
db_session.add(spf_policy)
else:
spf_policy_db.display_name = spf_pol[1]
spf_policy_db.color = spf_pol[2]
db_session.commit()
def _seed_email_providers():
email_providers = [
("Google Apps", "l.google.com.", True),
("Office 365", "protection.outlook.", True),
("ProofPoint", "pphosted.com.", True),
("Minecast", "mimecast.com.", True),
("MailRoute", "mailroute.net.", True), ("Zoho", "zoho.com.", True),
("Barracuda Networks", "barracudanetworks.com.", True),
("FastMail", "messagingengine.com.", True),
("Cisco Cloud Email Security", "iphmx.com.", True),
("Self-Hosted", "domain.", False),
("Symantec Messaging Security", "messagelabs.com.", True),
("FireEyeCloud", "fireeyecloud.com.", True),
("ProofPoint Essentials", "ppe-hosted.com.", True),
("Amazon Web Services", "amazonaws.com.", True),
("DreamHost", "dreamhost.com.", True),
("Office 365", "eo.outlook.com.", True),
("OSU OpenSource Lab", "osuosl.org.", True),
("Gandi", "gandi.net.", True), ("Rackspace", "emailsrvr.com.", True),
("TrendMicro Hosted Email Security", "in.hes.trendmicro.com.", True),
("Self-Hosted", "amazon-smtp.amazon.com.", True),
("TrendMicro Hosted Email Security", "in.hes.trendmicro.eu.", True),
("Self-Hosted", "wikimedia.org.", True),
("GoDaddy", "secureserver.net.", True), ("NoMail", '{"0."}', True),
("QQ", "qq.com.", True), ("No mail", "nxdomain.", False),
('Unknown', 'Unknown.', False),
("Namecheap", ".web-hosting.com.", True),
("Google Apps", ".googlemail.com.", True)
]
for email_provider in email_providers:
email_provider_s = db_session.query(models.EmailProvider).filter_by(
search_regex=email_provider[1]).scalar()
if not email_provider_s:
email_provider = models.EmailProvider(
display_name=email_provider[0],
search_regex=email_provider[1],
is_regex=email_provider[2])
db_session.add(email_provider)
db_session.commit()
def _seed_ns_providers():
ns_providers = [('DNSimple', 'dnsimple.com.', True),
('Hurricane Electric', 'he.net.', True),
('OVH', 'ovh.net.', True),
('CloudFlare', 'ns.cloudflare.com.', True),
('Amazon Web Services', '.awsdns-', True),
('DigitalOcean', 'digitalocean.com.', True),
('Inmotion Hosting', 'inmotionhosting.com.', True),
('GoDaddy', 'domaincontrol.com.', True),
('Hostgator', 'hostgator.com.', True),
('Wordpress', 'wordpress.com.', True),
('Linode', 'linode.com.', True),
('NameCheap', 'registrar-servers.com.', True),
('FastMail', 'messagingengine.com.', True),
('DNS Made Easy', 'dnsmadeeasy.com.', True),
('Gandi', 'gandi.net.', True),
('UltraDNS', 'ultradns.com.', True),
('Azure', '.azure-dns.com.', True),
('Alfa Hosting', '.alfahosting.info.', True),
('Google DNS', '.googledomains.com.', True),
('Mark Monitor', 'markmonitor.com.', True),
('Comcast Business', '.comcastbusiness.net.', True),
('DreamHost', '.dreamhost.com.', True),
('Akamai', '.akam.net.', True),
('Liquid Web', '.sourcedns.com.', True),
('Media Temple', 'mediatemple.net.', True),
('XSERVER', '.xserver.jp.', True),
('Internet Invest', '.srv53.net.', True),
('Flex Web Hosting', '.flexwebhosting.nl.', True),
('HostGator', '.hostgator.com.', True),
('NameCheap', '.namecheaphosting.com.', True),
('Self-hosted', 'Self-hosted', False),
('Unknown', 'Unknown.', False),
('Self-hosted', '.google.com', True),
('Self-hosted', 'twtrdns.net.', True),
('DynDNS', 'dynect.net', True),
('Self-hosted', '.msft.net.', True),
('Self-hosted', '.taobao.com.', True),
('Self-hosted', '.wikimedia.org.', True),
('360Safe', '.360safe.com.', True),
('Self-hosted', '.sina.com.', True),
('CDNS.CN', '.cdns.cn.', True),
('Self-hosted', '.vkontakte.ru.', True),
('Alibaba DNS', 'alibabadns.com.', True),
('Self-hosted', '.dig.com.', True),
('Self-hosted', '.automattic.com.', True),
('SURFnet', '.surfnet.nl.', True),
('No-IP (Vitalwerks LLC)', '.no-ip.com.', True),
('NS1.', '.nsone.net.', True),
('EasyDNS', '.easydns.com.', True),
('Self-hosted', '.apple.com.', True),
('Self-hosted', '.bbc.co.uk.', True),
('AliDNS', '.alidns.com.', True),
('Self-hosted', '.whatsapp.net.', True),
('Self-hosted', '.facebook.com.', True),
('Move', '.move.com.', True),
('MasterWeb', '.masterweb.net.', True),
('JD.com (Jingdong)', '.jd.com.', True),
('JD.com (Jingdong)', '.jdcache.com.', True),
('Internet Systems Consortium', '.isc.org.', True),
('Duodecad ITS', '.dditservices.com.', True),
('Self-hosted', 'bkngs.com.', True),
('Self-hosted', '.thomsonreuters.net.', True),
('Self-hosted', '.bng-ns.com.', True),
('HiChina', '.hichina.com.', True),
('DNSPod', '.dnspod.net.', True),
('DNS.com', '.dns.com.', True),
('Network Solutions', '.worldnic.com.', True),
('Fast24', '.fastdns24.com.', True),
('Fast24', '.fastdns24.eu.', True),
('CSC', '.cscdns.net', True),
('Domain.com', '.domain.com.', True),
('Wix', 'wixdns.net.', True),
('Cafe24', '.cafe24.com.', True),
('LightEdge', '.lightedge.com.', True),
('BlueHost', '.bluehost.com.', True),
('dinahosting', '.dinahosting.com.', True),
('MyHostAdmin', '.myhostadmin.net.', True),
('eNom', 'name-services.com.', True),
('RU-center', '.nic.ru.', True),
('ClouDNS', '.cloudns.net.', True),
('Name', '.name.com.', True),
('XinNet', '.xincache.com.', True)]
for ns_provider in ns_providers:
nsp_s = db_session.query(models.DnsProvider).filter_by(
search_regex=ns_provider[1]).scalar()
if not nsp_s:
nsp = models.DnsProvider(display_name=ns_provider[0],
search_regex=ns_provider[1],
is_regex=ns_provider[2])
db_session.add(nsp)
db_session.commit()
def _unrank_domain(domain: str):
site = db_session.query(models.Site).filter_by(domain=domain).first()
if site:
site.current_rank = 0
db_session.commit()
logger.debug("Unranking site: {}".format(domain))
