def _process_new_sites_chunked(domains_ranked: dict) -> None:
for domain in domains_ranked.keys():
site = models.Site(domain=str(domain), current_rank = domains_ranked[domain])
db_session.add(site)
logger.debug("Adding site: {}".format(domain))
db_session.commit()
logger.info("New site chunk updated")
def process_result(result: dict):
logger.debug("Processing site: %", result['site_id'])
processed = dict()
site = db_session.query(models.Site).filter_by(id=result['site_id']).one()
processed.update(dnutils.get_dmarc_stats(result['dmarc']))
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string=processed['dmarc_policy']).scalar()
if dmarc_policy_db is None:
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string='invalid').scalar()
sub_dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string=processed['dmarc_sub_policy']).scalar()
if sub_dmarc_policy_db is None:
sub_dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string='invalid').scalar()
processed.update(dnutils.caa_stats(result['caa']))
processed.update(spfutils.get_spf_stats(result['txt']))
spf_db = db_session.query(models.SpfPolicy).filter_by(qualifier=processed['spf_policy']).scalar()
processed['email_provider_id'] = mxutils.get_provider_from_mx_records(result['mx'], site.domain)
processed['dns_provider_id'] = dnutils.get_provider_from_ns_records(result['ns'], site.domain)
processed.update(parse_ds(result['ds']))
processed['dnssec_dnskey_algorithm'] = parse_dnskey(result['dnskey'])
sr = models.SiteRun(site_id=result['site_id'], run_id=result['run_id'], run_rank=result['rank'], caa_record=result['caa'], has_caa=processed['caa_exists'],
has_caa_reporting=processed['caa_has_reporting'], caa_issue_count=processed['caa_issue_count'], caa_wildcard_count=processed['caa_wildcard_count'],
has_dmarc=processed['dmarc_exists'], dmarc_policy_id=dmarc_policy_db.id,
dmarc_sub_policy_id=sub_dmarc_policy_db.id, has_dmarc_aggregate_reporting=processed['dmarc_has_aggregate'],
has_dmarc_forensic_reporting=processed['dmarc_has_forensic'], dmarc_record=result['dmarc'], has_spf=processed['spf_exists'],
spf_policy_id=spf_db.id, txt_records=result['txt'], ds_records=result['ds'], mx_records=result['mx'],
ns_records=result['ns'], email_provider_id=processed['email_provider_id'], dns_provider_id=processed['dns_provider_id'],
dnssec_ds_algorithm=processed['ds_algorithm'], dnssec_digest_type=processed['ds_digest_type'],
dnssec_dnskey_algorithm=processed['dnssec_dnskey_algorithm'], has_securitytxt=result['has_dnssec'], has_msdc=result['is_msdcs'],
j_caa_records=result['caa'], j_dmarc_record=result['dmarc'], j_txt_records=result['txt'],
j_ns_records=result['ns'], j_mx_records=result['mx'], j_ds_recoreds=result['ds'],
ns_ip_addresses=result['name_server_ips'], ns_server_ns_results=result['ns_server_ns_results'],
j_soa_records=result['soa'], start_time=result['start_time'], end_time=result['end_time'],
j_bimi_records=result['bimi'])
db_session.add(sr)
db_session.commit()
do_grading(sr)
def _grade_errors(errors: list, grade_type: str, site_run_id: int):
remark_type = db_session.query(models.RemarkType).filter_by(name=grade_type).one()
for error in errors:
remark = db_session.query(models.Remark).filter_by(remark_type_id=remark_type.id, enum_value=error.value).one()
remark_siterun = models.SiterunRemark(site_run_id=site_run_id, remark_id=remark.id)
db_session.add(remark_siterun)
db_session.commit()
def _process_new_site(domain: bytes, new_rank: int) -> None:
site = db_session.query(models.Site).filter_by(domain=domain).first()
if site:
site.current_rank = new_rank
else:
site = models.Site(domain=str(domain), current_rank=new_rank)
db_session.add(site)
logger.debug("Adding site: {}".format(domain))
db_session.commit()
def _seed_sites(filename):
with open(filename, 'r') as file:
csv_reader = csv.DictReader(file)
for row in csv_reader:
site = models.Site(current_rank=int(row['rank']),
domain=row['site'])
db_session.add(site)
db_session.commit()
def _seed_remark_types():
remark_types = ['spf', 'dmarc', 'caa', 'ns', 'soa', 'mx', 'bimi']
for remark_type in remark_types:
remark_type_s = db_session.query(
models.RemarkType).filter_by(name=remark_type).scalar()
if not remark_type_s:
remark_type_db = models.RemarkType(name=remark_type)
db_session.add(remark_type_db)
db_session.commit()
def _seed_dmarc_policy():
dmarc_policies = [('none', 'None', '#FFBF7F'),
('quarantine', 'Quarantine', '#72e572'),
('reject', 'Reject', '#8080FF'),
('no_policy', 'No Policy', '#FF8080'),
('invalid', 'Invalid', '#FF00FF')]
for dmarc_policy in dmarc_policies:
dmarc_policy = models.DmarcPolicy(policy_string=dmarc_policy[0],
display_name=dmarc_policy[1],
color=dmarc_policy[2])
db_session.add(dmarc_policy)
db_session.commit()
def do_run():
date = datetime.datetime.now()
if settings.DNSTATS_ENV == 'Development':
run = models.Run(start_time=date, start_rank=1, end_rank=50)
logger.warning("[DO RUN]: Running a Debug top 50 sites runs")
else:
run = models.Run(start_time=date, start_rank=1, end_rank=1000000)
logger.warning("[DO RUN]: Running a normal run of top 1,000,000 sites runs")
db_session.add(run)
db_session.commit()
run = db_session.query(models.Run).filter_by(start_time=date).first()
_send_start_email(date, run.id)
launch_run(run.id)
def _seed_spf():
spf_policies = [('+all', 'Pass', '#FF00FF'),
('?all', 'Neutral', '#FFBF7F'),
('~all', 'Soft-fail', '#72e572'),
('-all', 'Fail', '#8080FF'),
('no_policy', 'No Policy', '#FF8080')]
for spf_pol in spf_policies:
spf_policy = models.SpfPolicy(qualifier=spf_pol[0],
display_name=spf_pol[1],
color=spf_pol[2])
db_session.add(spf_policy)
db_session.commit()
def _seed_remark_arrays(remark_type_db_spf: models.RemarkType,
spf: list) -> None:
for remark in spf:
remark_db = db_session.query(
models.Remark).filter_by(remark_type_id=remark_type_db_spf.id,
enum_value=remark[2]).scalar()
if remark_db:
remark_db.remark_level = remark[0]
remark_db.name = remark[1]
remark_db.enum_value = remark[2]
else:
remark_db = models.Remark(remark_type_id=remark_type_db_spf.id,
name=remark[1],
remark_level=remark[0],
enum_value=remark[2])
db_session.add(remark_db)
db_session.commit()
def _seed_dmarc_policy():
dmarc_policies = [('none', 'None', '#FFBF7F'),
('quarantine', 'Quarantine', '#72e572'),
('reject', 'Reject', '#8080FF'),
('no_policy', 'No Policy', '#FF8080'),
('invalid', 'Invalid', '#FF00FF')]
for dmarc_policy in dmarc_policies:
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(
policy_string=dmarc_policy[0]).scalar()
if not dmarc_policy_db:
dmarc_policy = models.DmarcPolicy(policy_string=dmarc_policy[0],
display_name=dmarc_policy[1],
color=dmarc_policy[2])
db_session.add(dmarc_policy)
else:
dmarc_policy_db.display_name = dmarc_policy[1]
dmarc_policy_db.color = dmarc_policy[2]
db_session.commit()
def _seed_spf():
spf_policies = [('+all', 'Pass', '#FF00FF'),
('?all', 'Neutral', '#FFBF7F'),
('~all', 'Soft-fail', '#72e572'),
('-all', 'Fail', '#8080FF'),
('no_policy', 'No Policy', '#FF8080')]
for spf_pol in spf_policies:
spf_policy_db = db_session.query(
models.SpfPolicy).filter_by(qualifier=spf_pol[0]).scalar()
if not spf_policy_db:
spf_policy = models.SpfPolicy(qualifier=spf_pol[0],
display_name=spf_pol[1],
color=spf_pol[2])
db_session.add(spf_policy)
else:
spf_policy_db.display_name = spf_pol[1]
spf_policy_db.color = spf_pol[2]
db_session.commit()
def _seed_email_providers():
email_providers = [
("Google Apps", "l.google.com.", True),
("Office 365", "protection.outlook.", True),
("ProofPoint", "pphosted.com.", True),
("Minecast", "mimecast.com.", True),
("MailRoute", "mailroute.net.", True), ("Zoho", "zoho.com.", True),
("Barracuda Networks", "barracudanetworks.com.", True),
("FastMail", "messagingengine.com.", True),
("Cisco Cloud Email Security", "iphmx.com.", True),
("Self-Hosted", "domain.", False),
("Symantec Messaging Security", "messagelabs.com.", True),
("FireEyeCloud", "fireeyecloud.com.", True),
("ProofPoint Essentials", "ppe-hosted.com.", True),
("Amazon Web Services", "amazonaws.com.", True),
("DreamHost", "dreamhost.com.", True),
("Office 365", "eo.outlook.com.", True),
("OSU OpenSource Lab", "osuosl.org.", True),
("Gandi", "gandi.net.", True), ("Rackspace", "emailsrvr.com.", True),
("TrendMicro Hosted Email Security", "in.hes.trendmicro.com.", True),
("Self-Hosted", "amazon-smtp.amazon.com.", True),
("TrendMicro Hosted Email Security", "in.hes.trendmicro.eu.", True),
("Self-Hosted", "wikimedia.org.", True),
("GoDaddy", "secureserver.net.", True), ("NoMail", '{"0."}', True),
("QQ", "qq.com.", True), ("No mail", "nxdomain.", False),
('Unknown', 'Unknown.', False),
("Namecheap", ".web-hosting.com.", True),
("Google Apps", ".googlemail.com.", True)
]
for email_provider in email_providers:
email_provider_s = db_session.query(models.EmailProvider).filter_by(
search_regex=email_provider[1]).scalar()
if not email_provider_s:
email_provider = models.EmailProvider(
display_name=email_provider[0],
search_regex=email_provider[1],
is_regex=email_provider[2])
db_session.add(email_provider)
db_session.commit()
def _seed_ns_providers():
ns_providers = [('DNSimple', 'dnsimple.com.', True),
('Hurricane Electric', 'he.net.', True),
('OVH', 'ovh.net.', True),
('CloudFlare', 'ns.cloudflare.com.', True),
('Amazon Web Services', '.awsdns-', True),
('DigitalOcean', 'digitalocean.com.', True),
('Inmotion Hosting', 'inmotionhosting.com.', True),
('GoDaddy', 'domaincontrol.com.', True),
('Hostgator', 'hostgator.com.', True),
('Wordpress', 'wordpress.com.', True),
('Linode', 'linode.com.', True),
('NameCheap', 'registrar-servers.com.', True),
('FastMail', 'messagingengine.com.', True),
('DNS Made Easy', 'dnsmadeeasy.com.', True),
('Gandi', 'gandi.net.', True),
('UltraDNS', 'ultradns.com.', True),
('Azure', '.azure-dns.com.', True),
('Alfa Hosting', '.alfahosting.info.', True),
('Google DNS', '.googledomains.com.', True),
('Mark Monitor', 'markmonitor.com.', True),
('Comcast Business', '.comcastbusiness.net.', True),
('DreamHost', '.dreamhost.com.', True),
('Akamai', '.akam.net.', True),
('Liquid Web', '.sourcedns.com.', True),
('Media Temple', 'mediatemple.net.', True),
('XSERVER', '.xserver.jp.', True),
('Internet Invest', '.srv53.net.', True),
('Flex Web Hosting', '.flexwebhosting.nl.', True),
('HostGator', '.hostgator.com.', True),
('NameCheap', '.namecheaphosting.com.', True),
('Self-hosted', 'Self-hosted', False),
('Unknown', 'Unknown.', False),
('Self-hosted', '.google.com', True),
('Self-hosted', 'twtrdns.net.', True),
('DynDNS', 'dynect.net', True),
('Self-hosted', '.msft.net.', True),
('Self-hosted', '.taobao.com.', True),
('Self-hosted', '.wikimedia.org.', True),
('360Safe', '.360safe.com.', True),
('Self-hosted', '.sina.com.', True),
('CDNS.CN', '.cdns.cn.', True),
('Self-hosted', '.vkontakte.ru.', True),
('Alibaba DNS', 'alibabadns.com.', True),
('Self-hosted', '.dig.com.', True),
('Self-hosted', '.automattic.com.', True),
('SURFnet', '.surfnet.nl.', True),
('No-IP (Vitalwerks LLC)', '.no-ip.com.', True),
('NS1.', '.nsone.net.', True),
('EasyDNS', '.easydns.com.', True),
('Self-hosted', '.apple.com.', True),
('Self-hosted', '.bbc.co.uk.', True),
('AliDNS', '.alidns.com.', True),
('Self-hosted', '.whatsapp.net.', True),
('Self-hosted', '.facebook.com.', True),
('Move', '.move.com.', True),
('MasterWeb', '.masterweb.net.', True),
('JD.com (Jingdong)', '.jd.com.', True),
('JD.com (Jingdong)', '.jdcache.com.', True),
('Internet Systems Consortium', '.isc.org.', True),
('Duodecad ITS', '.dditservices.com.', True),
('Self-hosted', 'bkngs.com.', True),
('Self-hosted', '.thomsonreuters.net.', True),
('Self-hosted', '.bng-ns.com.', True),
('HiChina', '.hichina.com.', True),
('DNSPod', '.dnspod.net.', True),
('DNS.com', '.dns.com.', True),
('Network Solutions', '.worldnic.com.', True),
('Fast24', '.fastdns24.com.', True),
('Fast24', '.fastdns24.eu.', True),
('CSC', '.cscdns.net', True),
('Domain.com', '.domain.com.', True),
('Wix', 'wixdns.net.', True),
('Cafe24', '.cafe24.com.', True),
('LightEdge', '.lightedge.com.', True),
('BlueHost', '.bluehost.com.', True),
('dinahosting', '.dinahosting.com.', True),
('MyHostAdmin', '.myhostadmin.net.', True),
('eNom', 'name-services.com.', True),
('RU-center', '.nic.ru.', True),
('ClouDNS', '.cloudns.net.', True),
('Name', '.name.com.', True),
('XinNet', '.xincache.com.', True)]
for ns_provider in ns_providers:
nsp_s = db_session.query(models.DnsProvider).filter_by(
search_regex=ns_provider[1]).scalar()
if not nsp_s:
nsp = models.DnsProvider(display_name=ns_provider[0],
search_regex=ns_provider[1],
is_regex=ns_provider[2])
db_session.add(nsp)
db_session.commit()