def site_stat(site_id: int, run_id: int):
logger.debug('start site stat site {} run id {}'.format(site_id, run_id))
result = dict()
site = db_session.query(models.Site).filter(models.Site.id == site_id).scalar()
logger.debug('got site for {}'.format(site.domain))
result['mx'] = dnutils.safe_query(site.domain, 'mx')
logger.debug('got mx for {}'.format(site.domain))
result['txt'] = dnutils.safe_query(site.domain, 'txt')
logger.debug('got txtf or {}'.format(site.domain))
result['caa'] = dnutils.safe_query(site.domain, 'caa')
logger.debug('got caa for {}'.format(site.domain))
result['ds'] = dnutils.safe_query(site.domain, 'ds')
result['dnskey'] = dnutils.safe_query(site.domain, 'dnskey')
logger.debug('got dnskey for {}'.format(site.domain))
result['ns'] = dnutils.safe_query(site.domain, 'ns')
logger.debug('got ns')
result['dmarc'] = dnutils.safe_query('_dmarc.' + site.domain, 'txt')
logger.debug('got dmarc for {}'.format(site.domain))
result['has_dnssec'] = has_security_txt(site.domain, logger)
logger.debug('got security.txt for {}'.format(site.domain))
result['is_msdcs'] = dnstats.dnsutils.is_a_msft_dc(site.domain)
logger.debug('got has msdc for {}'.format(site.domain))
result['site_id'] = site.id
logger.debug('set site id {}'.format(site.domain))
result['rank'] = site.current_rank
logger.debug('set rank {}'.format(site.domain))
result['run_id'] = run_id
logger.debug('set run id {} -- done'.format(site.domain))
result['name_server_ips'] = get_name_server_ips(result['ns'])
logger.debug('got the IP addresses for all the name servers')
result['ns_server_ns_results'] = get_name_server_results(result['name_server_ips'], site.domain)
logger.debug('got name server results from each name server')
result['soa'] = dnutils.safe_query(site.domain, 'soa')
logger.debug('got soa for {}'.format(site.domain))
return result
def process_result(result: dict):
logger.debug("Processing site: {}".format(result['site_id']))
processed = dict()
site = db_session.query(models.Site).filter_by(id=result['site_id']).one()
processed.update(dnutils.get_dmarc_stats(result['dmarc']))
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string=processed['dmarc_policy']).scalar()
if dmarc_policy_db is None:
dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string='invalid').scalar()
sub_dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string=processed['dmarc_sub_policy']).scalar()
if sub_dmarc_policy_db is None:
sub_dmarc_policy_db = db_session.query(models.DmarcPolicy).filter_by(policy_string='invalid').scalar()
processed.update(dnutils.caa_stats(result['caa']))
processed.update(spfutils.get_spf_stats(result['txt']))
spf_db = db_session.query(models.SpfPolicy).filter_by(qualifier=processed['spf_policy']).scalar()
processed['email_provider_id'] = mxutils.get_provider_from_mx_records(result['mx'], site.domain)
processed['dns_provider_id'] = dnutils.get_provider_from_ns_records(result['ns'], site.domain)
processed.update(parse_ds(result['ds']))
processed['dnssec_dnskey_algorithm'] = parse_dnskey(result['dnskey'])
sr = models.SiteRun(site_id=result['site_id'], run_id=result['run_id'], run_rank=result['rank'], caa_record=result['caa'], has_caa=processed['caa_exists'],
has_caa_reporting=processed['caa_has_reporting'], caa_issue_count=processed['caa_issue_count'], caa_wildcard_count=processed['caa_wildcard_count'],
has_dmarc=processed['dmarc_exists'], dmarc_policy_id=dmarc_policy_db.id,
dmarc_sub_policy_id=sub_dmarc_policy_db.id, has_dmarc_aggregate_reporting=processed['dmarc_has_aggregate'],
has_dmarc_forensic_reporting=processed['dmarc_has_forensic'], dmarc_record=result['dmarc'], has_spf=processed['spf_exists'],
spf_policy_id=spf_db.id, txt_records=result['txt'], ds_records=result['ds'], mx_records=result['mx'],
ns_records=result['ns'], email_provider_id=processed['email_provider_id'], dns_provider_id=processed['dns_provider_id'],
dnssec_ds_algorithm=processed['ds_algorithm'], dnssec_digest_type=processed['ds_digest_type'],
dnssec_dnskey_algorithm=processed['dnssec_dnskey_algorithm'], has_securitytxt=result['has_dnssec'], has_msdc=result['is_msdcs'],
j_caa_records=result['caa'], j_dmarc_record=result['dmarc'], j_txt_records=result['txt'],
j_ns_records=result['ns'], j_mx_records=result['mx'], j_ds_recoreds=result['ds'],
ns_ip_addresses=result['name_server_ips'], ns_server_ns_results=result['ns_server_ns_results'],
j_soa_records=result['soa'])
db_session.add(sr)
db_session.commit()
grade_spf.s(sr.id).apply_async()
grade_dmarc.s(sr.id).apply_async()
grade_caa.s(sr.id).apply_async()
grade_ns.s(sr.id).apply_async()
grade_soa.s(sr.id).apply_async()
return
def _seed_ns_providers():
ns_providers = [('DNSimple', 'dnsimple.com.', True),
('Hurricane Electric', 'he.net.', True),
('OVH', 'ovh.net.', True),
('CloudFlare', 'ns.cloudflare.com.', True),
('Amazon Web Services', '.awsdns-', True),
('DigitalOcean', 'digitalocean.com.', True),
('Inmotion Hosting', 'inmotionhosting.com.', True),
('GoDaddy', 'domaincontrol.com.', True),
('Hostgator', 'hostgator.com.', True),
('Wordpress', 'wordpress.com.', True),
('Linode', 'linode.com.', True),
('NameCheap', 'registrar-servers.com.', True),
('FastMail', 'messagingengine.com.', True),
('DNS Made Easy', 'dnsmadeeasy.com.', True),
('Gandi', 'gandi.net.', True),
('UltraDNS', 'ultradns.com.', True),
('Azure', '.azure-dns.com.', True),
('Alfa Hosting', '.alfahosting.info.', True),
('Google DNS', '.googledomains.com.', True),
('Mark Monitor', 'markmonitor.com.', True),
('Comcast Business', '.comcastbusiness.net.', True),
('DreamHost', '.dreamhost.com.', True),
('Akamai', '.akam.net.', True),
('Liquid Web', '.sourcedns.com.', True),
('Media Temple', 'mediatemple.net.', True),
('XSERVER', '.xserver.jp.', True),
('Internet Invest', '.srv53.net.', True),
('Flex Web Hosting', '.flexwebhosting.nl.', True),
('HostGator', '.hostgator.com.', True),
('NameCheap', '.namecheaphosting.com.', True),
('Self-hosted', 'Self-hosted', False),
('Unknown', 'Unknown.', False),
('Self-hosted', '.google.com', True),
('Self-hosted', 'twtrdns.net.', True),
('DynDNS', 'dynect.net', True),
('Self-hosted', '.msft.net.', True),
('Self-hosted', '.taobao.com.', True),
('Self-hosted', '.wikimedia.org.', True),
('360Safe', '.360safe.com.', True),
('Self-hosted', '.sina.com.', True),
('CDNS.CN', '.cdns.cn.', True),
('Self-hosted', '.vkontakte.ru.', True),
('Alibaba DNS', 'alibabadns.com.', True),
('Self-hosted', '.dig.com.', True),
('Self-hosted', '.automattic.com.', True),
('SURFnet', '.surfnet.nl.', True),
('No-IP (Vitalwerks LLC)', '.no-ip.com.', True),
('NS1.', '.nsone.net.', True),
('EasyDNS', '.easydns.com.', True),
('Self-hosted', '.apple.com.', True),
('Self-hosted', '.bbc.co.uk.', True),
('AliDNS', '.alidns.com.', True),
('Self-hosted', '.whatsapp.net.', True),
('Self-hosted', '.facebook.com.', True),
('Move', '.move.com.', True),
('MasterWeb', '.masterweb.net.', True),
('JD.com (Jingdong)', '.jd.com.', True),
('JD.com (Jingdong)', '.jdcache.com.', True),
('Internet Systems Consortium', '.isc.org.', True),
('Duodecad ITS', '.dditservices.com.', True),
('Self-hosted', 'bkngs.com.', True),
('Self-hosted', '.thomsonreuters.net.', True),
('Self-hosted', '.bng-ns.com.', True),
('HiChina', '.hichina.com.', True),
('DNSPod', '.dnspod.net.', True),
('DNS.com', '.dns.com.', True),
('Network Solutions', '.worldnic.com.', True),
('Fast24', '.fastdns24.com.', True),
('Fast24', '.fastdns24.eu.', True),
('CSC', '.cscdns.net', True),
('Domain.com', '.domain.com.', True),
('Wix', 'wixdns.net.', True),
('Cafe24', '.cafe24.com.', True),
('LightEdge', '.lightedge.com.', True),
('BlueHost', '.bluehost.com.', True),
('dinahosting', '.dinahosting.com.', True),
('MyHostAdmin', '.myhostadmin.net.', True),
('eNom', 'name-services.com.', True),
('RU-center', '.nic.ru.', True),
('ClouDNS', '.cloudns.net.', True),
('Name', '.name.com.', True),
('XinNet', '.xincache.com.', True)]
for ns_provider in ns_providers:
nsp_s = db_session.query(models.DnsProvider).filter_by(
search_regex=ns_provider[1]).scalar()
if not nsp_s:
nsp = models.DnsProvider(display_name=ns_provider[0],
search_regex=ns_provider[1],
is_regex=ns_provider[2])
db_session.add(nsp)
db_session.commit()
def _seed_remarks():
"""
This method seed the remarks
Remark Levels:
0 - Fatal
1 - Error, assuming default
2 - Warning, value should be used
3 - Deprecation Warning - Value was once valid, no longer
4 - Info - No action need. Just additional data
"""
dmarc = [
(0, 'Invalid DMARC Record', 0),
(1, 'Invalid DKIM alignment mode (adkim) value', 1),
(1, 'Invalid SPF alignment mode ASPF, value', 2),
(1, 'Invalid Failure Reporting Value', 3),
(0, 'Invalid Policy', 4),
(1, 'Invalid Subdomain Policy', 5),
(0, 'Multiple Dmarc Records', 6),
(1, 'Invalid Failure reporting (rf) Value', 7),
(1, 'Invalid Aggregate Reporting interval (ri) Value', 8),
(1, 'Invalid Percent Value', 9),
(0, 'Invalid DMARC Record Start', 10),
(0, 'No DMARC Record', 11),
]
spf = [(0, 'None', 0), (0, 'Invalid Record Start', 1), (3, 'Has Ptr', 2),
(2, 'Too Many DNS Lookups', 3), (1, 'Default All Qualifier', 4),
(1, 'Invalid Include Format', 5),
(0, 'Include Returned Many Spf', 6),
(2, 'Too Many A Records Returned', 7),
(1, 'Invalid A Mechanism', 8), (1, 'Invalid MX Mechanism', 9),
(2, 'Too Many MX Records Returned', 10),
(1, 'Invalid Redirect Mechanism', 11),
(0, 'No Record At Redirect', 12),
(0, 'Redirect Returned Many Spf', 13),
(1, 'Invalid IPv4 Mechanism', 14),
(1, 'Invalid IPv6 Mechanism', 15), (1, 'Invalid Mechanism', 16),
(0, 'Multiple Spf Records', 17), (0, 'No Spf Found', 18),
(1, 'Invalid IPv4 Cidr', 19), (1, 'Invalid IPv6 Cidr', 20),
(1, 'Too Many Endings', 21), (0, 'Too Many Starts', 22),
(4, 'No MX Records', 23),
(2, 'No A Records Returned In Mechanism', 24),
(2, 'No MX Records Returned In Mechanism', 25),
(1, 'Invalid IPv4 Value', 26), (1, 'Invalid IPv6 Value', 27)]
caa = [(0, 'Invalid Property Structure', 0), (0, 'No Caa Records', 1),
(1, 'Invalid Flag', 2), (1, 'Invalid Tag', 3),
(1, 'Invalid Value', 4), (0, 'Value Quote Error', 5),
(0, 'Value Not Quoted', 6), (1, 'Iodef No Scheme', 7),
(1, 'Iodef Invalid Email', 8), (1, 'Iodef Invalid Url', 9),
(1, 'Issuewild Domain Invalid', 10),
(1, 'Issue Domain Invalid', 11), (1, 'Tag Too Long', 12)]
ns = [
(0, 'No NS Records', 0),
(2, 'Only One Name Server', 1),
(0, 'Null NS Record', 2),
(1, 'Name Server Has no A Record', 3),
(0, 'Name Server Has Invalid Response', 4),
(2, 'Name Server Is Not Public', 5),
(0, 'No Name Servers Returned', 6),
(2, 'Name Server Mismatch', 7),
]
soa = [
(0, 'No SOA', 0),
(0, 'Too Many SOA', 1),
(0, 'SOA Invalid', 2),
(1, 'Invalid MNAME', 3),
(1, 'Invalid RNAME', 4),
(1, 'Invalid Serial', 5),
(1, 'Invalid Refresh', 6),
(1, 'Invalid Retry', 7),
(1, 'Invalid Expire', 8),
(1, 'Invalid Minimum', 9),
(1, 'Serial Not In Range', 10),
(1, 'Refresh Not in Range', 11),
(1, 'Retry Not In Range', 12),
(1, 'Minimum Not In Range', 13),
(1, 'Expire Not In Range', 14),
]
mx = [(0, 'NO MX RECORDS', 0), (0, 'BLANK MX RECORD', 1),
(0, 'TOO MANY PARTS', 2), (0, 'TOO FEW PARTS', 3),
(0, 'PREFERENCE OUT OF RANGE', 4), (0, 'INVALID PREFERENCE', 5),
(0, 'INVALID EXCHANGE', 6), (2, 'EXCHANGE IS AN IP', 7),
(2, 'NOT PUBLIC DOMAIN', 8), (2, 'POSSIBLE BAD EXCHANGE', 9)]
bimi = [
(0, 'N0 BIMI RECORDS', 0),
(0, 'TOO MANY BIMI RECORDS', 1),
(0, 'DMARC STRICT ENOUGH POLICY', 2),
(0, 'DMARC STRICT ENOUGH PERCENT', 3),
(0, 'INVALID START', 4),
(0, 'LOGO NOT DEFINED', 5),
(0, 'LOGO NOT HTTPS', 6),
(0, 'LOGO INVALID LOCATION', 7),
(0, 'LOGO INVALID FORMAT', 8),
(0, 'SELECTOR NOT DEFINED', 9),
(0, 'DUPLICATE TAG FOUND', 10),
(0, 'BIMI OPTED OUT', 11),
(0, 'LOGO LOCATION BLANK', 12),
(0, 'DMARC NOT DEFINED', 13),
]
remark_type_db_dmarc = db_session.query(
models.RemarkType).filter_by(name='dmarc').one()
_seed_remark_arrays(remark_type_db_dmarc, dmarc)
remark_type_db_spf = db_session.query(
models.RemarkType).filter_by(name='spf').one()
_seed_remark_arrays(remark_type_db_spf, spf)
remark_type_db_caa = db_session.query(
models.RemarkType).filter_by(name='caa').one()
_seed_remark_arrays(remark_type_db_caa, caa)
remark_type_db_ns = db_session.query(
models.RemarkType).filter_by(name='ns').one()
_seed_remark_arrays(remark_type_db_ns, ns)
remark_type_db_soa = db_session.query(
models.RemarkType).filter_by(name='soa').one()
_seed_remark_arrays(remark_type_db_soa, soa)
remark_type_db_mx = db_session.query(
models.RemarkType).filter_by(name='mx').one()
_seed_remark_arrays(remark_type_db_mx, mx)
remark_type_db_mx = db_session.query(
models.RemarkType).filter_by(name='bimi').one()
_seed_remark_arrays(remark_type_db_mx, bimi)
def do_charts_latest():
the_time = db_session.query(func.Max(models.Run.start_time)).scalar()
run = db_session.query(models.Run).filter_by(start_time=the_time).scalar()
do_charts.s(run.id).apply_async()
def _unrank_domain(domain: str):
site = db_session.query(models.Site).filter_by(domain=domain).first()
if site:
site.current_rank = 0
db_session.commit()
logger.debug("Unranking site: {}".format(domain))
def _seed_remarks():
"""
This method seed the remarks
Remark Levels:
0 - Fatal
1 - Error, assuming default
2 - Warning, value should be used
3 - Deprecation Warning - Value was once valid, no longer
4 - Info - No action need. Just additional data
"""
dmarc = [(0, 'Invalid DMARC Record', 0),
(1, 'Invalid DKIM alignment mode (adkim) value', 1),
(1, 'Invalid SPF alignment mode ASPF, value', 2),
(1, 'Invalid Failure Reporting Value', 3),
(0, 'Invalid Policy', 4), (1, 'Invalid Subdomain Policy', 5),
(0, 'Multiple Dmarc Records', 6),
(1, 'Invalid Failure reporting (rf) Value', 7),
(1, 'Invalid Aggregate Reporting interval (ri) Value', 8),
(1, 'Invalid Percent Value', 9),
(0, 'Invalid DMARC Record Start', 10)]
spf = [(0, 'None', 0), (0, 'Invalid Record Start', 1), (3, 'Has Ptr', 2),
(2, 'Too Many DNS Lookups', 3), (1, 'Default All Qualifier', 4),
(1, 'Invalid Include Format', 5),
(0, 'Include Returned Many Spf', 6),
(2, 'Too Many A Records Returned', 7),
(1, 'Invalid A Mechanism', 8), (1, 'Invalid MX Mechanism', 9),
(2, 'Too Many MX Records Returned', 10),
(1, 'Invalid Redirect Mechanism', 11),
(0, 'No Record At Redirect', 12),
(0, 'Redirect Returned Many Spf', 13),
(1, 'Invalid IPv4 Mechanism', 14),
(1, 'Invalid IPv6 Mechanism', 15), (1, 'Invalid Mechanism', 16),
(0, 'Multiple Spf Records', 17), (0, 'No Spf Found', 18),
(1, 'Invalid IPv4 Cidr', 19), (1, 'Invalid IPv6 Cidr', 20),
(1, 'Too Many Endings', 21), (0, 'Too Many Starts', 22),
(4, 'No MX Records', 23),
(2, 'No A Records Returned In Mechanism', 24),
(2, 'No MX Records Returned In Mechanism', 25),
(1, 'Invalid IPv4 Value', 26), (1, 'Invalid IPv6 Value', 27)]
caa = [(0, 'Invalid Property Structure', 0), (0, 'No Caa Records', 1),
(1, 'Invalid Flag', 2), (1, 'Invalid Tag', 3),
(1, 'Invalid Value', 4), (0, 'Value Quote Error', 5),
(0, 'Value Not Quoted', 6), (1, 'Iodef No Scheme', 7),
(1, 'Iodef Invalid Email', 8), (1, 'Iodef Invalid Url', 9),
(1, 'Issuewild Domain Invalid', 10),
(1, 'Issue Domain Invalid', 11), (1, 'Tag Too Long', 12)]
ns = [
(0, 'No NS Records', 0),
(2, 'Only One Name Server', 1),
(0, 'Null NS Record', 2),
(1, 'Name Server Has no A Record', 3),
(0, 'Name Server Has Invalid Response', 4),
(2, 'Name Server Is Not Public', 5),
(0, 'No Name Servers Returned', 6),
(2, 'Name Server Mismatch', 7),
]
soa = [
(0, 'No SOA', 0),
(0, 'Too Many SOA', 1),
(0, 'SOA Invalid', 2),
(1, 'Invalid MNAME', 3),
(1, 'Invalid RNAME', 4),
(1, 'Invalid Serial', 5),
(1, 'Invalid Refresh', 6),
(1, 'Invalid Retry', 7),
(1, 'Invalid Expire', 8),
(1, 'Invalid Minimum', 9),
(1, 'Serial Not In Range', 10),
(1, 'Refresh Not in Range', 11),
(1, 'Retry Not In Range', 12),
(1, 'Minimum Not In Range', 13),
(1, 'Expire Not In Range', 14),
]
remark_type_db_dmarc = db_session.query(
models.RemarkType).filter_by(name='dmarc').one()
_seed_remark_arrays(remark_type_db_dmarc, dmarc)
remark_type_db_spf = db_session.query(
models.RemarkType).filter_by(name='spf').one()
_seed_remark_arrays(remark_type_db_spf, spf)
remark_type_db_caa = db_session.query(
models.RemarkType).filter_by(name='spf').one()
_seed_remark_arrays(remark_type_db_caa, caa)
remark_type_db_ns = db_session.query(
models.RemarkType).filter_by(name='ns').one()
_seed_remark_arrays(remark_type_db_ns, ns)
remark_type_db_soa = db_session.query(
models.RemarkType).filter_by(name='soa').one()
_seed_remark_arrays(remark_type_db_soa, soa)
def _get_file_name_from_run_id(run_id: int, report: dict, suffix: str):
run = db_session.query(models.Run).filter_by(id=run_id).one()
return run.start_time.strftime("%Y/%m/%d/{report_name}.{suffix}".format(
report_name=report['name'], suffix=suffix))
def get_site_and_site_run(site_run_id):
site_run = db_session.query(models.SiteRun).filter(models.SiteRun.id == site_run_id).one()
site = db_session.query(models.Site).filter(models.Site.id == site_run.site_id).one()
return site, site_run
