def build_recommended_dns(env):
ret = []
domains = get_dns_domains(env)
zonefiles = get_dns_zones(env)
additional_records = list(get_custom_dns_config(env))
from web_update import get_default_www_redirects
www_redirect_domains = get_default_www_redirects(env)
for domain, zonefile in zonefiles:
records = build_zone(domain, domains, additional_records, www_redirect_domains, env)
# remove records that we don't dislay
records = [r for r in records if r[3] is not False]
# put Required at the top, then Recommended, then everythiing else
records.sort(key = lambda r : 0 if r[3].startswith("Required.") else (1 if r[3].startswith("Recommended.") else 2))
# expand qnames
for i in range(len(records)):
if records[i][0] == None:
qname = domain
else:
qname = records[i][0] + "." + domain
records[i] = {
"qname": qname,
"rtype": records[i][1],
"value": records[i][2],
"explanation": records[i][3],
}
# return
ret.append((domain, records))
return ret
def build_recommended_dns(env):
ret = []
domains = get_dns_domains(env)
zonefiles = get_dns_zones(env)
additional_records = list(get_custom_dns_config(env))
from web_update import get_default_www_redirects
www_redirect_domains = get_default_www_redirects(env)
for domain, zonefile in zonefiles:
records = build_zone(domain, domains, additional_records,
www_redirect_domains, env)
# remove records that we don't dislay
records = [r for r in records if r[3] is not False]
# put Required at the top, then Recommended, then everythiing else
records.sort(key=lambda r: 0 if r[3].startswith("Required.") else
(1 if r[3].startswith("Recommended.") else 2))
# expand qnames
for i in range(len(records)):
if records[i][0] == None:
qname = domain
else:
qname = records[i][0] + "." + domain
records[i] = {
"qname": qname,
"rtype": records[i][1],
"value": records[i][2],
"explanation": records[i][3],
}
# return
ret.append((domain, records))
return ret
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env) + get_default_www_redirects(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
# for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = (
(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
for domain in domains_to_check
)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env) + get_default_www_redirects(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
ssl_certificates = get_ssl_certificates(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def do_dns_update(env, force=False):
# What domains (and their zone filenames) should we build?
domains = get_dns_domains(env)
zonefiles = get_dns_zones(env)
# Custom records to add to zones.
additional_records = list(get_custom_dns_config(env))
from web_update import get_default_www_redirects
www_redirect_domains = get_default_www_redirects(env)
# Write zone files.
os.makedirs('/etc/nsd/zones', exist_ok=True)
updated_domains = []
for i, (domain, zonefile) in enumerate(zonefiles):
# Build the records to put in the zone.
records = build_zone(domain, domains, additional_records, www_redirect_domains, env)
# See if the zone has changed, and if so update the serial number
# and write the zone file.
if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env, force):
# Zone was not updated. There were no changes.
continue
# If this is a .justtesting.email domain, then post the update.
try:
justtestingdotemail(domain, records)
except:
# Hmm. Might be a network issue. If we stop now, will we end
# up in an inconsistent state? Let's just continue.
pass
# Mark that we just updated this domain.
updated_domains.append(domain)
# Sign the zone.
#
# Every time we sign the zone we get a new result, which means
# we can't sign a zone without bumping the zone's serial number.
# Thus we only sign a zone if write_nsd_zone returned True
# indicating the zone changed, and thus it got a new serial number.
# write_nsd_zone is smart enough to check if a zone's signature
# is nearing expiration and if so it'll bump the serial number
# and return True so we get a chance to re-sign it.
sign_zone(domain, zonefile, env)
# Now that all zones are signed (some might not have changed and so didn't
# just get signed now, but were before) update the zone filename so nsd.conf
# uses the signed file.
for i in range(len(zonefiles)):
zonefiles[i][1] += ".signed"
# Write the main nsd.conf file.
if write_nsd_conf(zonefiles, additional_records, env):
# Make sure updated_domains contains *something* if we wrote an updated
# nsd.conf so that we know to restart nsd.
if len(updated_domains) == 0:
updated_domains.append("DNS configuration")
# Kick nsd if anything changed.
if len(updated_domains) > 0:
shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
# Write the OpenDKIM configuration tables.
if write_opendkim_tables(domains, env):
# Settings changed. Kick opendkim.
shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
if len(updated_domains) == 0:
# If this is the only thing that changed?
updated_domains.append("OpenDKIM configuration")
if len(updated_domains) == 0:
# if nothing was updated (except maybe OpenDKIM's files), don't show any output
return ""
else:
return "updated DNS: " + ",".join(updated_domains) + "\n"
def do_dns_update(env, force=False):
# What domains (and their zone filenames) should we build?
domains = get_dns_domains(env)
zonefiles = get_dns_zones(env)
# Custom records to add to zones.
additional_records = list(get_custom_dns_config(env))
from web_update import get_default_www_redirects
www_redirect_domains = get_default_www_redirects(env)
# Write zone files.
os.makedirs('/etc/nsd/zones', exist_ok=True)
updated_domains = []
for i, (domain, zonefile) in enumerate(zonefiles):
# Build the records to put in the zone.
records = build_zone(domain, domains, additional_records,
www_redirect_domains, env)
# See if the zone has changed, and if so update the serial number
# and write the zone file.
if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records,
env, force):
# Zone was not updated. There were no changes.
continue
# If this is a .justtesting.email domain, then post the update.
try:
justtestingdotemail(domain, records)
except:
# Hmm. Might be a network issue. If we stop now, will we end
# up in an inconsistent state? Let's just continue.
pass
# Mark that we just updated this domain.
updated_domains.append(domain)
# Sign the zone.
#
# Every time we sign the zone we get a new result, which means
# we can't sign a zone without bumping the zone's serial number.
# Thus we only sign a zone if write_nsd_zone returned True
# indicating the zone changed, and thus it got a new serial number.
# write_nsd_zone is smart enough to check if a zone's signature
# is nearing expiration and if so it'll bump the serial number
# and return True so we get a chance to re-sign it.
sign_zone(domain, zonefile, env)
# Now that all zones are signed (some might not have changed and so didn't
# just get signed now, but were before) update the zone filename so nsd.conf
# uses the signed file.
for i in range(len(zonefiles)):
zonefiles[i][1] += ".signed"
# Write the main nsd.conf file.
if write_nsd_conf(zonefiles, additional_records, env):
# Make sure updated_domains contains *something* if we wrote an updated
# nsd.conf so that we know to restart nsd.
if len(updated_domains) == 0:
updated_domains.append("DNS configuration")
# Kick nsd if anything changed.
if len(updated_domains) > 0:
shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
# Write the OpenDKIM configuration tables.
if write_opendkim_tables(domains, env):
# Settings changed. Kick opendkim.
shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
if len(updated_domains) == 0:
# If this is the only thing that changed?
updated_domains.append("OpenDKIM configuration")
if len(updated_domains) == 0:
# if nothing was updated (except maybe OpenDKIM's files), don't show any output
return ""
else:
return "updated DNS: " + ",".join(updated_domains) + "\n"
total_size += stat.st_size
return total_size
def wait_for_service(port, public, env, timeout):
# Block until a service on a given port (bound privately or publicly)
# is taking connections, with a maximum timeout.
import socket, time
start = time.perf_counter()
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout / 3)
try:
s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port))
return True
except OSError:
if time.perf_counter() > start + timeout:
return False
time.sleep(min(timeout / 4, 1))
if __name__ == "__main__":
from dns_update import get_dns_domains
from web_update import get_web_domains, get_default_www_redirects
env = load_environment()
domains = get_dns_domains(env) | set(
get_web_domains(env) + get_default_www_redirects(env))
domains = sort_domains(domains, env)
for domain in domains:
print(domain)
if stat.st_ino in seen:
continue
seen.add(stat.st_ino)
total_size += stat.st_size
return total_size
def wait_for_service(port, public, env, timeout):
# Block until a service on a given port (bound privately or publicly)
# is taking connections, with a maximum timeout.
import socket, time
start = time.perf_counter()
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout/3)
try:
s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port))
return True
except OSError:
if time.perf_counter() > start+timeout:
return False
time.sleep(min(timeout/4, 1))
if __name__ == "__main__":
from dns_update import get_dns_domains
from web_update import get_web_domains, get_default_www_redirects
env = load_environment()
domains = get_dns_domains(env) | set(get_web_domains(env) + get_default_www_redirects(env))
domains = sort_domains(domains, env)
for domain in domains:
print(domain)
