def get_web_domain_flags(env):
flags = dict()
zones = get_dns_zones(env)
email_domains = get_mail_domains(env)
user_domains = get_mail_domains(env, users_only=True)
external = get_domains_with_a_records(env)
redirects = get_web_domains_with_root_overrides(env)
for d in email_domains:
flags[d] = flags.get(d, 0)
flags[f"mta-sts.{d}"] = flags.get(d, 0)
flags[f"openpgpkey.{d}"] = flags.get(d, 0) | DOMAIN_WKD
for d in user_domains:
flags[f"autoconfig.{d}"] = flags.get(d, 0)
flags[f"autodiscover.{d}"] = flags.get(d, 0)
for d, _ in zones:
flags[f"www.{d}"] = flags.get(d, 0) | DOMAIN_WWW
for d in redirects:
flags[d] = flags.get(d, 0) | DOMAIN_REDIRECT
flags[env["PRIMARY_HOSTNAME"]] |= DOMAIN_PRIMARY
# Last check for websites hosted elsewhere
for d in flags.keys():
if d in external:
flags[d] = DOMAIN_EXTERNAL # -1 = All bits set to 1, assuming twos-complement
return flags
def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
# What domains should we serve HTTP(S) for?
domains = set()
# Serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one.
domains |= get_mail_domains(env)
if include_www_redirects:
# Add 'www.' subdomains that we want to provide default redirects
# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
# the topmost of each domain we serve.
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
# 'autoconfig.' for Mozilla Thunderbird auto setup.
# 'autodiscover.' for Activesync autodiscovery.
domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
if exclude_dns_elsewhere:
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains -= get_domains_with_a_records(env)
# Ensure the PRIMARY_HOSTNAME is in the list so we can serve webmail
# as well as Z-Push for Exchange ActiveSync. This can't be removed
# by a custom A/AAAA record and is never a 'www.' redirect.
domains.add(env['PRIMARY_HOSTNAME'])
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
# What domains should we serve HTTP(S) for?
domains = set()
# Serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one.
domains |= get_mail_domains(env)
if include_www_redirects:
# Add 'www.' subdomains that we want to provide default redirects
# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
# the topmost of each domain we serve.
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
if exclude_dns_elsewhere:
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains -= get_domains_with_a_records(env)
# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
# 'autoconfig.' for Mozilla Thunderbird auto setup.
# 'autodiscover.' for Activesync autodiscovery.
domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
# Ensure the PRIMARY_HOSTNAME is in the list so we can serve webmail
# as well as Z-Push for Exchange ActiveSync. This can't be removed
# by a custom A/AAAA record and is never a 'www.' redirect.
domains.add(env['PRIMARY_HOSTNAME'])
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
ssl_certificates = get_ssl_certificates(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, rtype, value in dns:
if domain not in domains: continue
if rtype == "CNAME" or (rtype in ("A", "AAAA") and value != "local"):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
# Check the domains.
for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
env["out"].add_heading(domain)
if domain == env["PRIMARY_HOSTNAME"]:
check_primary_hostname_dns(domain, env)
if domain in dns_domains:
check_dns_zone(domain, env, dns_zonefiles)
if domain in mail_domains:
check_mail_domain(domain, env)
if domain in web_domains:
check_web_domain(domain, env)
def run_domain_checks(env): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) # Check the domains. for domain in sort_domains(mail_domains | dns_domains | web_domains, env): env["out"].add_heading(domain) if domain == env["PRIMARY_HOSTNAME"]: check_primary_hostname_dns(domain, env, dns_domains, dns_zonefiles) if domain in dns_domains: check_dns_zone(domain, env, dns_zonefiles) if domain in mail_domains: check_mail_domain(domain, env) if domain in web_domains: check_web_domain(domain, env) if domain in dns_domains: check_dns_zone_suggestions(domain, env, dns_zonefiles)
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
# Check the domains.
for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
print(domain)
print("=" * len(domain))
if domain == env["PRIMARY_HOSTNAME"]:
check_primary_hostname_dns(domain, env)
check_alias_exists("administrator@" + domain, env)
if domain in dns_domains:
check_dns_zone(domain, env, dns_zonefiles)
if domain in mail_domains:
check_mail_domain(domain, env)
if domain == env["PRIMARY_HOSTNAME"] or domain in web_domains:
# We need a SSL certificate for PRIMARY_HOSTNAME because that's where the
# user will log in with IMAP or webmail. Any other domain we serve a
# website for also needs a signed certificate.
check_ssl_cert(domain, env)
print()
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide Webfinger and ActiveSync auto-discover of email settings
# (though the latter isn't really working). These will require that
# an SSL cert be installed.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, value in dns.items():
if domain not in domains: continue
if (isinstance(value, str) and (value != "local")) \
or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env) + get_default_www_redirects(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
# for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = (
(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
for domain in domains_to_check
)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def get_dns_domains(env): # Add all domain names in use by email users and mail aliases and ensure # PRIMARY_HOSTNAME is in the list. domains = set() domains |= get_mail_domains(env) domains.add(env['PRIMARY_HOSTNAME']) return domains
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, env, dns_domains, dns_zonefiles, mail_domains,
web_domains) for domain in domains_to_check)
pool = multiprocessing.pool.Pool(processes=10)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
output = BufferedOutput()
for domain in sort_domains(ret, env):
ret[domain].playback(output)
return output
def get_dns_domains(env):
# Add all domain names in use by email users and mail aliases and ensure
# PRIMARY_HOSTNAME is in the list.
domains = set()
domains |= get_mail_domains(env)
domains.add(env['PRIMARY_HOSTNAME'])
return domains
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide Webfinger and ActiveSync auto-discover of email settings
# (though the latter isn't really working). These will require that
# an SSL cert be installed.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, value in dns.items():
if domain not in domains: continue
if (isinstance(value, str) and (value != "local")) \
or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, rtype, value in dns:
if domain not in domains: continue
if rtype == "CNAME" or (rtype in ("A", "AAAA") and value != "local"):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def do_dns_update(env, force=False):
# Write zone files.
os.makedirs('/etc/nsd/zones', exist_ok=True)
zonefiles = []
updated_domains = []
for (domain, zonefile, records) in build_zones(env):
# The final set of files will be signed.
zonefiles.append((domain, zonefile + ".signed"))
# See if the zone has changed, and if so update the serial number
# and write the zone file.
if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records,
env, force):
# Zone was not updated. There were no changes.
continue
# Mark that we just updated this domain.
updated_domains.append(domain)
# Sign the zone.
#
# Every time we sign the zone we get a new result, which means
# we can't sign a zone without bumping the zone's serial number.
# Thus we only sign a zone if write_nsd_zone returned True
# indicating the zone changed, and thus it got a new serial number.
# write_nsd_zone is smart enough to check if a zone's signature
# is nearing expiration and if so it'll bump the serial number
# and return True so we get a chance to re-sign it.
sign_zone(domain, zonefile, env)
# Write the main nsd.conf file.
if write_nsd_conf(zonefiles, list(get_custom_dns_config(env)), env):
# Make sure updated_domains contains *something* if we wrote an updated
# nsd.conf so that we know to restart nsd.
if len(updated_domains) == 0:
updated_domains.append("DNS configuration")
# Kick nsd if anything changed.
if len(updated_domains) > 0:
shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
# Write the OpenDKIM configuration tables for all of the domains.
if write_opendkim_tables(get_mail_domains(env), env):
# Settings changed. Kick opendkim.
shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
if len(updated_domains) == 0:
# If this is the only thing that changed?
updated_domains.append("OpenDKIM configuration")
# Clear bind9's DNS cache so our own DNS resolver is up to date.
# (ignore errors with trap=True)
shell('check_call', ["/usr/sbin/rndc", "flush"], trap=True)
if len(updated_domains) == 0:
# if nothing was updated (except maybe OpenDKIM's files), don't show any output
return ""
else:
return "updated DNS: " + ",".join(updated_domains) + "\n"
def do_dns_update(env, force=False):
# Write zone files.
os.makedirs('/etc/nsd/zones', exist_ok=True)
zonefiles = []
updated_domains = []
for (domain, zonefile, records) in build_zones(env):
# The final set of files will be signed.
zonefiles.append((domain, zonefile + ".signed"))
# See if the zone has changed, and if so update the serial number
# and write the zone file.
if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env, force):
# Zone was not updated. There were no changes.
continue
# Mark that we just updated this domain.
updated_domains.append(domain)
# Sign the zone.
#
# Every time we sign the zone we get a new result, which means
# we can't sign a zone without bumping the zone's serial number.
# Thus we only sign a zone if write_nsd_zone returned True
# indicating the zone changed, and thus it got a new serial number.
# write_nsd_zone is smart enough to check if a zone's signature
# is nearing expiration and if so it'll bump the serial number
# and return True so we get a chance to re-sign it.
sign_zone(domain, zonefile, env)
# Write the main nsd.conf file.
if write_nsd_conf(zonefiles, list(get_custom_dns_config(env)), env):
# Make sure updated_domains contains *something* if we wrote an updated
# nsd.conf so that we know to restart nsd.
if len(updated_domains) == 0:
updated_domains.append("DNS configuration")
# Kick nsd if anything changed.
if len(updated_domains) > 0:
shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
# Write the OpenDKIM configuration tables for all of the domains.
if write_opendkim_tables(get_mail_domains(env), env):
# Settings changed. Kick opendkim.
shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
if len(updated_domains) == 0:
# If this is the only thing that changed?
updated_domains.append("OpenDKIM configuration")
# Clear bind9's DNS cache so our own DNS resolver is up to date.
# (ignore errors with trap=True)
shell('check_call', ["/usr/sbin/rndc", "flush"], trap=True)
if len(updated_domains) == 0:
# if nothing was updated (except maybe OpenDKIM's files), don't show any output
return ""
else:
return "updated DNS: " + ",".join(updated_domains) + "\n"
def build_zones(env):
# What domains (and their zone filenames) should we build?
domains = get_dns_domains(env)
zonefiles = get_dns_zones(env)
# Create a dictionary of domains to a set of attributes for each
# domain, such as whether there are mail users at the domain.
from mailconfig import get_mail_domains
from web_update import get_web_domains
mail_domains = set(get_mail_domains(env))
mail_user_domains = set(get_mail_domains(env, users_only=True)) # i.e. will log in for mail, Nextcloud
web_domains = set(get_web_domains(env))
auto_domains = web_domains - set(get_web_domains(env, include_auto=False))
domains |= auto_domains # www redirects not included in the initial list, see above
# Add ns1/ns2+PRIMARY_HOSTNAME which must also have A/AAAA records
# when the box is acting as authoritative DNS server for its domains.
for ns in ("ns1", "ns2"):
d = ns + "." + env["PRIMARY_HOSTNAME"]
domains.add(d)
auto_domains.add(d)
domains = {
domain: {
"user": domain in mail_user_domains,
"mail": domain in mail_domains,
"web": domain in web_domains,
"auto": domain in auto_domains,
}
for domain in domains
}
# For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is
# singned and valid. Check that now rather than repeatedly for each domain.
domains[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env)
# Load custom records to add to zones.
additional_records = list(get_custom_dns_config(env))
# Build DNS records for each zone.
for domain, zonefile in zonefiles:
# Build the records to put in the zone.
records = build_zone(domain, domains, additional_records, env)
yield (domain, zonefile, records)
def get_dns_domains(env): # Add all domain names in use by email users and mail aliases, any # domains we serve web for (except www redirects because that would # lead to infinite recursion here) and ensure PRIMARY_HOSTNAME is in the list. from mailconfig import get_mail_domains from web_update import get_web_domains domains = set() domains |= set(get_mail_domains(env)) domains |= set(get_web_domains(env, include_www_redirects=False)) domains.add(env['PRIMARY_HOSTNAME']) return domains
def get_web_domains(env): # What domains should we serve HTTP/HTTPS for? domains = set() # Add all domain names in use by email users and mail aliases. domains |= get_mail_domains(env) # Ensure the PRIMARY_HOSTNAME is in the list. domains.add(env['PRIMARY_HOSTNAME']) # Sort the list. Put PRIMARY_HOSTNAME first so it becomes the # default server (nginx's default_server). domains = sort_domains(domains, env) return domains
def do_dns_update(env):
# What domains should we serve DNS for?
domains = set()
# Ensure the PUBLIC_HOSTNAME is in that list.
domains.add(env['PUBLIC_HOSTNAME'])
# Add all domain names in use by email users and mail aliases.
domains |= get_mail_domains(env)
# Make a nice and safe filename for each domain.
zonefiles = []
for domain in domains:
zonefiles.append((domain, urllib.parse.quote(domain, safe='') + ".txt" ))
# Write zone files.
os.makedirs('/etc/nsd/zones', exist_ok=True)
updated_domains = []
for domain, zonefile in zonefiles:
records = build_zone(domain, env)
if write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env):
justtestingdotemail(domain, records)
updated_domains.append(domain)
# Write the main nsd.conf file.
if write_nsd_conf(zonefiles):
# Make sure updated_domains contains *something* if we wrote an updated
# nsd.conf so that we know to restart nsd.
if len(updated_domains) == 0:
updated_domains.append("DNS configuration")
# Kick nsd if anything changed.
if len(updated_domains) > 0:
shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
# Write the OpenDKIM configuration tables.
write_opendkim_tables(zonefiles, env)
# Kick opendkim.
shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
if len(updated_domains) == 0:
# if nothing was updated (except maybe DKIM), don't show any output
return ""
else:
return "updated: " + ",".join(updated_domains) + "\n"
def do_dns_update(env):
# What domains should we serve DNS for?
domains = set()
# Ensure the PUBLIC_HOSTNAME is in that list.
domains.add(env['PUBLIC_HOSTNAME'])
# Add all domain names in use by email users and mail aliases.
domains |= get_mail_domains(env)
# Make a nice and safe filename for each domain.
zonefiles = []
for domain in domains:
zonefiles.append((domain, urllib.parse.quote(domain, safe='') + ".txt" ))
# Write zone files.
os.makedirs('/etc/nsd/zones', exist_ok=True)
updated_domains = []
for domain, zonefile in zonefiles:
records = build_zone(domain, env)
if write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env):
justtestingdotemail(domain, records)
updated_domains.append(domain)
# Write the main nsd.conf file.
if write_nsd_conf(zonefiles):
# Make sure updated_domains contains *something* if we wrote an updated
# nsd.conf so that we know to restart nsd.
if len(updated_domains) == 0:
updated_domains.append("DNS configuration")
# Kick nsd if anything changed.
if len(updated_domains) > 0:
os.system("service nsd restart")
# Write the OpenDKIM configuration tables.
write_opendkim_tables(zonefiles, env)
# Kick opendkim.
os.system("service opendkim restart")
if len(updated_domains) == 0:
# if nothing was updated (except maybe DKIM), don't show any output
return ""
else:
return "updated: " + ",".join(updated_domains) + "\n"
def build_wkd():
# Clean everything
try:
shutil.rmtree(WKD_LOCATION)
except FileNotFoundError:
pass
os.mkdir(WKD_LOCATION, mode=0o755)
# We serve WKD for all our emails and aliases (even if there are no keys)
for domain in mailconfig.get_mail_domains(env, users_only=False):
os.mkdir(f"{WKD_LOCATION}/{domain}/", mode=0o755)
for email, fpr in parse_wkd_list()[1]:
local, domain = email.split("@", 1)
localhash = zbase32(sha1(local.lower().encode()))
with open(f"{WKD_LOCATION}/{domain}/{localhash}", "wb") as k:
k.write(strip_and_export(fpr, email))
def get_web_domains(env): # What domains should we serve websites for? domains = set() # At the least it's the PRIMARY_HOSTNAME so we can serve webmail # as well as Z-Push for Exchange ActiveSync. domains.add(env['PRIMARY_HOSTNAME']) # Also serve web for all mail domains so that we might at least # provide auto-discover of email settings, and also a static website # if the user wants to make one. These will require an SSL cert. # ...Unless the domain has an A/AAAA record that maps it to a different # IP address than this box. Remove those domains from our list. domains |= (get_mail_domains(env) - get_domains_with_a_records(env)) # Sort the list so the nginx conf gets written in a stable order. domains = sort_domains(domains, env) return domains
def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) domains_to_check = mail_domains | dns_domains | web_domains # Get the list of domains that we don't serve web for because of a custom CNAME/A record. domains_with_a_records = get_domains_with_a_records(env) # Serial version: Other one currently dies... for domain in sort_domains(domains_to_check, env): run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains |= (get_mail_domains(env) - get_domains_with_a_records(env))
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
# if their parent is in the domains to check list.
domains_to_check = [
d for d in domains_to_check
if not (
d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
and len(d.split(".", 1)) == 2
and d.split(".", 1)[1] in domains_to_check
)
]
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def mail_domains():
return "".join(x+"\n" for x in get_mail_domains(env))
def build_zone(domain, all_domains, additional_records, www_redirect_domains, env, is_zone=True):
records = []
# For top-level zones, define the authoritative name servers.
#
# Normally we are our own nameservers. Some TLDs require two distinct IP addresses,
# so we allow the user to override the second nameserver definition so that
# secondary DNS can be set up elsewhere.
#
# 'False' in the tuple indicates these records would not be used if the zone
# is managed outside of the box.
if is_zone:
# Obligatory definition of ns1.PRIMARY_HOSTNAME.
records.append((None, "NS", "ns1.%s." % env["PRIMARY_HOSTNAME"], False))
# Define ns2.PRIMARY_HOSTNAME or whatever the user overrides.
# User may provide one or more additional nameservers
secondary_ns_list = get_secondary_dns(additional_records, mode="NS") \
or ["ns2." + env["PRIMARY_HOSTNAME"]]
for secondary_ns in secondary_ns_list:
records.append((None, "NS", secondary_ns+'.', False))
# In PRIMARY_HOSTNAME...
if domain == env["PRIMARY_HOSTNAME"]:
# Define ns1 and ns2.
# 'False' in the tuple indicates these records would not be used if the zone
# is managed outside of the box.
records.append(("ns1", "A", env["PUBLIC_IP"], False))
records.append(("ns2", "A", env["PUBLIC_IP"], False))
if env.get('PUBLIC_IPV6'):
records.append(("ns1", "AAAA", env["PUBLIC_IPV6"], False))
records.append(("ns2", "AAAA", env["PUBLIC_IPV6"], False))
# Set the A/AAAA records. Do this early for the PRIMARY_HOSTNAME so that the user cannot override them
# and we can provide different explanatory text.
records.append((None, "A", env["PUBLIC_IP"], "Required. Sets the IP address of the box."))
if env.get("PUBLIC_IPV6"): records.append((None, "AAAA", env["PUBLIC_IPV6"], "Required. Sets the IPv6 address of the box."))
# Add a DANE TLSA record for SMTP.
records.append(("_25._tcp", "TLSA", build_tlsa_record(env), "Recommended when DNSSEC is enabled. Advertises to mail servers connecting to the box that mandatory encryption should be used."))
# Add a DANE TLSA record for HTTPS, which some browser extensions might make use of.
records.append(("_443._tcp", "TLSA", build_tlsa_record(env), "Optional. When DNSSEC is enabled, provides out-of-band HTTPS certificate validation for a few web clients that support it."))
# Add a SSHFP records to help SSH key validation. One per available SSH key on this system.
for value in build_sshfp_records():
records.append((None, "SSHFP", value, "Optional. Provides an out-of-band method for verifying an SSH key before connecting. Use 'VerifyHostKeyDNS yes' (or 'VerifyHostKeyDNS ask') when connecting with ssh."))
# Add DNS records for any subdomains of this domain. We should not have a zone for
# both a domain and one of its subdomains.
subdomains = [d for d in all_domains if d.endswith("." + domain)]
for subdomain in subdomains:
subdomain_qname = subdomain[0:-len("." + domain)]
subzone = build_zone(subdomain, [], additional_records, www_redirect_domains, env, is_zone=False)
for child_qname, child_rtype, child_value, child_explanation in subzone:
if child_qname == None:
child_qname = subdomain_qname
else:
child_qname += "." + subdomain_qname
records.append((child_qname, child_rtype, child_value, child_explanation))
has_rec_base = list(records) # clone current state
def has_rec(qname, rtype, prefix=None):
for rec in has_rec_base:
if rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)):
return True
return False
# The user may set other records that don't conflict with our settings.
# Don't put any TXT records above this line, or it'll prevent any custom TXT records.
for qname, rtype, value in filter_custom_records(domain, additional_records):
# Don't allow custom records for record types that override anything above.
# But allow multiple custom records for the same rtype --- see how has_rec_base is used.
if has_rec(qname, rtype): continue
# The "local" keyword on A/AAAA records are short-hand for our own IP.
# This also flags for web configuration that the user wants a website here.
if rtype == "A" and value == "local":
value = env["PUBLIC_IP"]
if rtype == "AAAA" and value == "local":
if "PUBLIC_IPV6" in env:
value = env["PUBLIC_IPV6"]
else:
continue
records.append((qname, rtype, value, "(Set by user.)"))
# Add defaults if not overridden by the user's custom settings (and not otherwise configured).
# Any CNAME or A record on the qname overrides A and AAAA. But when we set the default A record,
# we should not cause the default AAAA record to be skipped because it thinks a custom A record
# was set. So set has_rec_base to a clone of the current set of DNS settings, and don't update
# during this process.
has_rec_base = list(records)
defaults = [
(None, "A", env["PUBLIC_IP"], "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain),
(None, "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain),
]
if "www." + domain in www_redirect_domains:
defaults += [
("www", "A", env["PUBLIC_IP"], "Optional. Sets the IP address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain),
("www", "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain),
]
for qname, rtype, value, explanation in defaults:
if value is None or value.strip() == "": continue # skip IPV6 if not set
if not is_zone and qname == "www": continue # don't create any default 'www' subdomains on what are themselves subdomains
# Set the default record, but not if:
# (1) there is not a user-set record of the same type already
# (2) there is not a CNAME record already, since you can't set both and who knows what takes precedence
# (2) there is not an A record already (if this is an A record this is a dup of (1), and if this is an AAAA record then don't set a default AAAA record if the user sets a custom A record, since the default wouldn't make sense and it should not resolve if the user doesn't provide a new AAAA record)
if not has_rec(qname, rtype) and not has_rec(qname, "CNAME") and not has_rec(qname, "A"):
records.append((qname, rtype, value, explanation))
# Don't pin the list of records that has_rec checks against anymore.
has_rec_base = records
# The MX record says where email for the domain should be delivered: Here!
if not has_rec(None, "MX", prefix="10 "):
records.append((None, "MX", "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
# SPF record: Permit the box ('mx', see above) to send mail on behalf of
# the domain, and no one else.
# Skip if the user has set a custom SPF record.
if not has_rec(None, "TXT", prefix="v=spf1 "):
records.append((None, "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain))
# Append the DKIM TXT record to the zone as generated by OpenDKIM.
# Skip if the user has set a DKIM record already.
opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt')
with open(opendkim_record_file) as orf:
m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain))
# Append a DMARC record.
# Skip if the user has set a DMARC record already.
if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "):
records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain))
# For any subdomain with an A record but no SPF or DMARC record, add strict policy records.
all_resolvable_qnames = set(r[0] for r in records if r[1] in ("A", "AAAA"))
for qname in all_resolvable_qnames:
if not has_rec(qname, "TXT", prefix="v=spf1 "):
records.append((qname, "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % (qname + "." + domain)))
dmarc_qname = "_dmarc" + ("" if qname is None else "." + qname)
if not has_rec(dmarc_qname, "TXT", prefix="v=DMARC1; "):
records.append((dmarc_qname, "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % (qname + "." + domain)))
# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
# for autoconfiguration of mail clients (so only domains hosting user accounts need it).
# The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot).
if domain != env["PRIMARY_HOSTNAME"] and domain in get_mail_domains(env, users_only=True):
for dav in ("card", "cal"):
qname = "_" + dav + "davs._tcp"
if not has_rec(qname, "SRV"):
records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain."))
# Adds autoconfiguration A records for all domains that there are user accounts at.
# This allows the following clients to automatically configure email addresses in the respective applications.
# autodiscover.* - Z-Push ActiveSync Autodiscover
# autoconfig.* - Thunderbird Autoconfig
if domain in get_mail_domains(env, users_only=True):
autodiscover_records = [
("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
]
for qname, rtype, value, explanation in autodiscover_records:
if value is None or value.strip() == "": continue # skip IPV6 if not set
if not has_rec(qname, rtype):
records.append((qname, rtype, value, explanation))
# If this is a domain name that there are email addresses configured for, i.e. "something@"
# this domain name, then the domain name is a MTA-STS (https://tools.ietf.org/html/rfc8461)
# Policy Domain.
#
# A "_mta-sts" TXT record signals the presence of a MTA-STS policy. The id field helps clients
# cache the policy. It should be stable so we don't update DNS unnecessarily but change when
# the policy changes. It must be at most 32 letters and numbers, so we compute a hash of the
# policy file.
#
# The policy itself is served at the "mta-sts" (no underscore) subdomain over HTTPS. Therefore
# the TLS certificate used by Postfix for STARTTLS must be a valid certificate for the MX
# domain name (PRIMARY_HOSTNAME) *and* the TLS certificate used by nginx for HTTPS on the mta-sts
# subdomain must be valid certificate for that domain. Do not set an MTA-STS policy if either
# certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not
# yet been provisioned). Since we cannot provision a certificate without A/AAAA records, we
# always set them --- only the TXT records depend on there being valid certificates.
mta_sts_enabled = False
mta_sts_records = [
("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
]
if domain in get_mail_domains(env):
# Check that PRIMARY_HOSTNAME and the mta_sts domain both have valid certificates.
for d in (env['PRIMARY_HOSTNAME'], "mta-sts." + domain):
cert = get_ssl_certificates(env).get(d)
if not cert:
break # no certificate provisioned for this domain
cert_status = check_certificate(d, cert['certificate'], cert['private-key'])
if cert_status[0] != 'OK':
break # certificate is not valid
else:
# 'break' was not encountered above, so both domains are good
mta_sts_enabled = True
if mta_sts_enabled:
# Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy
# file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters
# instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its
# first 20 characters, which is more than sufficient to change whenever the policy file changes
# (and ensures any '=' padding at the end of the base64 encoding is dropped).
with open("/var/lib/mailinabox/mta-sts.txt", "rb") as f:
mta_sts_policy_id = base64.b64encode(hashlib.sha1(f.read()).digest(), altchars=b"AA").decode("ascii")[0:20]
mta_sts_records.extend([
("_mta-sts", "TXT", "v=STSv1; id=" + mta_sts_policy_id, "Optional. Part of the MTA-STS policy for incoming mail. If set, a MTA-STS policy must also be published.")
])
# Rules can be custom configured accoring to https://tools.ietf.org/html/rfc8460.
# Skip if the rules below if the user has set a custom _smtp._tls record.
if not has_rec("_smtp._tls", "TXT", prefix="v=TLSRPTv1;"):
tls_rpt_string = ""
tls_rpt_email = env.get("MTA_STS_TLSRPT_EMAIL", "postmaster@%s" % env['PRIMARY_HOSTNAME'])
if tls_rpt_email: # if a reporting address is not cleared
tls_rpt_string = " rua=mailto:%s" % tls_rpt_email
mta_sts_records.append(("_smtp._tls", "TXT", "v=TLSRPTv1;%s" % tls_rpt_string, "Optional. Enables MTA-STS reporting."))
for qname, rtype, value, explanation in mta_sts_records:
if value is None or value.strip() == "": continue # skip IPV6 if not set
if not has_rec(qname, rtype):
records.append((qname, rtype, value, explanation))
# Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter.
records.sort(key = lambda rec : list(reversed(rec[0].split(".")) if rec[0] is not None else ""))
return records
def mail_domains():
return "".join(x + "\n" for x in get_mail_domains(env))
