def ssl_get_status():
from ssl_certificates import get_certificates_to_provision
from web_update import get_web_domains_info, get_web_domains
# What domains can we provision certificates for? What unexpected problems do we have?
provision, cant_provision = get_certificates_to_provision(env, show_extended_problems=False)
# What's the current status of TLS certificates on all of the domain?
domains_status = get_web_domains_info(env)
domains_status = [{"domain": d["domain"], "status": d["ssl_certificate"][0], "text": d["ssl_certificate"][1]} for d
in domains_status]
# Warn the user about domain names not hosted here because of other settings.
for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
domains_status.append({
"domain": domain,
"status": "not-applicable",
"text": "The domain's website is hosted elsewhere.",
})
return json_response({
"can_provision": utils.sort_domains(provision, env),
"cant_provision": [{"domain": domain, "problem": cant_provision[domain]} for domain in
utils.sort_domains(cant_provision, env)],
"status": domains_status,
})
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, rtype, value in dns:
if domain not in domains: continue
if rtype == "CNAME" or (rtype in ("A", "AAAA") and value != "local"):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, rtype, value in dns:
if domain not in domains: continue
if rtype == "CNAME" or (rtype in ("A", "AAAA") and value != "local"):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env) + get_default_www_redirects(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
# for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = (
(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
for domain in domains_to_check
)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
# Check the domains.
for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
env["out"].add_heading(domain)
if domain == env["PRIMARY_HOSTNAME"]:
check_primary_hostname_dns(domain, env)
if domain in dns_domains:
check_dns_zone(domain, env, dns_zonefiles)
if domain in mail_domains:
check_mail_domain(domain, env)
if domain in web_domains:
check_web_domain(domain, env)
def get_web_domains(env, include_www_redirects=True):
# What domains should we serve HTTP(S) for?
domains = set()
# Serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one.
domains |= get_mail_domains(env)
if include_www_redirects:
# Add 'www.' subdomains that we want to provide default redirects
# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
# the topmost of each domain we serve.
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains -= get_domains_with_a_records(env)
# Ensure the PRIMARY_HOSTNAME is in the list so we can serve webmail
# as well as Z-Push for Exchange ActiveSync. This can't be removed
# by a custom A/AAAA record and is never a 'www.' redirect.
domains.add(env['PRIMARY_HOSTNAME'])
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def run_domain_checks(env): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) # Check the domains. for domain in sort_domains(mail_domains | dns_domains | web_domains, env): env["out"].add_heading(domain) if domain == env["PRIMARY_HOSTNAME"]: check_primary_hostname_dns(domain, env, dns_domains, dns_zonefiles) if domain in dns_domains: check_dns_zone(domain, env, dns_zonefiles) if domain in mail_domains: check_mail_domain(domain, env) if domain in web_domains: check_web_domain(domain, env) if domain in dns_domains: check_dns_zone_suggestions(domain, env, dns_zonefiles)
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, env, dns_domains, dns_zonefiles, mail_domains,
web_domains) for domain in domains_to_check)
pool = multiprocessing.pool.Pool(processes=10)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
output = BufferedOutput()
for domain in sort_domains(ret, env):
ret[domain].playback(output)
return output
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide Webfinger and ActiveSync auto-discover of email settings
# (though the latter isn't really working). These will require that
# an SSL cert be installed.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, value in dns.items():
if domain not in domains: continue
if (isinstance(value, str) and (value != "local")) \
or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
# Check the domains.
for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
print(domain)
print("=" * len(domain))
if domain == env["PRIMARY_HOSTNAME"]:
check_primary_hostname_dns(domain, env)
check_alias_exists("administrator@" + domain, env)
if domain in dns_domains:
check_dns_zone(domain, env, dns_zonefiles)
if domain in mail_domains:
check_mail_domain(domain, env)
if domain == env["PRIMARY_HOSTNAME"] or domain in web_domains:
# We need a SSL certificate for PRIMARY_HOSTNAME because that's where the
# user will log in with IMAP or webmail. Any other domain we serve a
# website for also needs a signed certificate.
check_ssl_cert(domain, env)
print()
def get_dns_zones(env):
# What domains should we create DNS zones for? Never create a zone for
# a domain & a subdomain of that domain.
domains = get_dns_domains(env)
# Exclude domains that are subdomains of other domains we know. Proceed
# by looking at shorter domains first.
zone_domains = set()
for domain in sorted(domains, key=lambda d : len(d)):
for d in zone_domains:
if domain.endswith("." + d):
# We found a parent domain already in the list.
break
else:
# 'break' did not occur: there is no parent domain.
zone_domains.add(domain)
# Make a nice and safe filename for each domain.
zonefiles = []
for domain in zone_domains:
zonefiles.append([domain, safe_domain_name(domain) + ".txt"])
# Sort the list so that the order is nice and so that nsd.conf has a
# stable order so we don't rewrite the file & restart the service
# meaninglessly.
zone_order = sort_domains([ zone[0] for zone in zonefiles ], env)
zonefiles.sort(key = lambda zone : zone_order.index(zone[0]) )
return zonefiles
def get_dns_zones(env):
# What domains should we create DNS zones for? Never create a zone for
# a domain & a subdomain of that domain.
domains = get_dns_domains(env)
# Exclude domains that are subdomains of other domains we know. Proceed
# by looking at shorter domains first.
zone_domains = set()
for domain in sorted(domains, key=lambda d: len(d)):
for d in zone_domains:
if domain.endswith("." + d):
# We found a parent domain already in the list.
break
else:
# 'break' did not occur: there is no parent domain.
zone_domains.add(domain)
# Make a nice and safe filename for each domain.
zonefiles = []
for domain in zone_domains:
zonefiles.append([domain, safe_domain_name(domain) + ".txt"])
# Sort the list so that the order is nice and so that nsd.conf has a
# stable order so we don't rewrite the file & restart the service
# meaninglessly.
zone_order = sort_domains([zone[0] for zone in zonefiles], env)
zonefiles.sort(key=lambda zone: zone_order.index(zone[0]))
return zonefiles
def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
# What domains should we serve HTTP(S) for?
domains = set()
# Serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one.
domains |= get_mail_domains(env)
if include_www_redirects:
# Add 'www.' subdomains that we want to provide default redirects
# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
# the topmost of each domain we serve.
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
if exclude_dns_elsewhere:
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains -= get_domains_with_a_records(env)
# Ensure the PRIMARY_HOSTNAME is in the list so we can serve webmail
# as well as Z-Push for Exchange ActiveSync. This can't be removed
# by a custom A/AAAA record and is never a 'www.' redirect.
domains.add(env['PRIMARY_HOSTNAME'])
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide Webfinger and ActiveSync auto-discover of email settings
# (though the latter isn't really working). These will require that
# an SSL cert be installed.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, value in dns.items():
if domain not in domains: continue
if (isinstance(value, str) and (value != "local")) \
or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def get_default_www_redirects(env):
# Returns a list of www subdomains that we want to provide default redirects
# for, i.e. any www's that aren't domains the user has actually configured
# to serve for real. Which would be unusual.
web_domains = set(get_web_domains(env))
www_domains = set('www.' + zone for zone, zonefile in get_dns_zones(env))
return sort_domains(www_domains - web_domains - get_domains_with_a_records(env), env)
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
ssl_certificates = get_ssl_certificates(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def dns_get_records(qname=None, rtype=None):
# Get the current set of custom DNS records.
from dns_update import get_custom_dns_config, get_dns_zones
records = get_custom_dns_config(env, only_real_records=True)
# Filter per the arguments for the more complex GET routes below.
records = [
r for r in records
if (not qname or r[0] == qname) and (not rtype or r[1] == rtype)
]
# Make a better data structure.
records = [{
"qname": r[0],
"rtype": r[1],
"value": r[2],
"ttl": r[3],
"sort-order": {},
} for r in records]
# To help with grouping by zone in qname sorting, label each record with which zone it is in.
# There's an inconsistency in how we handle zones in get_dns_zones and in sort_domains, so
# do this first before sorting the domains within the zones.
zones = utils.sort_domains([z[0] for z in get_dns_zones(env)], env)
for r in records:
for z in zones:
if r["qname"] == z or r["qname"].endswith("." + z):
r["zone"] = z
break
# Add sorting information. The 'created' order follows the order in the YAML file on disk,
# which tracs the order entries were added in the control panel since we append to the end.
# The 'qname' sort order sorts by our standard domain name sort (by zone then by qname),
# then by rtype, and last by the original order in the YAML file (since sorting by value
# may not make sense, unless we parse IP addresses, for example).
for i, r in enumerate(records):
r["sort-order"]["created"] = i
domain_sort_order = utils.sort_domains([r["qname"] for r in records], env)
for i, r in enumerate(
sorted(records,
key=lambda r:
(zones.index(r["zone"]), domain_sort_order.index(r[
"qname"]), r["rtype"]))):
r["sort-order"]["qname"] = i
# Return.
return json_response(records)
def get_default_www_redirects(env):
# Returns a list of www subdomains that we want to provide default redirects
# for, i.e. any www's that aren't domains the user has actually configured
# to serve for real. Which would be unusual.
web_domains = set(get_web_domains(env))
www_domains = set('www.' + zone for zone, zonefile in get_dns_zones(env))
return sort_domains(
www_domains - web_domains - get_domains_with_a_records(env), env)
def get_mail_aliases_ex(env):
# Returns a complex data structure of all mail aliases, similar
# to get_mail_users_ex.
#
# [
# {
# domain: "domain.tld",
# alias: [
# {
# source: "*****@*****.**", # IDNA-encoded
# source_display: "*****@*****.**", # full Unicode
# destination: ["*****@*****.**", "*****@*****.**", ...],
# required: True|False
# },
# ...
# ]
# },
# ...
# ]
required_aliases = get_required_aliases(env)
domains = {}
for source, destination in get_mail_aliases(env):
# get alias info
domain = get_domain(source)
required = ((source in required_aliases)
or (source == get_system_administrator(env)))
# add to list
if not domain in domains:
domains[domain] = {
"domain": domain,
"aliases": [],
}
domains[domain]["aliases"].append({
"source":
source,
"source_display":
prettify_idn_email_address(source),
"destination": [
prettify_idn_email_address(d.strip())
for d in destination.split(",")
],
"required":
required,
})
# Sort domains.
domains = [
domains[domain] for domain in utils.sort_domains(domains.keys(), env)
]
# Sort aliases within each domain first by required-ness then lexicographically by source address.
for domain in domains:
domain["aliases"].sort(
key=lambda alias: (alias["required"], alias["source"]))
return domains
def get_mail_aliases_ex(env):
# Returns a complex data structure of all mail aliases, similar
# to get_mail_users_ex.
#
# [
# {
# domain: "domain.tld",
# alias: [
# {
# address: "*****@*****.**", # IDNA-encoded
# address_display: "*****@*****.**", # full Unicode
# forwards_to: ["*****@*****.**", "*****@*****.**", ...],
# permitted_senders: ["*****@*****.**", "*****@*****.**", ...] OR null,
# auto: True|False
# },
# ...
# ]
# },
# ...
# ]
domains = {}
for address, forwards_to, permitted_senders, auto in get_mail_aliases(env):
# skip auto domain maps since these are not informative in the control panel's aliases list
if auto and address.startswith("@"): continue
# get alias info
domain = get_domain(address)
# add to list
if not domain in domains:
domains[domain] = {
"domain": domain,
"aliases": [],
}
domains[domain]["aliases"].append({
"address": address,
"address_display": prettify_idn_email_address(address),
"forwards_to": [prettify_idn_email_address(r.strip()) for r in forwards_to.split(",")],
"permitted_senders": [prettify_idn_email_address(s.strip()) for s in permitted_senders.split(",")] if permitted_senders is not None else None,
"auto": bool(auto),
})
# Sort domains.
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
# Sort aliases within each domain first by required-ness then lexicographically by address.
for domain in domains:
domain["aliases"].sort(key = lambda alias : (alias["auto"], alias["address"]))
return domains
def get_mail_aliases_ex(env):
# Returns a complex data structure of all mail aliases, similar
# to get_mail_users_ex.
#
# [
# {
# domain: "domain.tld",
# alias: [
# {
# address: "*****@*****.**", # IDNA-encoded
# address_display: "*****@*****.**", # full Unicode
# forwards_to: ["*****@*****.**", "*****@*****.**", ...],
# permitted_senders: ["*****@*****.**", "*****@*****.**", ...] OR null,
# required: True|False
# },
# ...
# ]
# },
# ...
# ]
required_aliases = get_required_aliases(env)
domains = {}
for address, forwards_to, permitted_senders in get_mail_aliases(env):
# get alias info
domain = get_domain(address)
required = (address in required_aliases)
# add to list
if not domain in domains:
domains[domain] = {
"domain": domain,
"aliases": [],
}
domains[domain]["aliases"].append({
"address": address,
"address_display": prettify_idn_email_address(address),
"forwards_to": [prettify_idn_email_address(r.strip()) for r in forwards_to.split(",")],
"permitted_senders": [prettify_idn_email_address(s.strip()) for s in
permitted_senders.split(",")] if permitted_senders is not None else None,
"required": required,
})
# Sort domains.
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
# Sort aliases within each domain first by required-ness then lexicographically by address.
for domain in domains:
domain["aliases"].sort(key=lambda alias: (alias["required"], alias["address"]))
return domains
def get_web_domains(env): # What domains should we serve HTTP/HTTPS for? domains = set() # Add all domain names in use by email users and mail aliases. domains |= get_mail_domains(env) # Ensure the PRIMARY_HOSTNAME is in the list. domains.add(env['PRIMARY_HOSTNAME']) # Sort the list. Put PRIMARY_HOSTNAME first so it becomes the # default server (nginx's default_server). domains = sort_domains(domains, env) return domains
def get_web_domains(env,
include_www_redirects=True,
include_auto=True,
exclude_dns_elsewhere=True,
categories=['mail', 'ssl']):
# What domains should we serve HTTP(S) for?
domains = set()
# Serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one.
for category in categories:
domains |= get_mail_domains(env, category=category)
if include_www_redirects and include_auto:
# Add 'www.' subdomains that we want to provide default redirects
# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
# the topmost of each domain we serve.
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
if 'mail' in categories and include_auto:
# Add Autoconfiguration domains for domains that there are user accounts at:
# 'autoconfig.' for Mozilla Thunderbird auto setup.
# 'autodiscover.' for Activesync autodiscovery.
domains |= set(
'autoconfig.' + maildomain
for maildomain in get_mail_domains(env, users_only=True))
domains |= set(
'autodiscover.' + maildomain
for maildomain in get_mail_domains(env, users_only=True))
# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
domains |= set('mta-sts.' + maildomain
for maildomain in get_mail_domains(env))
if exclude_dns_elsewhere:
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains -= get_domains_with_a_records(env)
# Ensure the PRIMARY_HOSTNAME is in the list so we can serve webmail
# as well as Z-Push for Exchange ActiveSync. This can't be removed
# by a custom A/AAAA record and is never a 'www.' redirect.
domains.add(env['PRIMARY_HOSTNAME'])
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def get_mail_aliases_ex(env):
# Returns a complex data structure of all mail aliases, similar
# to get_mail_users_ex.
#
# [
# {
# domain: "domain.tld",
# alias: [
# {
# source: "*****@*****.**", # IDNA-encoded
# source_display: "*****@*****.**", # full Unicode
# destination: ["*****@*****.**", "*****@*****.**", ...],
# required: True|False
# },
# ...
# ]
# },
# ...
# ]
required_aliases = get_required_aliases(env)
domains = {}
for source, destination in get_mail_aliases(env):
# get alias info
domain = get_domain(source)
required = ((source in required_aliases) or (source == get_system_administrator(env)))
# add to list
if not domain in domains:
domains[domain] = {
"domain": domain,
"aliases": [],
}
domains[domain]["aliases"].append({
"source": source,
"source_display": prettify_idn_email_address(source),
"destination": [prettify_idn_email_address(d.strip()) for d in destination.split(",")],
"required": required,
})
# Sort domains.
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
# Sort aliases within each domain first by required-ness then lexicographically by source address.
for domain in domains:
domain["aliases"].sort(key = lambda alias : (alias["required"], alias["source"]))
return domains
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
domains |= (get_mail_domains(env) - get_domains_with_a_records(env))
# Sort the list so the nginx conf gets written in a stable order.
domains = sort_domains(domains, env)
return domains
def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) domains_to_check = mail_domains | dns_domains | web_domains # Get the list of domains that we don't serve web for because of a custom CNAME/A record. domains_with_a_records = get_domains_with_a_records(env) # Serial version: Other one currently dies... for domain in sort_domains(domains_to_check, env): run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
def get_web_domains(env): # What domains should we serve websites for? domains = set() # At the least it's the PRIMARY_HOSTNAME so we can serve webmail # as well as Z-Push for Exchange ActiveSync. domains.add(env['PRIMARY_HOSTNAME']) # Also serve web for all mail domains so that we might at least # provide auto-discover of email settings, and also a static website # if the user wants to make one. These will require an SSL cert. # ...Unless the domain has an A/AAAA record that maps it to a different # IP address than this box. Remove those domains from our list. domains |= (get_mail_domains(env) - get_domains_with_a_records(env)) # Sort the list so the nginx conf gets written in a stable order. domains = sort_domains(domains, env) return domains
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
# if their parent is in the domains to check list.
domains_to_check = [
d for d in domains_to_check
if not (
d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
and len(d.split(".", 1)) == 2
and d.split(".", 1)[1] in domains_to_check
)
]
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def get_mail_users_ex(env, with_archived=False):
# Returns a complex data structure of all user accounts, optionally
# including archived (status="inactive") accounts.
#
# [
# {
# domain: "domain.tld",
# users: [
# {
# email: "*****@*****.**",
# privileges: [ "priv1", "priv2", ... ],
# status: "active" | "inactive",
# },
# ...
# ]
# },
# ...
# ]
# Get users and their privileges.
users = []
active_accounts = set()
c = open_database(env)
c.execute('SELECT email, privileges, quota FROM users')
for email, privileges, quota in c.fetchall():
active_accounts.add(email)
(user, domain) = email.split('@')
box_size = 0
box_count = 0
box_quota = 0
percent = ''
try:
dirsize_file = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes/%s/%s/maildirsize' % (domain, user))
with open(dirsize_file, 'r') as f:
box_quota = int(f.readline().split('S')[0])
for line in f.readlines():
(size, count) = line.split(' ')
box_size += int(size)
box_count += int(count)
try:
percent = (box_size / box_quota) * 100
except:
percent = 'Error'
except:
box_size = '?'
box_count = '?'
box_quota = '?'
percent = '?'
if quota == '0':
percent = ''
user = {
"email": email,
"privileges": parse_privs(privileges),
"quota": quota,
"box_quota": box_quota,
"box_size": sizeof_fmt(box_size) if box_size != '?' else box_size,
"percent": '%3.0f%%' % percent if type(percent) != str else percent,
"box_count": box_count,
"status": "active",
}
users.append(user)
# Add in archived accounts.
if with_archived:
root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
for domain in os.listdir(root):
if os.path.isdir(os.path.join(root, domain)):
for user in os.listdir(os.path.join(root, domain)):
email = user + "@" + domain
mbox = os.path.join(root, domain, user)
if email in active_accounts: continue
user = {
"email": email,
"privileges": [],
"status": "inactive",
"mailbox": mbox,
"box_count": '?',
"box_size": '?',
"box_quota": '?',
"percent": '?',
}
users.append(user)
# Group by domain.
domains = { }
for user in users:
domain = get_domain(user["email"])
if domain not in domains:
domains[domain] = {
"domain": domain,
"users": []
}
domains[domain]["users"].append(user)
# Sort domains.
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
# Sort users within each domain first by status then lexicographically by email address.
for domain in domains:
domain["users"].sort(key = lambda user : (user["status"] != "active", user["email"]))
return domains
def get_mail_aliases_ex(env):
# Returns a complex data structure of all mail aliases, similar
# to get_mail_users_ex.
#
# [
# {
# domain: "domain.tld",
# alias: [
# {
# address: "*****@*****.**", # IDNA-encoded
# address_display: "*****@*****.**", # full Unicode
# forwards_to: ["*****@*****.**", "*****@*****.**", ...],
# permitted_senders: ["*****@*****.**", "*****@*****.**", ...] OR null,
# required: True|False,
# description: ""
# },
# ...
# ]
# },
# ...
# ]
aliases = get_mail_aliases(env, as_map=True)
required_aliases = get_required_aliases(env)
domains = {}
for alias_lc in aliases:
alias = aliases[alias_lc]
address = alias_lc
# get alias info
forwards_to = alias["forward_tos"]
permitted_senders = alias["permitted_senders"]
description = alias["description"]
domain = get_domain(address)
required = (address in required_aliases)
# add to list
if not domain in domains:
domains[domain] = {
"domain": domain,
"aliases": [],
}
domains[domain]["aliases"].append({
"address":
address,
"address_display":
prettify_idn_email_address(address),
"forwards_to":
[prettify_idn_email_address(r.strip()) for r in forwards_to],
"permitted_senders":
[prettify_idn_email_address(s.strip())
for s in permitted_senders] if permitted_senders is not None
and len(permitted_senders) > 0 else None,
"required":
required,
"description":
description
})
# Sort domains.
domains = [
domains[domain] for domain in utils.sort_domains(domains.keys(), env)
]
# Sort aliases within each domain first by required-ness then lexicographically by address.
for domain in domains:
domain["aliases"].sort(
key=lambda alias: (alias["required"], alias["address"]))
return domains
def provision_certificates(env, limit_domains):
# What domains should we provision certificates for? And what
# errors prevent provisioning for other domains.
domains, domains_cant_provision = get_certificates_to_provision(env, limit_domains=limit_domains)
# Build a list of what happened on each domain or domain-set.
ret = []
for domain, error in domains_cant_provision.items():
ret.append({
"domains": [domain],
"log": [error],
"result": "skipped",
})
# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
# limit for a single certificate. We'll sort to put related domains together.
max_domains_per_group = 100
domains = sort_domains(domains, env)
certs = []
while len(domains) > 0:
certs.append( domains[:max_domains_per_group] )
domains = domains[max_domains_per_group:]
# Prepare to provision.
# Where should we put our Let's Encrypt account info and state cache.
account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt')
if not os.path.exists(account_path):
os.mkdir(account_path)
# Provision certificates.
for domain_list in certs:
ret.append({
"domains": domain_list,
"log": [],
})
try:
# Create a CSR file for our master private key so that certbot
# uses our private key.
key_file = os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem')
with tempfile.NamedTemporaryFile() as csr_file:
# We could use openssl, but certbot requires
# that the CN domain and SAN domains match
# the domain list passed to certbot, and adding
# SAN domains openssl req is ridiculously complicated.
# subprocess.check_output([
# "openssl", "req", "-new",
# "-key", key_file,
# "-out", csr_file.name,
# "-subj", "/CN=" + domain_list[0],
# "-sha256" ])
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.primitives import hashes
from cryptography.x509.oid import NameOID
builder = x509.CertificateSigningRequestBuilder()
builder = builder.subject_name(x509.Name([ x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0]) ]))
builder = builder.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
builder = builder.add_extension(x509.SubjectAlternativeName(
[x509.DNSName(d) for d in domain_list]
), critical=False)
request = builder.sign(load_pem(load_cert_chain(key_file)[0]), hashes.SHA256(), default_backend())
with open(csr_file.name, "wb") as f:
f.write(request.public_bytes(Encoding.PEM))
# Provision, writing to a temporary file.
webroot = os.path.join(account_path, 'webroot')
os.makedirs(webroot, exist_ok=True)
with tempfile.TemporaryDirectory() as d:
cert_file = os.path.join(d, 'cert_and_chain.pem')
print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".")
certbotret = subprocess.check_output([
"certbot",
"certonly",
#"-v", # just enough to see ACME errors
"--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup
"-d", ",".join(domain_list), # first will be main domain
"--csr", csr_file.name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually
"--cert-path", os.path.join(d, 'cert'), # we only use the full chain
"--chain-path", os.path.join(d, 'chain'), # we only use the full chain
"--fullchain-path", cert_file,
"--webroot", "--webroot-path", webroot,
"--config-dir", account_path,
#"--staging",
], stderr=subprocess.STDOUT).decode("utf8")
install_cert_copy_file(cert_file, env)
ret[-1]["log"].append(certbotret)
ret[-1]["result"] = "installed"
except subprocess.CalledProcessError as e:
ret[-1]["log"].append(e.output.decode("utf8"))
ret[-1]["result"] = "error"
except Exception as e:
ret[-1]["log"].append(str(e))
ret[-1]["result"] = "error"
# Run post-install steps.
ret.extend(post_install_func(env))
# Return what happened with each certificate request.
return ret
def provision_certificates(env, limit_domains):
# What domains should we provision certificates for? And what
# errors prevent provisioning for other domains.
domains, domains_cant_provision = get_certificates_to_provision(
env, limit_domains=limit_domains)
# Build a list of what happened on each domain or domain-set.
ret = []
for domain, error in domains_cant_provision.items():
ret.append({
"domains": [domain],
"log": [error],
"result": "skipped",
})
# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
# limit for a single certificate. We'll sort to put related domains together.
max_domains_per_group = 100
domains = sort_domains(domains, env)
certs = []
while len(domains) > 0:
certs.append(domains[:max_domains_per_group])
domains = domains[max_domains_per_group:]
# Prepare to provision.
# Where should we put our Let's Encrypt account info and state cache.
account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt')
if not os.path.exists(account_path):
os.mkdir(account_path)
# Provision certificates.
for domain_list in certs:
ret.append({
"domains": domain_list,
"log": [],
})
try:
# Create a CSR file for our master private key so that certbot
# uses our private key.
key_file = os.path.join(env['STORAGE_ROOT'], 'ssl',
'ssl_private_key.pem')
with tempfile.NamedTemporaryFile() as csr_file:
# We could use openssl, but certbot requires
# that the CN domain and SAN domains match
# the domain list passed to certbot, and adding
# SAN domains openssl req is ridiculously complicated.
# subprocess.check_output([
# "openssl", "req", "-new",
# "-key", key_file,
# "-out", csr_file.name,
# "-subj", "/CN=" + domain_list[0],
# "-sha256" ])
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.primitives import hashes
from cryptography.x509.oid import NameOID
builder = x509.CertificateSigningRequestBuilder()
builder = builder.subject_name(
x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0])
]))
builder = builder.add_extension(x509.BasicConstraints(
ca=False, path_length=None),
critical=True)
builder = builder.add_extension(x509.SubjectAlternativeName(
[x509.DNSName(d) for d in domain_list]),
critical=False)
request = builder.sign(load_pem(load_cert_chain(key_file)[0]),
hashes.SHA256(), default_backend())
with open(csr_file.name, "wb") as f:
f.write(request.public_bytes(Encoding.PEM))
# Provision, writing to a temporary file.
webroot = os.path.join(account_path, 'webroot')
os.makedirs(webroot, exist_ok=True)
with tempfile.TemporaryDirectory() as d:
cert_file = os.path.join(d, 'cert_and_chain.pem')
print("Provisioning TLS certificates for " +
", ".join(domain_list) + ".")
certbotret = subprocess.check_output(
[
"certbot",
"certonly",
#"-v", # just enough to see ACME errors
"--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup
"-d",
",".join(domain_list), # first will be main domain
"--csr",
csr_file.
name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually
"--cert-path",
os.path.join(d,
'cert'), # we only use the full chain
"--chain-path",
os.path.join(
d, 'chain'), # we only use the full chain
"--fullchain-path",
cert_file,
"--webroot",
"--webroot-path",
webroot,
"--config-dir",
account_path,
#"--staging",
],
stderr=subprocess.STDOUT).decode("utf8")
install_cert_copy_file(cert_file, env)
ret[-1]["log"].append(certbotret)
ret[-1]["result"] = "installed"
except subprocess.CalledProcessError as e:
ret[-1]["log"].append(e.output.decode("utf8"))
ret[-1]["result"] = "error"
except Exception as e:
ret[-1]["log"].append(str(e))
ret[-1]["result"] = "error"
# Run post-install steps.
ret.extend(post_install_func(env))
# Return what happened with each certificate request.
return ret
def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
# Returns a complex data structure of all user accounts, optionally
# including archived (status="inactive") accounts.
#
# [
# {
# domain: "domain.tld",
# users: [
# {
# email: "*****@*****.**",
# privileges: [ "priv1", "priv2", ... ],
# status: "active" | "inactive",
# },
# ...
# ]
# },
# ...
# ]
# Get users and their privileges.
users = []
active_accounts = set()
c = open_database(env)
c.execute('SELECT email, privileges FROM users')
for email, privileges in c.fetchall():
active_accounts.add(email)
user = {
"email": email,
"privileges": parse_privs(privileges),
"status": "active",
}
users.append(user)
if with_slow_info:
user["mailbox_size"] = utils.du(os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes', *reversed(email.split("@"))))
# Add in archived accounts.
if with_archived:
root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
for domain in os.listdir(root):
if os.path.isdir(os.path.join(root, domain)):
for user in os.listdir(os.path.join(root, domain)):
email = user + "@" + domain
mbox = os.path.join(root, domain, user)
if email in active_accounts: continue
user = {
"email": email,
"privileges": "",
"status": "inactive",
"mailbox": mbox,
}
users.append(user)
if with_slow_info:
user["mailbox_size"] = utils.du(mbox)
# Group by domain.
domains = { }
for user in users:
domain = get_domain(user["email"])
if domain not in domains:
domains[domain] = {
"domain": domain,
"users": []
}
domains[domain]["users"].append(user)
# Sort domains.
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
# Sort users within each domain first by status then lexicographically by email address.
for domain in domains:
domain["users"].sort(key = lambda user : (user["status"] != "active", user["email"]))
return domains
def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extended_problems=True, force_domains=None, jsonable=False):
import requests.exceptions
import acme.messages
from free_tls_certificates import client
# What domains should we provision certificates for? And what
# errors prevent provisioning for other domains.
domains, problems = get_certificates_to_provision(env, force_domains=force_domains, show_extended_problems=show_extended_problems)
# Exit fast if there is nothing to do.
if len(domains) == 0:
return {
"requests": [],
"problems": problems,
}
# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
# limit for a single certificate. We'll sort to put related domains together.
domains = sort_domains(domains, env)
certs = []
while len(domains) > 0:
certs.append( domains[0:100] )
domains = domains[100:]
# Prepare to provision.
# Where should we put our Let's Encrypt account info and state cache.
account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt')
if not os.path.exists(account_path):
os.mkdir(account_path)
# Where should we put ACME challenge files. This is mapped to /.well-known/acme_challenge
# by the nginx configuration.
challenges_path = os.path.join(account_path, 'acme_challenges')
if not os.path.exists(challenges_path):
os.mkdir(challenges_path)
# Read in the private key that we use for all TLS certificates. We'll need that
# to generate a CSR (done by free_tls_certificates).
with open(os.path.join(env['STORAGE_ROOT'], 'ssl/ssl_private_key.pem'), 'rb') as f:
private_key = f.read()
# Provision certificates.
ret = []
for domain_list in certs:
# For return.
ret_item = {
"domains": domain_list,
"log": [],
}
ret.append(ret_item)
# Logging for free_tls_certificates.
def my_logger(message):
if logger: logger(message)
ret_item["log"].append(message)
# Attempt to provision a certificate.
try:
try:
cert = client.issue_certificate(
domain_list,
account_path,
agree_to_tos_url=agree_to_tos_url,
private_key=private_key,
logger=my_logger)
except client.NeedToTakeAction as e:
# Write out the ACME challenge files.
for action in e.actions:
if isinstance(action, client.NeedToInstallFile):
fn = os.path.join(challenges_path, action.file_name)
with open(fn, 'w') as f:
f.write(action.contents)
else:
raise ValueError(str(action))
# Try to provision now that the challenge files are installed.
cert = client.issue_certificate(
domain_list,
account_path,
private_key=private_key,
logger=my_logger)
except client.NeedToAgreeToTOS as e:
# The user must agree to the Let's Encrypt terms of service agreement
# before any further action can be taken.
ret_item.update({
"result": "agree-to-tos",
"url": e.url,
})
except client.WaitABit as e:
# We need to hold on for a bit before querying again to see if we can
# acquire a provisioned certificate.
import time, datetime
ret_item.update({
"result": "wait",
"until": e.until_when if not jsonable else e.until_when.isoformat(),
"seconds": (e.until_when - datetime.datetime.now()).total_seconds()
})
except client.AccountDataIsCorrupt as e:
# This is an extremely rare condition.
ret_item.update({
"result": "error",
"message": "Something unexpected went wrong. It looks like your local Let's Encrypt account data is corrupted. There was a problem with the file " + e.account_file_path + ".",
})
except (client.InvalidDomainName, client.NeedToTakeAction, client.ChallengeFailed, acme.messages.Error, requests.exceptions.RequestException) as e:
ret_item.update({
"result": "error",
"message": "Something unexpected went wrong: " + str(e),
})
else:
# A certificate was issued.
install_status = install_cert(domain_list[0], cert['cert'].decode("ascii"), b"\n".join(cert['chain']).decode("ascii"), env, raw=True)
# str indicates the certificate was not installed.
if isinstance(install_status, str):
ret_item.update({
"result": "error",
"message": "Something unexpected was wrong with the provisioned certificate: " + install_status,
})
else:
# A list indicates success and what happened next.
ret_item["log"].extend(install_status)
ret_item.update({
"result": "installed",
})
# Return what happened with each certificate request.
return {
"requests": ret,
"problems": problems,
}
def provision_certificates_cmdline():
import sys
from utils import load_environment, exclusive_process
exclusive_process("update_tls_certificates")
env = load_environment()
verbose = False
headless = False
force_domains = None
show_extended_problems = True
args = list(sys.argv)
args.pop(0) # program name
if args and args[0] == "-v":
verbose = True
args.pop(0)
if args and args[0] == "q":
show_extended_problems = False
args.pop(0)
if args and args[0] == "--headless":
headless = True
args.pop(0)
if args and args[0] == "--force":
force_domains = "ALL"
args.pop(0)
else:
force_domains = args
agree_to_tos_url = None
while True:
# Run the provisioning script. This installs certificates. If there are
# a very large number of domains on this box, it issues separate
# certificates for groups of domains. We have to check the result for
# each group.
def my_logger(message):
if verbose:
print(">", message)
status = provision_certificates(env, agree_to_tos_url=agree_to_tos_url, logger=my_logger, force_domains=force_domains, show_extended_problems=show_extended_problems)
agree_to_tos_url = None # reset to prevent infinite looping
if not status["requests"]:
# No domains need certificates.
if not headless or verbose:
if len(status["problems"]) == 0:
print("No domains hosted on this box need a new TLS certificate at this time.")
elif len(status["problems"]) > 0:
print("No TLS certificates could be provisoned at this time:")
print()
for domain in sort_domains(status["problems"], env):
print("%s: %s" % (domain, status["problems"][domain]))
sys.exit(0)
# What happened?
wait_until = None
wait_domains = []
for request in status["requests"]:
if request["result"] == "agree-to-tos":
# We may have asked already in a previous iteration.
if agree_to_tos_url is not None:
continue
# Can't ask the user a question in this mode. Warn the user that something
# needs to be done.
if headless:
print(", ".join(request["domains"]) + " need a new or renewed TLS certificate.")
print()
print("This box can't do that automatically for you until you agree to Let's Encrypt's")
print("Terms of Service agreement. Use the Mail-in-a-Box control panel to provision")
print("certificates for these domains.")
sys.exit(1)
print("""
I'm going to provision a TLS certificate (formerly called a SSL certificate)
for you from Let's Encrypt (letsencrypt.org).
TLS certificates are cryptographic keys that ensure communication between
you and this box are secure when getting and sending mail and visiting
websites hosted on this box. Let's Encrypt is a free provider of TLS
certificates.
Please open this document in your web browser:
%s
It is Let's Encrypt's terms of service agreement. If you agree, I can
provision that TLS certificate. If you don't agree, you will have an
opportunity to install your own TLS certificate from the Mail-in-a-Box
control panel.
Do you agree to the agreement? Type Y or N and press <ENTER>: """
% request["url"], end='', flush=True)
if sys.stdin.readline().strip().upper() != "Y":
print("\nYou didn't agree. Quitting.")
sys.exit(1)
# Okay, indicate agreement on next iteration.
agree_to_tos_url = request["url"]
if request["result"] == "wait":
# Must wait. We'll record until when. The wait occurs below.
if wait_until is None:
wait_until = request["until"]
else:
wait_until = max(wait_until, request["until"])
wait_domains += request["domains"]
if request["result"] == "error":
print(", ".join(request["domains"]) + ":")
print(request["message"])
if request["result"] == "installed":
print("A TLS certificate was successfully installed for " + ", ".join(request["domains"]) + ".")
if wait_until:
# Wait, then loop.
import time, datetime
print()
print("A TLS certificate was requested for: " + ", ".join(wait_domains) + ".")
first = True
while wait_until > datetime.datetime.now():
if not headless or first:
print ("We have to wait", int(round((wait_until - datetime.datetime.now()).total_seconds())), "seconds for the certificate to be issued...")
time.sleep(10)
first = False
continue # Loop!
if agree_to_tos_url:
# The user agrees to the TOS. Loop to try again by agreeing.
continue # Loop!
# Unless we were instructed to wait, or we just agreed to the TOS,
# we're done for now.
break
# And finally show the domains with problems.
if len(status["problems"]) > 0:
print("TLS certificates could not be provisoned for:")
for domain in sort_domains(status["problems"], env):
print("%s: %s" % (domain, status["problems"][domain]))
def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
# Returns a complex data structure of all user accounts, optionally
# including archived (status="inactive") accounts.
#
# [
# {
# domain: "domain.tld",
# users: [
# {
# email: "*****@*****.**",
# privileges: [ "priv1", "priv2", ... ],
# status: "active",
# aliases: [
# ("*****@*****.**", ["*****@*****.**", ...]),
# ...
# ]
# },
# ...
# ]
# },
# ...
# ]
# Pre-load all aliases.
aliases = get_mail_alias_map(env)
# Get users and their privileges.
users = []
active_accounts = set()
c = open_database(env)
c.execute('SELECT email, privileges FROM users')
for email, privileges in c.fetchall():
active_accounts.add(email)
user = {
"email": email,
"privileges": parse_privs(privileges),
"status": "active",
}
users.append(user)
if with_slow_info:
user["aliases"] = [
(alias, sorted(evaluate_mail_alias_map(alias, aliases, env)))
for alias in aliases.get(email.lower(), [])
]
user["mailbox_size"] = utils.du(os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes', *reversed(email.split("@"))))
# Add in archived accounts.
if with_archived:
root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
for domain in os.listdir(root):
for user in os.listdir(os.path.join(root, domain)):
email = user + "@" + domain
mbox = os.path.join(root, domain, user)
if email in active_accounts: continue
user = {
"email": email,
"privileges": "",
"status": "inactive",
"mailbox": mbox,
}
users.append(user)
if with_slow_info:
user["mailbox_size"] = utils.du(mbox)
# Group by domain.
domains = { }
for user in users:
domain = get_domain(user["email"])
if domain not in domains:
domains[domain] = {
"domain": domain,
"users": []
}
domains[domain]["users"].append(user)
# Sort domains.
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
# Sort users within each domain first by status then lexicographically by email address.
for domain in domains:
domain["users"].sort(key = lambda user : (user["status"] != "active", user["email"]))
return domains
def get_mail_users_ex(env, with_archived=False):
# Returns a complex data structure of all user accounts, optionally
# including archived (status="inactive") accounts.
#
# [
# {
# domain: "domain.tld",
# users: [
# {
# email: "*****@*****.**",
# privileges: [ "priv1", "priv2", ... ],
# status: "active" | "inactive",
# display_name: ""
# },
# ...
# ]
# },
# ...
# ]
# Get users and their privileges.
users = []
active_accounts = set()
c = open_database(env)
response = c.wait(
c.search(env.LDAP_USERS_BASE,
"(objectClass=mailUser)",
attributes=['maildrop', 'mailaccess', 'cn']))
for rec in response:
email = rec['maildrop'][0]
privileges = rec['mailaccess']
display_name = rec['cn'][0]
active_accounts.add(email)
user = {
"email": email,
"privileges": privileges,
"status": "active",
"display_name": display_name
}
users.append(user)
# Add in archived accounts.
if with_archived:
root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
for domain in os.listdir(root):
if os.path.isdir(os.path.join(root, domain)):
for user in os.listdir(os.path.join(root, domain)):
email = user + "@" + domain
mbox = os.path.join(root, domain, user)
if email in active_accounts: continue
user = {
"email": email,
"privileges": [],
"status": "inactive",
"mailbox": mbox,
"display_name": ""
}
users.append(user)
# Group by domain.
domains = {}
for user in users:
domain = get_domain(user["email"])
if domain not in domains:
domains[domain] = {"domain": domain, "users": []}
domains[domain]["users"].append(user)
# Sort domains.
domains = [
domains[domain] for domain in utils.sort_domains(domains.keys(), env)
]
# Sort users within each domain first by status then lexicographically by email address.
for domain in domains:
domain["users"].sort(
key=lambda user: (user["status"] != "active", user["email"]))
return domains
def provision_certificates(env,
agree_to_tos_url=None,
logger=None,
show_extended_problems=True,
force_domains=None,
jsonable=False):
import requests.exceptions
import acme.messages
from free_tls_certificates import client
# What domains should we provision certificates for? And what
# errors prevent provisioning for other domains.
domains, problems = get_certificates_to_provision(
env,
force_domains=force_domains,
show_extended_problems=show_extended_problems)
# Exit fast if there is nothing to do.
if len(domains) == 0:
return {
"requests": [],
"problems": problems,
}
# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
# limit for a single certificate. We'll sort to put related domains together.
domains = sort_domains(domains, env)
certs = []
while len(domains) > 0:
certs.append(domains[0:100])
domains = domains[100:]
# Prepare to provision.
# Where should we put our Let's Encrypt account info and state cache.
account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt')
if not os.path.exists(account_path):
os.mkdir(account_path)
# Where should we put ACME challenge files. This is mapped to /.well-known/acme_challenge
# by the nginx configuration.
challenges_path = os.path.join(account_path, 'acme_challenges')
if not os.path.exists(challenges_path):
os.mkdir(challenges_path)
# Read in the private key that we use for all TLS certificates. We'll need that
# to generate a CSR (done by free_tls_certificates).
with open(os.path.join(env['STORAGE_ROOT'], 'ssl/ssl_private_key.pem'),
'rb') as f:
private_key = f.read()
# Provision certificates.
ret = []
for domain_list in certs:
# For return.
ret_item = {
"domains": domain_list,
"log": [],
}
ret.append(ret_item)
# Logging for free_tls_certificates.
def my_logger(message):
if logger: logger(message)
ret_item["log"].append(message)
# Attempt to provision a certificate.
try:
try:
cert = client.issue_certificate(
domain_list,
account_path,
agree_to_tos_url=agree_to_tos_url,
private_key=private_key,
logger=my_logger)
except client.NeedToTakeAction as e:
# Write out the ACME challenge files.
for action in e.actions:
if isinstance(action, client.NeedToInstallFile):
fn = os.path.join(challenges_path, action.file_name)
with open(fn, 'w') as f:
f.write(action.contents)
else:
raise ValueError(str(action))
# Try to provision now that the challenge files are installed.
cert = client.issue_certificate(domain_list,
account_path,
private_key=private_key,
logger=my_logger)
except client.NeedToAgreeToTOS as e:
# The user must agree to the Let's Encrypt terms of service agreement
# before any further action can be taken.
ret_item.update({
"result": "agree-to-tos",
"url": e.url,
})
except client.WaitABit as e:
# We need to hold on for a bit before querying again to see if we can
# acquire a provisioned certificate.
import time, datetime
ret_item.update({
"result":
"wait",
"until":
e.until_when if not jsonable else e.until_when.isoformat(),
"seconds":
(e.until_when - datetime.datetime.now()).total_seconds()
})
except client.AccountDataIsCorrupt as e:
# This is an extremely rare condition.
ret_item.update({
"result":
"error",
"message":
"Something unexpected went wrong. It looks like your local Let's Encrypt account data is corrupted. There was a problem with the file "
+ e.account_file_path + ".",
})
except (client.InvalidDomainName, client.NeedToTakeAction,
client.ChallengeFailed, client.RateLimited,
acme.messages.Error,
requests.exceptions.RequestException) as e:
ret_item.update({
"result":
"error",
"message":
"Something unexpected went wrong: " + str(e),
})
else:
# A certificate was issued.
install_status = install_cert(domain_list[0],
cert['cert'].decode("ascii"),
b"\n".join(
cert['chain']).decode("ascii"),
env,
raw=True)
# str indicates the certificate was not installed.
if isinstance(install_status, str):
ret_item.update({
"result":
"error",
"message":
"Something unexpected was wrong with the provisioned certificate: "
+ install_status,
})
else:
# A list indicates success and what happened next.
ret_item["log"].extend(install_status)
ret_item.update({
"result": "installed",
})
# Return what happened with each certificate request.
return {
"requests": ret,
"problems": problems,
}
def provision_certificates(env, limit_domains):
# What domains should we provision certificates for? And what
# errors prevent provisioning for other domains.
domains, domains_cant_provision = get_certificates_to_provision(
env, limit_domains=limit_domains)
# Build a list of what happened on each domain or domain-set.
ret = []
for domain, error in domains_cant_provision.items():
ret.append({
"domains": [domain],
"log": [error],
"result": "skipped",
})
# Break into groups by DNS zone: Group every domain with its parent domain, if
# its parent domain is in the list of domains to request a certificate for.
# Start with the zones so that if the zone doesn't need a certificate itself,
# its children will still be grouped together. Sort the provision domains to
# put parents ahead of children.
# Since Let's Encrypt requests are limited to 100 domains at a time,
# we'll create a list of lists of domains where the inner lists have
# at most 100 items. By sorting we also get the DNS zone domain as the first
# entry in each list (unless we overflow beyond 100) which ends up as the
# primary domain listed in each certificate.
from dns_update import get_dns_zones
certs = {}
for zone, zonefile in get_dns_zones(env):
certs[zone] = [[]]
for domain in sort_domains(domains, env):
# Does the domain end with any domain we've seen so far.
for parent in certs.keys():
if domain.endswith("." + parent):
# Add this to the parent's list of domains.
# Start a new group if the list already has
# 100 items.
if len(certs[parent][-1]) == 100:
certs[parent].append([])
certs[parent][-1].append(domain)
break
else:
# This domain is not a child of any domain we've seen yet, so
# start a new group. This shouldn't happen since every zone
# was already added.
certs[domain] = [[domain]]
# Flatten to a list of lists of domains (from a mapping). Remove empty
# lists (zones with no domains that need certs).
certs = sum(certs.values(), [])
certs = [_ for _ in certs if len(_) > 0]
# Prepare to provision.
# Where should we put our Let's Encrypt account info and state cache.
account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt')
if not os.path.exists(account_path):
os.mkdir(account_path)
# Provision certificates.
for domain_list in certs:
ret.append({
"domains": domain_list,
"log": [],
})
try:
# Create a CSR file for our master private key so that certbot
# uses our private key.
key_file = os.path.join(env['STORAGE_ROOT'], 'ssl',
'ssl_private_key.pem')
with tempfile.NamedTemporaryFile() as csr_file:
# We could use openssl, but certbot requires
# that the CN domain and SAN domains match
# the domain list passed to certbot, and adding
# SAN domains openssl req is ridiculously complicated.
# subprocess.check_output([
# "openssl", "req", "-new",
# "-key", key_file,
# "-out", csr_file.name,
# "-subj", "/CN=" + domain_list[0],
# "-sha256" ])
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.primitives import hashes
from cryptography.x509.oid import NameOID
builder = x509.CertificateSigningRequestBuilder()
builder = builder.subject_name(
x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0])
]))
builder = builder.add_extension(x509.BasicConstraints(
ca=False, path_length=None),
critical=True)
builder = builder.add_extension(x509.SubjectAlternativeName(
[x509.DNSName(d) for d in domain_list]),
critical=False)
request = builder.sign(load_pem(load_cert_chain(key_file)[0]),
hashes.SHA256(), default_backend())
with open(csr_file.name, "wb") as f:
f.write(request.public_bytes(Encoding.PEM))
# Provision, writing to a temporary file.
webroot = os.path.join(account_path, 'webroot')
os.makedirs(webroot, exist_ok=True)
with tempfile.TemporaryDirectory() as d:
cert_file = os.path.join(d, 'cert_and_chain.pem')
print("Provisioning TLS certificates for " +
", ".join(domain_list) + ".")
certbotret = subprocess.check_output(
[
"certbot",
"certonly",
#"-v", # just enough to see ACME errors
"--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup
"-d",
",".join(domain_list), # first will be main domain
"--csr",
csr_file.
name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually
"--cert-path",
os.path.join(d,
'cert'), # we only use the full chain
"--chain-path",
os.path.join(
d, 'chain'), # we only use the full chain
"--fullchain-path",
cert_file,
"--webroot",
"--webroot-path",
webroot,
"--config-dir",
account_path,
#"--staging",
],
stderr=subprocess.STDOUT).decode("utf8")
install_cert_copy_file(cert_file, env)
ret[-1]["log"].append(certbotret)
ret[-1]["result"] = "installed"
except subprocess.CalledProcessError as e:
ret[-1]["log"].append(e.output.decode("utf8"))
ret[-1]["result"] = "error"
except Exception as e:
ret[-1]["log"].append(str(e))
ret[-1]["result"] = "error"
# Run post-install steps.
ret.extend(post_install_func(env))
# Return what happened with each certificate request.
return ret
def provision_certificates_cmdline():
import sys
from exclusiveprocess import Lock
from utils import load_environment
Lock(die=True).forever()
env = load_environment()
verbose = False
headless = False
force_domains = None
show_extended_problems = True
args = list(sys.argv)
args.pop(0) # program name
if args and args[0] == "-v":
verbose = True
args.pop(0)
if args and args[0] == "-q":
show_extended_problems = False
args.pop(0)
if args and args[0] == "--headless":
headless = True
args.pop(0)
if args and args[0] == "--force":
force_domains = "ALL"
args.pop(0)
else:
force_domains = args
agree_to_tos_url = None
while True:
# Run the provisioning script. This installs certificates. If there are
# a very large number of domains on this box, it issues separate
# certificates for groups of domains. We have to check the result for
# each group.
def my_logger(message):
if verbose:
print(">", message)
status = provision_certificates(
env,
agree_to_tos_url=agree_to_tos_url,
logger=my_logger,
force_domains=force_domains,
show_extended_problems=show_extended_problems)
agree_to_tos_url = None # reset to prevent infinite looping
if not status["requests"]:
# No domains need certificates.
if not headless or verbose:
if len(status["problems"]) == 0:
print(
"No domains hosted on this box need a new TLS certificate at this time."
)
elif len(status["problems"]) > 0:
print(
"No TLS certificates could be provisoned at this time:"
)
print()
for domain in sort_domains(status["problems"], env):
print("%s: %s" % (domain, status["problems"][domain]))
sys.exit(0)
# What happened?
wait_until = None
wait_domains = []
for request in status["requests"]:
if request["result"] == "agree-to-tos":
# We may have asked already in a previous iteration.
if agree_to_tos_url is not None:
continue
# Can't ask the user a question in this mode. Warn the user that something
# needs to be done.
if headless:
print(", ".join(request["domains"]) +
" need a new or renewed TLS certificate.")
print()
print(
"This box can't do that automatically for you until you agree to Let's Encrypt's"
)
print(
"Terms of Service agreement. Use the Mail-in-a-Box control panel to provision"
)
print("certificates for these domains.")
sys.exit(1)
print("""
I'm going to provision a TLS certificate (formerly called a SSL certificate)
for you from Let's Encrypt (letsencrypt.org).
TLS certificates are cryptographic keys that ensure communication between
you and this box are secure when getting and sending mail and visiting
websites hosted on this box. Let's Encrypt is a free provider of TLS
certificates.
Please open this document in your web browser:
%s
It is Let's Encrypt's terms of service agreement. If you agree, I can
provision that TLS certificate. If you don't agree, you will have an
opportunity to install your own TLS certificate from the Mail-in-a-Box
control panel.
Do you agree to the agreement? Type Y or N and press <ENTER>: """ %
request["url"],
end='',
flush=True)
if sys.stdin.readline().strip().upper() != "Y":
print("\nYou didn't agree. Quitting.")
sys.exit(1)
# Okay, indicate agreement on next iteration.
agree_to_tos_url = request["url"]
if request["result"] == "wait":
# Must wait. We'll record until when. The wait occurs below.
if wait_until is None:
wait_until = request["until"]
else:
wait_until = max(wait_until, request["until"])
wait_domains += request["domains"]
if request["result"] == "error":
print(", ".join(request["domains"]) + ":")
print(request["message"])
if request["result"] == "installed":
print("A TLS certificate was successfully installed for " +
", ".join(request["domains"]) + ".")
if wait_until:
# Wait, then loop.
import time, datetime
print()
print("A TLS certificate was requested for: " +
", ".join(wait_domains) + ".")
first = True
while wait_until > datetime.datetime.now():
if not headless or first:
print(
"We have to wait",
int(
round((wait_until -
datetime.datetime.now()).total_seconds())),
"seconds for the certificate to be issued...")
time.sleep(10)
first = False
continue # Loop!
if agree_to_tos_url:
# The user agrees to the TOS. Loop to try again by agreeing.
continue # Loop!
# Unless we were instructed to wait, or we just agreed to the TOS,
# we're done for now.
break
# And finally show the domains with problems.
if len(status["problems"]) > 0:
print("TLS certificates could not be provisoned for:")
for domain in sort_domains(status["problems"], env):
print("%s: %s" % (domain, status["problems"][domain]))
