def ssl_get_status():
from ssl_certificates import get_certificates_to_provision
from web_update import get_web_domains_info, get_web_domains
# What domains can we provision certificates for? What unexpected problems do we have?
provision, cant_provision = get_certificates_to_provision(env, show_valid_certs=False)
# What's the current status of TLS certificates on all of the domain?
domains_status = get_web_domains_info(env)
domains_status = [
{
"domain": d["domain"],
"status": d["ssl_certificate"][0],
"text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else ""))
} for d in domains_status ]
# Warn the user about domain names not hosted here because of other settings.
for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
domains_status.append({
"domain": domain,
"status": "not-applicable",
"text": "The domain's website is hosted elsewhere.",
})
return json_response({
"can_provision": utils.sort_domains(provision, env),
"status": domains_status,
})
def ssl_get_status():
from ssl_certificates import get_certificates_to_provision
from web_update import get_web_domains_info, get_web_domains
# What domains can we provision certificates for? What unexpected problems do we have?
provision, cant_provision = get_certificates_to_provision(env, show_valid_certs=False)
# What's the current status of TLS certificates on all of the domain?
domains_status = get_web_domains_info(env)
domains_status = [
{
"domain": d["domain"],
"status": d["ssl_certificate"][0],
"text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else ""))
} for d in domains_status ]
# Warn the user about domain names not hosted here because of other settings.
for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
domains_status.append({
"domain": domain,
"status": "not-applicable",
"text": "The domain's website is hosted elsewhere.",
})
return json_response({
"can_provision": utils.sort_domains(provision, env),
"status": domains_status,
})
def build_zones(env): # What domains (and their zone filenames) should we build? domains = get_dns_domains(env) zonefiles = get_dns_zones(env) # Custom records to add to zones. additional_records = list(get_custom_dns_config(env)) from web_update import get_web_domains www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False)) # Build DNS records for each zone. for domain, zonefile in zonefiles: # Build the records to put in the zone. records = build_zone(domain, domains, additional_records, www_redirect_domains, env) yield (domain, zonefile, records)
def build_zones(env): # What domains (and their zone filenames) should we build? domains = get_dns_domains(env) zonefiles = get_dns_zones(env) # Custom records to add to zones. additional_records = list(get_custom_dns_config(env)) from web_update import get_web_domains www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False)) # Build DNS records for each zone. for domain, zonefile in zonefiles: # Build the records to put in the zone. records = build_zone(domain, domains, additional_records, www_redirect_domains, env) yield (domain, zonefile, records)
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
ssl_certificates = get_ssl_certificates(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
# Check the domains.
for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
env["out"].add_heading(domain)
if domain == env["PRIMARY_HOSTNAME"]:
check_primary_hostname_dns(domain, env)
if domain in dns_domains:
check_dns_zone(domain, env, dns_zonefiles)
if domain in mail_domains:
check_mail_domain(domain, env)
if domain in web_domains:
check_web_domain(domain, env)
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env) + get_default_www_redirects(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
# for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = (
(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
for domain in domains_to_check
)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, env, dns_domains, dns_zonefiles, mail_domains,
web_domains) for domain in domains_to_check)
pool = multiprocessing.pool.Pool(processes=10)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
output = BufferedOutput()
for domain in sort_domains(ret, env):
ret[domain].playback(output)
return output
def run_domain_checks(env):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
# Check the domains.
for domain in sort_domains(mail_domains | dns_domains | web_domains, env):
print(domain)
print("=" * len(domain))
if domain == env["PRIMARY_HOSTNAME"]:
check_primary_hostname_dns(domain, env)
check_alias_exists("administrator@" + domain, env)
if domain in dns_domains:
check_dns_zone(domain, env, dns_zonefiles)
if domain in mail_domains:
check_mail_domain(domain, env)
if domain == env["PRIMARY_HOSTNAME"] or domain in web_domains:
# We need a SSL certificate for PRIMARY_HOSTNAME because that's where the
# user will log in with IMAP or webmail. Any other domain we serve a
# website for also needs a signed certificate.
check_ssl_cert(domain, env)
print()
def run_domain_checks(env): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) # Check the domains. for domain in sort_domains(mail_domains | dns_domains | web_domains, env): env["out"].add_heading(domain) if domain == env["PRIMARY_HOSTNAME"]: check_primary_hostname_dns(domain, env, dns_domains, dns_zonefiles) if domain in dns_domains: check_dns_zone(domain, env, dns_zonefiles) if domain in mail_domains: check_mail_domain(domain, env) if domain in web_domains: check_web_domain(domain, env) if domain in dns_domains: check_dns_zone_suggestions(domain, env, dns_zonefiles)
def ssl_install_cert():
from web_update import get_web_domains
from ssl_certificates import install_cert
domain = request.form.get('domain')
ssl_cert = request.form.get('cert')
ssl_chain = request.form.get('chain')
if domain not in get_web_domains(env):
return "Invalid domain name."
return install_cert(domain, ssl_cert, ssl_chain, env)
def ssl_install_cert():
from web_update import get_web_domains
from ssl_certificates import install_cert
domain = request.form.get('domain')
ssl_cert = request.form.get('cert')
ssl_chain = request.form.get('chain')
if domain not in get_web_domains(env):
return "Invalid domain name."
return install_cert(domain, ssl_cert, ssl_chain, env)
def build_zones(env):
# What domains (and their zone filenames) should we build?
domains = get_dns_domains(env)
zonefiles = get_dns_zones(env)
# Create a dictionary of domains to a set of attributes for each
# domain, such as whether there are mail users at the domain.
from mailconfig import get_mail_domains
from web_update import get_web_domains
mail_domains = set(get_mail_domains(env))
mail_user_domains = set(get_mail_domains(env, users_only=True)) # i.e. will log in for mail, Nextcloud
web_domains = set(get_web_domains(env))
auto_domains = web_domains - set(get_web_domains(env, include_auto=False))
domains |= auto_domains # www redirects not included in the initial list, see above
# Add ns1/ns2+PRIMARY_HOSTNAME which must also have A/AAAA records
# when the box is acting as authoritative DNS server for its domains.
for ns in ("ns1", "ns2"):
d = ns + "." + env["PRIMARY_HOSTNAME"]
domains.add(d)
auto_domains.add(d)
domains = {
domain: {
"user": domain in mail_user_domains,
"mail": domain in mail_domains,
"web": domain in web_domains,
"auto": domain in auto_domains,
}
for domain in domains
}
# For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is
# singned and valid. Check that now rather than repeatedly for each domain.
domains[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env)
# Load custom records to add to zones.
additional_records = list(get_custom_dns_config(env))
# Build DNS records for each zone.
for domain, zonefile in zonefiles:
# Build the records to put in the zone.
records = build_zone(domain, domains, additional_records, env)
yield (domain, zonefile, records)
def build_zones(env): # What domains (and their zone filenames) should we build? domains = get_dns_domains(env) zonefiles = get_dns_zones(env) # Custom records to add to zones. additional_records = list(get_custom_dns_config(env)) from web_update import get_web_domains www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False)) # For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is # singned and valid. Check that now rather than repeatedly for each domain. env["-primary-hostname-certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env) # Build DNS records for each zone. for domain, zonefile in zonefiles: # Build the records to put in the zone. records = build_zone(domain, domains, additional_records, www_redirect_domains, env) yield (domain, zonefile, records)
def get_dns_domains(env): # Add all domain names in use by email users and mail aliases, any # domains we serve web for (except www redirects because that would # lead to infinite recursion here) and ensure PRIMARY_HOSTNAME is in the list. from mailconfig import get_mail_domains from web_update import get_web_domains domains = set() domains |= set(get_mail_domains(env)) domains |= set(get_web_domains(env, include_www_redirects=False)) domains.add(env['PRIMARY_HOSTNAME']) return domains
def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env)) domains_to_check = mail_domains | dns_domains | web_domains # Get the list of domains that we don't serve web for because of a custom CNAME/A record. domains_with_a_records = get_domains_with_a_records(env) # Serial version: Other one currently dies... for domain in sort_domains(domains_to_check, env): run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
def run_domain_checks(rounded_time, env, output, pool):
# Get the list of domains we handle mail for.
mail_domains = get_mail_domains(env)
# Get the list of domains we serve DNS zones for (i.e. does not include subdomains).
dns_zonefiles = dict(get_dns_zones(env))
dns_domains = set(dns_zonefiles)
# Get the list of domains we serve HTTPS for.
web_domains = set(get_web_domains(env))
domains_to_check = mail_domains | dns_domains | web_domains
# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
# if their parent is in the domains to check list.
domains_to_check = [
d for d in domains_to_check
if not (
d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
and len(d.split(".", 1)) == 2
and d.split(".", 1)[1] in domains_to_check
)
]
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
# Serial version:
#for domain in sort_domains(domains_to_check, env):
# run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
# Parallelize the checks across a worker pool.
args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
for domain in domains_to_check)
ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
ret = dict(ret) # (domain, output) => { domain: output }
for domain in sort_domains(ret, env):
ret[domain].playback(output)
def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True):
# Get a set of domain names that we can provision certificates for
# using certbot. We start with domains that the box is serving web
# for and subtract:
# * domains not in limit_domains if limit_domains is not empty
# * domains with custom "A" records, i.e. they are hosted elsewhere
# * domains with actual "A" records that point elsewhere
# * domains that already have certificates that will be valid for a while
from web_update import get_web_domains
from status_checks import query_dns, normalize_ip
existing_certs = get_ssl_certificates(env)
plausible_web_domains = get_web_domains(env, exclude_dns_elsewhere=False)
actual_web_domains = get_web_domains(env)
domains_to_provision = set()
domains_cant_provision = { }
for domain in plausible_web_domains:
# Skip domains that the user doesn't want to provision now.
if limit_domains and domain not in limit_domains:
continue
# Check that there isn't an explicit A/AAAA record.
if domain not in actual_web_domains:
domains_cant_provision[domain] = "The domain has a custom DNS A/AAAA record that points the domain elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
# Check that the DNS resolves to here.
else:
# Does the domain resolve to this machine in public DNS? If not,
# we can't do domain control validation. For IPv6 is configured,
# make sure both IPv4 and IPv6 are correct because we don't know
# how Let's Encrypt will connect.
bad_dns = []
for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]:
if not value: continue # IPv6 is not configured
response = query_dns(domain, rtype)
if response != normalize_ip(value):
bad_dns.append("%s (%s)" % (response, rtype))
if bad_dns:
domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \
+ (", ".join(bad_dns)) \
+ "."
else:
# DNS is all good.
# Check for a good existing cert.
existing_cert = get_domain_ssl_files(domain, existing_certs, env, use_main_cert=False, allow_missing_cert=True)
if existing_cert:
existing_cert_check = check_certificate(domain, existing_cert['certificate'], existing_cert['private-key'],
warn_if_expiring_soon=14)
if existing_cert_check[0] == "OK":
if show_valid_certs:
domains_cant_provision[domain] = "The domain has a valid certificate already. ({} Certificate: {}, private key {})".format(
existing_cert_check[1],
existing_cert['certificate'],
existing_cert['private-key'])
continue
domains_to_provision.add(domain)
return (domains_to_provision, domains_cant_provision)
total_size += stat.st_size
return total_size
def wait_for_service(port, public, env, timeout):
# Block until a service on a given port (bound privately or publicly)
# is taking connections, with a maximum timeout.
import socket, time
start = time.perf_counter()
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout / 3)
try:
s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port))
return True
except OSError:
if time.perf_counter() > start + timeout:
return False
time.sleep(min(timeout / 4, 1))
if __name__ == "__main__":
from dns_update import get_dns_domains
from web_update import get_web_domains, get_default_www_redirects
env = load_environment()
domains = get_dns_domains(env) | set(
get_web_domains(env) + get_default_www_redirects(env))
domains = sort_domains(domains, env)
for domain in domains:
print(domain)
def buy_ssl_certificate(api_key, domain, command, env):
if domain != env['PRIMARY_HOSTNAME'] \
and domain not in get_web_domains(env):
raise ValueError("Domain is not %s or a domain we're serving a website for." % env['PRIMARY_HOSTNAME'])
# Initialize.
gandi = xmlrpc.client.ServerProxy('https://rpc.gandi.net/xmlrpc/')
try:
existing_certs = gandi.cert.list(api_key)
except Exception as e:
if "Invalid API key" in str(e):
print("Invalid API key. Check that you copied the API Key correctly from https://www.gandi.net/admin/api_key.")
sys.exit(1)
else:
raise
# Where is the SSL cert stored?
ssl_key, ssl_certificate, ssl_csr_path = get_domain_ssl_files(domain, env)
# Have we already created a cert for this domain?
for cert in existing_certs:
if cert['cn'] == domain:
break
else:
# No existing cert found. Purchase one.
if command != 'purchase':
print("No certificate or order found yet. If you haven't yet purchased a certificate, run ths script again with the 'purchase' command. Otherwise wait a moment and try again.")
sys.exit(1)
else:
# Start an order for a single standard SSL certificate.
# Use DNS validation. Web-based validation won't work because they
# require a file on HTTP but not HTTPS w/o redirects and we don't
# serve anything plainly over HTTP. Email might be another way but
# DNS is easier to automate.
op = gandi.cert.create(api_key, {
"csr": open(ssl_csr_path).read(),
"dcv_method": "dns",
"duration": 1, # year?
"package": "cert_std_1_0_0",
})
print("An SSL certificate has been ordered.")
print()
print(op)
print()
print("In a moment please run this script again with the 'setup' command.")
if cert['status'] == 'pending':
# Get the information we need to update our DNS with a code so that
# Gandi can verify that we own the domain.
dcv = gandi.cert.get_dcv_params(api_key, {
"csr": open(ssl_csr_path).read(),
"cert_id": cert['id'],
"dcv_method": "dns",
"duration": 1, # year?
"package": "cert_std_1_0_0",
})
if dcv["dcv_method"] != "dns":
raise Exception("Certificate ordered with an unknown validation method.")
# Update our DNS data.
dns_config = env['STORAGE_ROOT'] + '/dns/custom.yaml'
if os.path.exists(dns_config):
dns_records = rtyaml.load(open(dns_config))
else:
dns_records = { }
qname = dcv['md5'] + '.' + domain
value = dcv['sha1'] + '.comodoca.com.'
dns_records[qname] = { "CNAME": value }
with open(dns_config, 'w') as f:
f.write(rtyaml.dump(dns_records))
shell('check_call', ['tools/dns_update'])
# Okay, done with this step.
print("DNS has been updated. Gandi will check within 60 minutes.")
print()
print("See https://www.gandi.net/admin/ssl/%d/details for the status of this order." % cert['id'])
elif cert['status'] == 'valid':
# The certificate is ready.
# Check before we overwrite something we shouldn't.
if os.path.exists(ssl_certificate):
cert_status, cert_status_details = check_certificate(None, ssl_certificate, None)
if cert_status != "SELF-SIGNED":
print("Please back up and delete the file %s so I can save your new certificate." % ssl_certificate)
sys.exit(1)
# Form the certificate.
# The certificate comes as a long base64-encoded string. Break in
# into lines in the usual way.
pem = "-----BEGIN CERTIFICATE-----\n"
pem += "\n".join(chunk for chunk in re.split(r"(.{64})", cert['cert']) if chunk != "")
pem += "\n-----END CERTIFICATE-----\n\n"
# Append intermediary certificates.
pem += urllib.request.urlopen("https://www.gandi.net/static/CAs/GandiStandardSSLCA.pem").read().decode("ascii")
# Write out.
with open(ssl_certificate, "w") as f:
f.write(pem)
print("The certificate has been installed in %s. Restarting services..." % ssl_certificate)
# Restart dovecot and if this is for PRIMARY_HOSTNAME.
if domain == env['PRIMARY_HOSTNAME']:
shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
# Restart nginx in all cases.
shell('check_call', ["/usr/sbin/service", "nginx", "restart"])
else:
print("The certificate has an unknown status. Please check https://www.gandi.net/admin/ssl/%d/details for the status of this order." % cert['id'])
def get_certificates_to_provision(env,
limit_domains=None,
show_valid_certs=True):
# Get a set of domain names that we can provision certificates for
# using certbot. We start with domains that the box is serving web
# for and subtract:
# * domains not in limit_domains if limit_domains is not empty
# * domains with custom "A" records, i.e. they are hosted elsewhere
# * domains with actual "A" records that point elsewhere (misconfiguration)
# * domains that already have certificates that will be valid for a while
from web_update import get_web_domains
from status_checks import query_dns, normalize_ip
existing_certs = get_ssl_certificates(env)
plausible_web_domains = get_web_domains(env, exclude_dns_elsewhere=False)
actual_web_domains = get_web_domains(env)
domains_to_provision = set()
domains_cant_provision = {}
for domain in plausible_web_domains:
# Skip domains that the user doesn't want to provision now.
if limit_domains and domain not in limit_domains:
continue
# Check that there isn't an explicit A/AAAA record.
if domain not in actual_web_domains:
domains_cant_provision[
domain] = "The domain has a custom DNS A/AAAA record that points the domain elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
# Check that the DNS resolves to here.
else:
# Does the domain resolve to this machine in public DNS? If not,
# we can't do domain control validation. For IPv6 is configured,
# make sure both IPv4 and IPv6 are correct because we don't know
# how Let's Encrypt will connect.
bad_dns = []
for rtype, value in [("A", env["PUBLIC_IP"]),
("AAAA", env.get("PUBLIC_IPV6"))]:
if not value: continue # IPv6 is not configured
response = query_dns(domain, rtype)
if response != normalize_ip(value):
bad_dns.append("%s (%s)" % (response, rtype))
if bad_dns:
domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \
+ (", ".join(bad_dns)) \
+ "."
else:
# DNS is all good.
# Check for a good existing cert.
existing_cert = get_domain_ssl_files(domain,
existing_certs,
env,
use_main_cert=False,
allow_missing_cert=True)
if existing_cert:
existing_cert_check = check_certificate(
domain,
existing_cert['certificate'],
existing_cert['private-key'],
warn_if_expiring_soon=14)
if existing_cert_check[0] == "OK":
if show_valid_certs:
domains_cant_provision[
domain] = "The domain has a valid certificate already. ({} Certificate: {}, private key {})".format(
existing_cert_check[1],
existing_cert['certificate'],
existing_cert['private-key'])
continue
domains_to_provision.add(domain)
return (domains_to_provision, domains_cant_provision)
def get_certificates_to_provision(env,
show_extended_problems=True,
force_domains=None):
# Get a set of domain names that we should now provision certificates
# for. Provision if a domain name has no valid certificate or if any
# certificate is expiring in 14 days. If provisioning anything, also
# provision certificates expiring within 30 days. The period between
# 14 and 30 days allows us to consolidate domains into multi-domain
# certificates for domains expiring around the same time.
from web_update import get_web_domains
import datetime
now = datetime.datetime.utcnow()
# Get domains with missing & expiring certificates.
certs = get_ssl_certificates(env)
domains = set()
domains_if_any = set()
problems = {}
for domain in get_web_domains(env):
# If the user really wants a cert for certain domains, include it.
if force_domains:
if force_domains == "ALL" or (isinstance(force_domains, list)
and domain in force_domains):
domains.add(domain)
continue
# Include this domain if its certificate is missing, self-signed, or expiring soon.
try:
cert = get_domain_ssl_files(domain,
certs,
env,
allow_missing_cert=True)
except FileNotFoundError as e:
# system certificate is not present
problems[domain] = "Error: " + str(e)
continue
if cert is None:
# No valid certificate available.
domains.add(domain)
else:
cert = cert["certificate_object"]
if cert.issuer == cert.subject:
# This is self-signed. Get a real one.
domains.add(domain)
# Valid certificate today, but is it expiring soon?
elif cert.not_valid_after - now < datetime.timedelta(days=14):
domains.add(domain)
elif cert.not_valid_after - now < datetime.timedelta(days=30):
domains_if_any.add(domain)
# It's valid. Should we report its validness?
elif show_extended_problems:
problems[
domain] = "The certificate is valid for at least another 30 days --- no need to replace."
# Warn the user about domains hosted elsewhere.
if not force_domains and show_extended_problems:
for domain in set(get_web_domains(
env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
problems[
domain] = "The domain's DNS is pointed elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
# Filter out domains that we can't provision a certificate for.
def can_provision_for_domain(domain):
from status_checks import normalize_ip
# Does the domain resolve to this machine in public DNS? If not,
# we can't do domain control validation. For IPv6 is configured,
# make sure both IPv4 and IPv6 are correct because we don't know
# how Let's Encrypt will connect.
import dns.resolver
for rtype, value in [("A", env["PUBLIC_IP"]),
("AAAA", env.get("PUBLIC_IPV6"))]:
if not value: continue # IPv6 is not configured
try:
# Must make the qname absolute to prevent a fall-back lookup with a
# search domain appended, by adding a period to the end.
response = dns.resolver.query(domain + ".", rtype)
except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN,
dns.resolver.NoAnswer) as e:
problems[
domain] = "DNS isn't configured properly for this domain: DNS resolution failed (%s: %s)." % (
rtype, str(e) or repr(e)) # NoAnswer's str is empty
return False
except Exception as e:
problems[
domain] = "DNS isn't configured properly for this domain: DNS lookup had an error: %s." % str(
e)
return False
# Unfortunately, the response.__str__ returns bytes
# instead of string, if it resulted from an AAAA-query.
# We need to convert manually, until this is fixed:
# https://github.com/rthalley/dnspython/issues/204
#
# BEGIN HOTFIX
def rdata__str__(r):
s = r.to_text()
if isinstance(s, bytes):
s = s.decode('utf-8')
return s
# END HOTFIX
if len(response) != 1 or normalize_ip(rdata__str__(
response[0])) != normalize_ip(value):
problems[
domain] = "Domain control validation cannot be performed for this domain because DNS points the domain to another machine (%s %s)." % (
rtype, ", ".join(rdata__str__(r) for r in response))
return False
return True
domains = set(filter(can_provision_for_domain, domains))
# If there are any domains we definitely will provision for, add in
# additional domains to do at this time.
if len(domains) > 0:
domains |= set(filter(can_provision_for_domain, domains_if_any))
return (domains, problems)
def get_certificates_to_provision(env, show_extended_problems=True, force_domains=None):
# Get a set of domain names that we should now provision certificates
# for. Provision if a domain name has no valid certificate or if any
# certificate is expiring in 14 days. If provisioning anything, also
# provision certificates expiring within 30 days. The period between
# 14 and 30 days allows us to consolidate domains into multi-domain
# certificates for domains expiring around the same time.
from web_update import get_web_domains
import datetime
now = datetime.datetime.utcnow()
# Get domains with missing & expiring certificates.
certs = get_ssl_certificates(env)
domains = set()
domains_if_any = set()
problems = { }
for domain in get_web_domains(env):
# If the user really wants a cert for certain domains, include it.
if force_domains:
if force_domains == "ALL" or (isinstance(force_domains, list) and domain in force_domains):
domains.add(domain)
continue
# Include this domain if its certificate is missing, self-signed, or expiring soon.
try:
cert = get_domain_ssl_files(domain, certs, env, allow_missing_cert=True)
except FileNotFoundError as e:
# system certificate is not present
problems[domain] = "Error: " + str(e)
continue
if cert is None:
# No valid certificate available.
domains.add(domain)
else:
cert = cert["certificate_object"]
if cert.issuer == cert.subject:
# This is self-signed. Get a real one.
domains.add(domain)
# Valid certificate today, but is it expiring soon?
elif cert.not_valid_after-now < datetime.timedelta(days=14):
domains.add(domain)
elif cert.not_valid_after-now < datetime.timedelta(days=30):
domains_if_any.add(domain)
# It's valid. Should we report its validness?
elif show_extended_problems:
problems[domain] = "The certificate is valid for at least another 30 days --- no need to replace."
# Warn the user about domains hosted elsewhere.
if not force_domains and show_extended_problems:
for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
problems[domain] = "The domain's DNS is pointed elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
# Filter out domains that we can't provision a certificate for.
def can_provision_for_domain(domain):
# Let's Encrypt doesn't yet support IDNA domains.
# We store domains in IDNA (ASCII). To see if this domain is IDNA,
# we'll see if its IDNA-decoded form is different.
if idna.decode(domain.encode("ascii")) != domain:
problems[domain] = "Let's Encrypt does not yet support provisioning certificates for internationalized domains."
return False
# Does the domain resolve to this machine in public DNS? If not,
# we can't do domain control validation. For IPv6 is configured,
# make sure both IPv4 and IPv6 are correct because we don't know
# how Let's Encrypt will connect.
import dns.resolver
for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]:
if not value: continue # IPv6 is not configured
try:
# Must make the qname absolute to prevent a fall-back lookup with a
# search domain appended, by adding a period to the end.
response = dns.resolver.query(domain + ".", rtype)
except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
problems[domain] = "DNS isn't configured properly for this domain: DNS resolution failed (%s: %s)." % (rtype, str(e) or repr(e)) # NoAnswer's str is empty
return False
except Exception as e:
problems[domain] = "DNS isn't configured properly for this domain: DNS lookup had an error: %s." % str(e)
return False
if len(response) != 1 or str(response[0]) != value:
problems[domain] = "Domain control validation cannot be performed for this domain because DNS points the domain to another machine (%s %s)." % (rtype, ", ".join(str(r) for r in response))
return False
return True
domains = set(filter(can_provision_for_domain, domains))
# If there are any domains we definitely will provision for, add in
# additional domains to do at this time.
if len(domains) > 0:
domains |= set(filter(can_provision_for_domain, domains_if_any))
return (domains, problems)
def buy_ssl_certificate(api_key, domain, command, env):
if domain != env['PRIMARY_HOSTNAME'] \
and domain not in get_web_domains(env):
raise ValueError(
"Domain is not %s or a domain we're serving a website for." %
env['PRIMARY_HOSTNAME'])
# Initialize.
gandi = xmlrpc.client.ServerProxy('https://rpc.gandi.net/xmlrpc/')
try:
existing_certs = gandi.cert.list(api_key)
except Exception as e:
if "Invalid API key" in str(e):
print(
"Invalid API key. Check that you copied the API Key correctly from https://www.gandi.net/admin/api_key."
)
sys.exit(1)
else:
raise
# Where is the SSL cert stored?
ssl_key, ssl_certificate, ssl_csr_path = get_domain_ssl_files(domain, env)
# Have we already created a cert for this domain?
for cert in existing_certs:
if cert['cn'] == domain:
break
else:
# No existing cert found. Purchase one.
if command != 'purchase':
print(
"No certificate or order found yet. If you haven't yet purchased a certificate, run ths script again with the 'purchase' command. Otherwise wait a moment and try again."
)
sys.exit(1)
else:
# Start an order for a single standard SSL certificate.
# Use DNS validation. Web-based validation won't work because they
# require a file on HTTP but not HTTPS w/o redirects and we don't
# serve anything plainly over HTTP. Email might be another way but
# DNS is easier to automate.
op = gandi.cert.create(
api_key,
{
"csr": open(ssl_csr_path).read(),
"dcv_method": "dns",
"duration": 1, # year?
"package": "cert_std_1_0_0",
})
print("An SSL certificate has been ordered.")
print()
print(op)
print()
print(
"In a moment please run this script again with the 'setup' command."
)
if cert['status'] == 'pending':
# Get the information we need to update our DNS with a code so that
# Gandi can verify that we own the domain.
dcv = gandi.cert.get_dcv_params(
api_key,
{
"csr": open(ssl_csr_path).read(),
"cert_id": cert['id'],
"dcv_method": "dns",
"duration": 1, # year?
"package": "cert_std_1_0_0",
})
if dcv["dcv_method"] != "dns":
raise Exception(
"Certificate ordered with an unknown validation method.")
# Update our DNS data.
dns_config = env['STORAGE_ROOT'] + '/dns/custom.yaml'
if os.path.exists(dns_config):
dns_records = rtyaml.load(open(dns_config))
else:
dns_records = {}
qname = dcv['md5'] + '.' + domain
value = dcv['sha1'] + '.comodoca.com.'
dns_records[qname] = {"CNAME": value}
with open(dns_config, 'w') as f:
f.write(rtyaml.dump(dns_records))
shell('check_call', ['tools/dns_update'])
# Okay, done with this step.
print("DNS has been updated. Gandi will check within 60 minutes.")
print()
print(
"See https://www.gandi.net/admin/ssl/%d/details for the status of this order."
% cert['id'])
elif cert['status'] == 'valid':
# The certificate is ready.
# Check before we overwrite something we shouldn't.
if os.path.exists(ssl_certificate):
cert_status = check_certificate(None, ssl_certificate, None)
if cert_status != "SELF-SIGNED":
print(
"Please back up and delete the file %s so I can save your new certificate."
% ssl_certificate)
sys.exit(1)
# Form the certificate.
# The certificate comes as a long base64-encoded string. Break in
# into lines in the usual way.
pem = "-----BEGIN CERTIFICATE-----\n"
pem += "\n".join(chunk for chunk in re.split(r"(.{64})", cert['cert'])
if chunk != "")
pem += "\n-----END CERTIFICATE-----\n\n"
# Append intermediary certificates.
pem += urllib.request.urlopen(
"https://www.gandi.net/static/CAs/GandiStandardSSLCA.pem").read(
).decode("ascii")
# Write out.
with open(ssl_certificate, "w") as f:
f.write(pem)
print(
"The certificate has been installed in %s. Restarting services..."
% ssl_certificate)
# Restart dovecot and if this is for PRIMARY_HOSTNAME.
if domain == env['PRIMARY_HOSTNAME']:
shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
# Restart nginx in all cases.
shell('check_call', ["/usr/sbin/service", "nginx", "restart"])
else:
print(
"The certificate has an unknown status. Please check https://www.gandi.net/admin/ssl/%d/details for the status of this order."
% cert['id'])
# is taking connections, with a maximum timeout.
import socket, time
start = time.perf_counter()
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout / 3)
try:
s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port))
return True
except OSError:
if time.perf_counter() > start + timeout:
return False
time.sleep(min(timeout / 4, 1))
def fix_boto():
# Google Compute Engine instances install some Python-2-only boto plugins that
# conflict with boto running under Python 3. Disable boto's default configuration
# file prior to importing boto so that GCE's plugin is not loaded:
import os
os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
if __name__ == "__main__":
from web_update import get_web_domains
env = load_environment()
domains = get_web_domains(env)
for domain in domains:
print(domain)
def provision_certificates(env,
limit_domains,
domain_to_be_renewed=None,
new_key=False):
# What domains should we provision certificates for? And what
# errors prevent provisioning for other domains.
ret = []
is_tlsa_update_required = False
if new_key:
from web_update import get_web_domains
domains = get_web_domains(env)
elif domain_to_be_renewed:
existing_certs = get_ssl_certificates(env)
existing_cert = get_domain_ssl_files(domain_to_be_renewed,
existing_certs,
env,
use_main_cert=False,
allow_missing_cert=True)
domains, primary_domain = get_certificate_domains(
load_pem(load_cert_chain(existing_cert["certificate"])[0]))
else:
# domains = domains for which a certificate can be provisioned
# domains_cant_provision = domains for which a certificate can't be provisioned and the reason
domains, domains_cant_provision = get_certificates_to_provision(
env, limit_domains=limit_domains)
# Build a list of what happened on each domain or domain-set.
for domain, error in domains_cant_provision.items():
ret.append({
"domains": [domain],
"log": [error],
"result": "skipped",
})
# Break into groups by DNS zone: Group every domain with its parent domain, if
# its parent domain is in the list of domains to request a certificate for.
# Start with the zones so that if the zone doesn't need a certificate itself,
# its children will still be grouped together. Sort the provision domains to
# put parents ahead of children.
# Since Let's Encrypt requests are limited to 100 domains at a time,
# we'll create a list of lists of domains where the inner lists have
# at most 100 items. By sorting we also get the DNS zone domain as the first
# entry in each list (unless we overflow beyond 100) which ends up as the
# primary domain listed in each certificate.
from dns_update import get_dns_zones
certs = {}
for zone, zonefile in get_dns_zones(env):
certs[zone] = [[]]
for domain in sort_domains(domains, env):
# Does the domain end with any domain we've seen so far.
for parent in certs.keys():
if domain.endswith("." + parent):
# Add this to the parent's list of domains.
# Start a new group if the list already has
# 100 items.
if len(certs[parent][-1]) == 100:
certs[parent].append([])
certs[parent][-1].append(domain)
break
else:
# This domain is not a child of any domain we've seen yet, so
# start a new group. This shouldn't happen since every zone
# was already added.
certs[domain] = [[domain]]
# Flatten to a list of lists of domains (from a mapping). Remove empty
# lists (zones with no domains that need certs).
certs = sum(certs.values(), [])
certs = [_ for _ in certs if len(_) > 0]
# Prepare to provision.
# Where should we put our Let's Encrypt account info and state cache.
account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt')
if not os.path.exists(account_path):
os.mkdir(account_path)
# Provision certificates.
for domain_list in certs:
ret.append({
"domains": domain_list,
"log": [],
})
try:
# Create a CSR file for our master private key so that certbot
# uses our private key.
key_file = os.path.join(env['STORAGE_ROOT'], 'ssl',
'ssl_private_key.pem')
if new_key:
key_file = os.path.join(env['STORAGE_ROOT'], 'ssl',
'next_ssl_private_key.pem')
with tempfile.NamedTemporaryFile() as csr_file:
# We could use openssl, but certbot requires
# that the CN domain and SAN domains match
# the domain list passed to certbot, and adding
# SAN domains openssl req is ridiculously complicated.
# subprocess.check_output([
# "openssl", "req", "-new",
# "-key", key_file,
# "-out", csr_file.name,
# "-subj", "/CN=" + domain_list[0],
# "-sha256" ])
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.primitives import hashes
from cryptography.x509.oid import NameOID
builder = x509.CertificateSigningRequestBuilder()
builder = builder.subject_name(
x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0])
]))
builder = builder.add_extension(x509.BasicConstraints(
ca=False, path_length=None),
critical=True)
builder = builder.add_extension(x509.SubjectAlternativeName(
[x509.DNSName(d) for d in domain_list]),
critical=False)
request = builder.sign(load_pem(load_cert_chain(key_file)[0]),
hashes.SHA256(), default_backend())
with open(csr_file.name, "wb") as f:
f.write(request.public_bytes(Encoding.PEM))
# Provision, writing to a temporary file.
webroot = os.path.join(account_path, 'webroot')
os.makedirs(webroot, exist_ok=True)
with tempfile.TemporaryDirectory() as d:
cert_file = os.path.join(d, 'cert_and_chain.pem')
print("Provisioning TLS certificates for " +
", ".join(domain_list) + ".")
certbotret = subprocess.check_output(
[
"certbot",
"certonly",
#"-v", # just enough to see ACME errors
"--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup
"-d",
",".join(domain_list), # first will be main domain
"--csr",
csr_file.
name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually
"--cert-path",
os.path.join(d,
'cert'), # we only use the full chain
"--chain-path",
os.path.join(
d, 'chain'), # we only use the full chain
"--fullchain-path",
cert_file,
"--webroot",
"--webroot-path",
webroot,
"--config-dir",
account_path,
#"--staging",
],
stderr=subprocess.STDOUT).decode("utf8")
install_cert_copy_file(cert_file, env)
ret[-1]["log"].append(certbotret)
ret[-1]["result"] = "installed"
if new_key and env['PRIMARY_HOSTNAME'] in domains:
is_tlsa_update_required = True
except subprocess.CalledProcessError as e:
ret[-1]["log"].append(e.output.decode("utf8"))
ret[-1]["result"] = "error"
except Exception as e:
ret[-1]["log"].append(str(e))
ret[-1]["result"] = "error"
# Run post-install steps.
ret.extend(
post_install_func(env,
is_tlsa_update_required=is_tlsa_update_required))
# Return what happened with each certificate request.
return ret
# Gets the version of PHP installed in the system.
return shell("check_output", ["/usr/bin/php", "-v"])[4:7]
os_codes = {None, "Debian10", "Ubuntu2004"}
def get_os_code():
# Massive mess incoming
dist = shell("check_output", ["/usr/bin/lsb_release", "-is"]).strip()
version = shell("check_output", ["/usr/bin/lsb_release", "-rs"]).strip()
if dist == "Debian":
if version == "10":
return "Debian10"
elif version == "11":
return "Debian11"
elif dist == "Ubuntu":
if version == "20.04":
return "Ubuntu2004"
return None
if __name__ == "__main__":
from web_update import get_web_domains
env = load_environment()
domains = get_web_domains(env)
for domain in domains:
print(domain)
if stat.st_ino in seen:
continue
seen.add(stat.st_ino)
total_size += stat.st_size
return total_size
def wait_for_service(port, public, env, timeout):
# Block until a service on a given port (bound privately or publicly)
# is taking connections, with a maximum timeout.
import socket, time
start = time.perf_counter()
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout/3)
try:
s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port))
return True
except OSError:
if time.perf_counter() > start+timeout:
return False
time.sleep(min(timeout/4, 1))
if __name__ == "__main__":
from dns_update import get_dns_domains
from web_update import get_web_domains, get_default_www_redirects
env = load_environment()
domains = get_dns_domains(env) | set(get_web_domains(env) + get_default_www_redirects(env))
domains = sort_domains(domains, env)
for domain in domains:
print(domain)