def build_recommended_dns(env): ret = [] domains = get_dns_domains(env) zonefiles = get_dns_zones(env) additional_records = list(get_custom_dns_config(env)) from web_update import get_default_www_redirects www_redirect_domains = get_default_www_redirects(env) for domain, zonefile in zonefiles: records = build_zone(domain, domains, additional_records, www_redirect_domains, env) # remove records that we don't dislay records = [r for r in records if r[3] is not False] # put Required at the top, then Recommended, then everythiing else records.sort(key = lambda r : 0 if r[3].startswith("Required.") else (1 if r[3].startswith("Recommended.") else 2)) # expand qnames for i in range(len(records)): if records[i][0] == None: qname = domain else: qname = records[i][0] + "." + domain records[i] = { "qname": qname, "rtype": records[i][1], "value": records[i][2], "explanation": records[i][3], } # return ret.append((domain, records)) return ret
def build_recommended_dns(env): ret = [] domains = get_dns_domains(env) zonefiles = get_dns_zones(env) additional_records = list(get_custom_dns_config(env)) from web_update import get_default_www_redirects www_redirect_domains = get_default_www_redirects(env) for domain, zonefile in zonefiles: records = build_zone(domain, domains, additional_records, www_redirect_domains, env) # remove records that we don't dislay records = [r for r in records if r[3] is not False] # put Required at the top, then Recommended, then everythiing else records.sort(key=lambda r: 0 if r[3].startswith("Required.") else (1 if r[3].startswith("Recommended.") else 2)) # expand qnames for i in range(len(records)): if records[i][0] == None: qname = domain else: qname = records[i][0] + "." + domain records[i] = { "qname": qname, "rtype": records[i][1], "value": records[i][2], "explanation": records[i][3], } # return ret.append((domain, records)) return ret
def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env) + get_default_www_redirects(env)) domains_to_check = mail_domains | dns_domains | web_domains # Serial version: # for domain in sort_domains(domains_to_check, env): # run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains) # Parallelize the checks across a worker pool. args = ( (domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains) for domain in domains_to_check ) ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1) ret = dict(ret) # (domain, output) => { domain: output } for domain in sort_domains(ret, env): ret[domain].playback(output)
def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains we handle mail for. mail_domains = get_mail_domains(env) # Get the list of domains we serve DNS zones for (i.e. does not include subdomains). dns_zonefiles = dict(get_dns_zones(env)) dns_domains = set(dns_zonefiles) # Get the list of domains we serve HTTPS for. web_domains = set(get_web_domains(env) + get_default_www_redirects(env)) domains_to_check = mail_domains | dns_domains | web_domains # Get the list of domains that we don't serve web for because of a custom CNAME/A record. domains_with_a_records = get_domains_with_a_records(env) ssl_certificates = get_ssl_certificates(env) # Serial version: #for domain in sort_domains(domains_to_check, env): # run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains) # Parallelize the checks across a worker pool. args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates) for domain in domains_to_check) ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1) ret = dict(ret) # (domain, output) => { domain: output } for domain in sort_domains(ret, env): ret[domain].playback(output)
def do_dns_update(env, force=False): # What domains (and their zone filenames) should we build? domains = get_dns_domains(env) zonefiles = get_dns_zones(env) # Custom records to add to zones. additional_records = list(get_custom_dns_config(env)) from web_update import get_default_www_redirects www_redirect_domains = get_default_www_redirects(env) # Write zone files. os.makedirs('/etc/nsd/zones', exist_ok=True) updated_domains = [] for i, (domain, zonefile) in enumerate(zonefiles): # Build the records to put in the zone. records = build_zone(domain, domains, additional_records, www_redirect_domains, env) # See if the zone has changed, and if so update the serial number # and write the zone file. if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env, force): # Zone was not updated. There were no changes. continue # If this is a .justtesting.email domain, then post the update. try: justtestingdotemail(domain, records) except: # Hmm. Might be a network issue. If we stop now, will we end # up in an inconsistent state? Let's just continue. pass # Mark that we just updated this domain. updated_domains.append(domain) # Sign the zone. # # Every time we sign the zone we get a new result, which means # we can't sign a zone without bumping the zone's serial number. # Thus we only sign a zone if write_nsd_zone returned True # indicating the zone changed, and thus it got a new serial number. # write_nsd_zone is smart enough to check if a zone's signature # is nearing expiration and if so it'll bump the serial number # and return True so we get a chance to re-sign it. sign_zone(domain, zonefile, env) # Now that all zones are signed (some might not have changed and so didn't # just get signed now, but were before) update the zone filename so nsd.conf # uses the signed file. for i in range(len(zonefiles)): zonefiles[i][1] += ".signed" # Write the main nsd.conf file. if write_nsd_conf(zonefiles, additional_records, env): # Make sure updated_domains contains *something* if we wrote an updated # nsd.conf so that we know to restart nsd. if len(updated_domains) == 0: updated_domains.append("DNS configuration") # Kick nsd if anything changed. if len(updated_domains) > 0: shell('check_call', ["/usr/sbin/service", "nsd", "restart"]) # Write the OpenDKIM configuration tables. if write_opendkim_tables(domains, env): # Settings changed. Kick opendkim. shell('check_call', ["/usr/sbin/service", "opendkim", "restart"]) if len(updated_domains) == 0: # If this is the only thing that changed? updated_domains.append("OpenDKIM configuration") if len(updated_domains) == 0: # if nothing was updated (except maybe OpenDKIM's files), don't show any output return "" else: return "updated DNS: " + ",".join(updated_domains) + "\n"
def do_dns_update(env, force=False): # What domains (and their zone filenames) should we build? domains = get_dns_domains(env) zonefiles = get_dns_zones(env) # Custom records to add to zones. additional_records = list(get_custom_dns_config(env)) from web_update import get_default_www_redirects www_redirect_domains = get_default_www_redirects(env) # Write zone files. os.makedirs('/etc/nsd/zones', exist_ok=True) updated_domains = [] for i, (domain, zonefile) in enumerate(zonefiles): # Build the records to put in the zone. records = build_zone(domain, domains, additional_records, www_redirect_domains, env) # See if the zone has changed, and if so update the serial number # and write the zone file. if not write_nsd_zone(domain, "/etc/nsd/zones/" + zonefile, records, env, force): # Zone was not updated. There were no changes. continue # If this is a .justtesting.email domain, then post the update. try: justtestingdotemail(domain, records) except: # Hmm. Might be a network issue. If we stop now, will we end # up in an inconsistent state? Let's just continue. pass # Mark that we just updated this domain. updated_domains.append(domain) # Sign the zone. # # Every time we sign the zone we get a new result, which means # we can't sign a zone without bumping the zone's serial number. # Thus we only sign a zone if write_nsd_zone returned True # indicating the zone changed, and thus it got a new serial number. # write_nsd_zone is smart enough to check if a zone's signature # is nearing expiration and if so it'll bump the serial number # and return True so we get a chance to re-sign it. sign_zone(domain, zonefile, env) # Now that all zones are signed (some might not have changed and so didn't # just get signed now, but were before) update the zone filename so nsd.conf # uses the signed file. for i in range(len(zonefiles)): zonefiles[i][1] += ".signed" # Write the main nsd.conf file. if write_nsd_conf(zonefiles, additional_records, env): # Make sure updated_domains contains *something* if we wrote an updated # nsd.conf so that we know to restart nsd. if len(updated_domains) == 0: updated_domains.append("DNS configuration") # Kick nsd if anything changed. if len(updated_domains) > 0: shell('check_call', ["/usr/sbin/service", "nsd", "restart"]) # Write the OpenDKIM configuration tables. if write_opendkim_tables(domains, env): # Settings changed. Kick opendkim. shell('check_call', ["/usr/sbin/service", "opendkim", "restart"]) if len(updated_domains) == 0: # If this is the only thing that changed? updated_domains.append("OpenDKIM configuration") if len(updated_domains) == 0: # if nothing was updated (except maybe OpenDKIM's files), don't show any output return "" else: return "updated DNS: " + ",".join(updated_domains) + "\n"
total_size += stat.st_size return total_size def wait_for_service(port, public, env, timeout): # Block until a service on a given port (bound privately or publicly) # is taking connections, with a maximum timeout. import socket, time start = time.perf_counter() while True: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout / 3) try: s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port)) return True except OSError: if time.perf_counter() > start + timeout: return False time.sleep(min(timeout / 4, 1)) if __name__ == "__main__": from dns_update import get_dns_domains from web_update import get_web_domains, get_default_www_redirects env = load_environment() domains = get_dns_domains(env) | set( get_web_domains(env) + get_default_www_redirects(env)) domains = sort_domains(domains, env) for domain in domains: print(domain)
if stat.st_ino in seen: continue seen.add(stat.st_ino) total_size += stat.st_size return total_size def wait_for_service(port, public, env, timeout): # Block until a service on a given port (bound privately or publicly) # is taking connections, with a maximum timeout. import socket, time start = time.perf_counter() while True: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout/3) try: s.connect(("127.0.0.1" if not public else env['PUBLIC_IP'], port)) return True except OSError: if time.perf_counter() > start+timeout: return False time.sleep(min(timeout/4, 1)) if __name__ == "__main__": from dns_update import get_dns_domains from web_update import get_web_domains, get_default_www_redirects env = load_environment() domains = get_dns_domains(env) | set(get_web_domains(env) + get_default_www_redirects(env)) domains = sort_domains(domains, env) for domain in domains: print(domain)