def consolidate_cert(cert_meta: Certificate):
"""
Consolidate cert targets of one cert meta.
This means cert and key files of instance in state "deployed"
are redistributed.
@param cert_meta: Cert meta
@type cert_meta: cert.Certificate
@rtype: None
@exceptions:
"""
deployed_ci = None
inst_dict = cert_meta.active_instances
sld('consolidate_cert: inst_list = {}'.format(inst_dict))
if not inst_dict: return
for state, ci in inst_dict.items():
if state == CertState('deployed'):
deployed_ci = ci
break
if not deployed_ci:
sli('consolidate_cert: No instance of {} in state "deployed"'.format(
cert_meta.name))
return
try:
deployCerts({cert_meta.name: cert_meta},
cert_instances=(deployed_ci, ),
allowed_states=(CertState('deployed'), ))
except MyException:
pass
return
def expire(cm, ci):
if opts.check_only:
sld('Would expire {}.'.format(ci.row_id))
return
sli('State transition from {} to EXPIRED of {}:{}'.
format(ci.state, cm.name, ci.row_id))
ci.state = cert.CertState('expired')
cm.save_instance(ci)
def archive(cm, ci):
if opts.check_only:
sld('Would archive {}.'.format(ci.row_id))
return
sli('State transition from {} to ARCHIVED of {}:{}'.
format(ci.state, cm.name, ci.row_id))
ci.state = cert.CertState('archived')
cm.save_instance(ci)
def issue(cm: cert.Certificate) -> Optional[cert.CertInstance]:
"""
If cert type is 'LE', issue a Letsencrypt cert
:param cm: cert meta
:return: ci of new cert or None
"""
if cm.cert_type == cert.CertType('local'):
return None
if opts.check_only:
sld('Would issue {}.'.format(cm.name))
return
if not cm.disabled:
sli('Requesting issue from LE for {}'.format(cm.name))
return issue_LE_cert(cm)
def distribute(cm: cert.Certificate, ci: cert.CertInstance, state: cert.CertState):
if opts.check_only:
sld('Would distribute {}.'.format(ci.row_id))
return
sli('Distributing {}:{}'.
format(cm.name, ci.row_id))
cm_dict = {cm.name: cm}
try:
deployCerts(cm_dict, (ci,), allowed_states=(state,))
except Exception:
sln('Skipping distribution of cert {} because {} [{}]'.format(
cm.name,
sys.exc_info()[0].__name__,
str(sys.exc_info()[1])))
def delete_TLSA(cert_meta: Certificate) -> None:
"""
Delete all TLSA RRs per fqdn of all altnames either in flatfile (make include file empty) or in dyn dns
:param cert_meta:
:return:
"""
if Pathes.tlsa_dns_master == '': # DNS master on local host
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for (zone, fqdn) in cert_meta.zone_and_FQDN_from_altnames():
filename = fqdn + '.tlsa'
dest = str(Pathes.zone_file_root / zone / filename)
#just open for write without writing, which makes file empty
with open(dest, 'w') as fd:
sli('Truncating {}'.format(dest))
updateZoneCache(zone)
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
zones = {}
for (zone, fqdn) in cert_meta.zone_and_FQDN_from_altnames():
if zone in zones:
if fqdn not in zones[zone]: zones[zone].append(fqdn)
else:
zones[zone] = [fqdn]
for zone in zones:
the_update = ddns_update(zone)
for fqdn in zones[zone]:
for prefix in cert_meta.tlsaprefixes.keys():
tag = str(prefix.format(fqdn)).split(maxsplit=1)[0]
sld('Deleting TLSA with tag {} an fqdn {} in zone {}'.
format(tag, fqdn, zone))
the_update.delete(tag)
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
rc = response.rcode()
if rc != 0:
sle('DNS update failed for zone {} with rcode: {}:\n{}'.
format(zone, response.rcode.to_text(rc),
response.rcode))
raise Exception(
'DNS update failed for zone {} with rcode: {}'.format(
zone, response.rcode.to_text(rc)))
def prepublish(cm: cert.Certificate, active_ci: cert.CertInstance, new_ci: cert.CertInstance) -> None:
"""
Prepublish cert hashes per TLSA RRs in DNS
:param cm: Our cert meta data instance
:param active_ci: CertInstance currently in use
:param new_ci: CertInstance just created but not yet deployed
:return:
"""
if opts.check_only:
sld('Would prepublish {} {}.'.format(active_ci.row_id, new_ci.row_id))
return
# collect hashes for all certs in all algos
hashes = tuple(cm.TLSA_hashes(active_ci).values()) + tuple(cm.TLSA_hashes(new_ci).values())
sli('Prepublishing {}:{}:{}'.
format(cm.name, active_ci.row_id, new_ci.row_id))
distribute_tlsa_rrs(cm, hashes)
new_ci.state = cert.CertState('prepublished')
cm.save_instance(new_ci)
def delete_challenge_responses_in_dns(zones):
"""
Delete the challenge response in dns, created by
create_challenge_responses_in_dns()
@param zones: dict of zones, where each zone has a list of fqdns
as values
@type zones: dict()
@rtype: None
@exceptions Can''t parse ddns key or
DNS update failed for zone {} with rcode: {}
"""
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for zone in zones.keys():
dest = str(Pathes.zone_file_root / zone /
Pathes.zone_file_include_name)
with open(dest, 'w') as file:
file.writelines(('', ))
updateZoneCache(zone)
updateSOAofUpdatedZones()
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
txt_datatape = rdatatype.from_text('TXT')
for zone in zones.keys():
the_update = ddns_update(zone)
for fqdn in zones[zone]:
the_update.delete('_acme-challenge.{}.'.format(fqdn),
txt_datatape)
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
sld('DNS update delete/add returned response: {}'.format(response))
rc = response.rcode()
if rc != 0:
sle('DNS update failed for zone {} with rcode: {}:\n{}'.format(
zone, rc.to_text(rc), rc))
raise Exception(
'DNS update failed for zone {} with rcode: {}'.format(
zone, rc.to_text(rc)))
def ssh_connection(dest_host):
"""
Open a ssh connection.
@param dest_host: fqdn of target host
@type dest_host: string
@rtype: paramiko.SSHClient (connected transport)
@exceptions:
If unable to connect
"""
client = SSHClient()
client.load_host_keys(expanduser('~/.ssh/known_hosts'))
sld('Connecting to {}'.format(dest_host))
try:
client.connect(dest_host, username=Misc.SSH_CLIENT_USER_NAME)
except Exception:
sln('Failed to connect to host {}, because {} [{}]'.format(
dest_host,
sys.exc_info()[0].__name__, str(sys.exc_info()[1])))
raise
else:
sld('Connected to host {}'.format(dest_host))
return client
def create_challenge_responses_in_dns(zones, fqdn_challenges):
"""
Create the expected challenge response in dns
@param zones: dict of zones, where each zone has a list of fqdns
as values
@type zones: dict()
@param fqdn_challenges: dict of zones, containing challenge response
(key) of zone
@type fqdn_challenges: dict()
@rtype: None
@exceptions Can''t parse ddns key or
DNS update failed for zone {} with rcode: {}
"""
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for zone in zones.keys():
dest = str(Pathes.zone_file_root / zone /
Pathes.zone_file_include_name)
lines = []
for fqdn in zones[zone]:
sld('fqdn: {}'.format(fqdn))
lines.append(
str('_acme-challenge.{}. IN TXT \"{}\"\n'.format(
fqdn, fqdn_challenges[fqdn].key)))
sli('Writing RRs: {}'.format(lines))
with open(dest, 'w') as file:
file.writelines(lines)
##os.chmod(file.fileno(), Pathes.zone_tlsa_inc_mode)
##os.chown(file.fileno(), pathes.zone_tlsa_inc_uid, pathes.zone_tlsa_inc_gid)
updateZoneCache(zone)
updateSOAofUpdatedZones()
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
txt_datatape = rdatatype.from_text('TXT')
for zone in zones.keys():
the_update = ddns_update(zone)
for fqdn in zones[zone]:
the_update.delete('_acme-challenge.{}.'.format(fqdn),
txt_datatape)
the_update.add('_acme-challenge.{}.'.format(fqdn), 60,
txt_datatape, fqdn_challenges[fqdn].key)
sld('DNS update of RR: {}'.format(
'_acme-challenge.{}. 60 TXT \"{}\"'.format(
fqdn, fqdn_challenges[fqdn].key)))
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
sld('DNS update delete/add returned response: {}'.format(response))
rc = response.rcode()
if rc != 0:
sle('DNS delete failed for zone {} with rcode: {}:\n{}'.format(
zone, rc.to_text(rc), rc))
raise Exception(
'DNS update failed for zone {} with rcode: {}'.format(
zone, rc.to_text(rc)))
def deployCerts(
cert_metas: Dict[str, Certificate],
cert_instances: Optional[Tuple[CertInstance]] = None,
allowed_states: Tuple[CertState] = (CertState('issued'), )
) -> bool:
"""
Deploy a list of (certificates. keys and TLSA RRs, using paramiko/sftp) and dyn DNS (or zone files).
Restart service at target host and reload nameserver (if using zone files).
:param cert_metas: Dict of cert metas, telling which certs to deploy, key is cert subject name
:param cert_instances: Optional list of CertInstance instances
:param allowed_states: States describing CertInstance states to act on
:return: True if successfully deployed certs
"""
error_found = False
limit_hosts = False
opts = get_options()
only_host = []
if opts.only_host: only_host = opts.only_host
if len(only_host) > 0: limit_hosts = True
skip_host = []
if opts.skip_host: skip_host = opts.skip_host
sld('limit_hosts={}, only_host={}, skip_host={}'.format(
limit_hosts, only_host, skip_host))
for cert_meta in cert_metas.values():
if len(cert_meta.disthosts) == 0: continue
the_instances = []
hashes = []
##FIXME## highly speculative!
insts = cert_instances if cert_instances else [
y for (x, y) in cert_meta.active_instances.items()
]
for ci in insts:
if ci.state in allowed_states:
the_instances.append(ci)
if len(the_instances) == 0:
etxt = 'No valid cerificate for {} in DB - create it first\n' \
'States being considered are {}'. \
format(cert_meta.name, [state for state in allowed_states])
sli(etxt)
if cert_instances: # let caller handle this error, if we have explicit inst ids
raise MyException(etxt)
else:
continue
# more than 1 member of the_instances only expected with cert.instance(i).encryption_algo == 'both'
for ci in the_instances:
state = ci.state
cacert_text = cert_meta.cacert_PEM(ci)
host_omitted = False
cksd = ci.the_cert_key_stores
for encryption_algo in cksd.keys():
cks = cksd[encryption_algo]
cert_text = cks.cert
key_text = cks.key
TLSA_text = cks.hash
hashes.append(TLSA_text)
for fqdn, dh in cert_meta.disthosts.items():
if fqdn in skip_host:
host_omitted = True
continue
if limit_hosts and (fqdn not in only_host):
host_omitted = True
continue
dest_path = PurePath('/')
sld('{}: {}'.format(cert_meta.name, fqdn))
for jail in (dh['jails'].keys()
or ('', )): # jail is empty if no jails
if '/' in jail:
sle('"/" in jail name "{}" not allowed with subject {}.'
.format(jail, cert_meta.name))
error_found = True
return False
jailroot = dh[
'jailroot'] if jail != '' else '' # may also be empty
dest_path = PurePath('/', jailroot, jail)
sld('{}: {}: {}'.format(cert_meta.name, fqdn,
dest_path))
the_jail = dh['jails'][jail]
if len(the_jail['places']) == 0:
sle('{} subject has no place attribute.'.format(
cert_meta.name))
error_found = True
return False
for place in the_jail['places'].values():
sld('Handling jail "{}" and place {}'.format(
jail, place.name))
fd_key = StringIO(key_text)
fd_cert = StringIO(cert_text)
key_file_name = key_name(cert_meta.name,
cert_meta.subject_type,
encryption_algo)
cert_file_name = cert_name(cert_meta.name,
cert_meta.subject_type,
encryption_algo)
pcp = place.cert_path
if '{}' in pcp: # we have a home directory named like the subject
pcp = pcp.format(cert_meta.name)
# make sure pcb does not start with '/', which would ignore dest_path:
if PurePath(pcp).is_absolute():
dest_dir = PurePath(
dest_path,
PurePath(pcp).relative_to('/'))
else:
dest_dir = PurePath(dest_path, PurePath(pcp))
sld('Handling fqdn {} and dest_dir "{}" in deployCerts'
.format(fqdn, dest_dir))
try:
if place.key_path:
key_dest_dir = PurePath(
dest_path, place.key_path)
distribute_cert(fd_key, fqdn, key_dest_dir,
key_file_name, place, None)
elif place.cert_file_type == 'separate':
distribute_cert(fd_key, fqdn, dest_dir,
key_file_name, place, None)
if cert_meta.cert_type == 'LE':
chain_file_name = cert_cacert_chain_name(
cert_meta.name,
cert_meta.subject_type,
encryption_algo)
fd_chain = StringIO(cert_text +
cacert_text)
distribute_cert(
fd_chain, fqdn, dest_dir,
chain_file_name, place, jail)
elif place.cert_file_type == 'combine key':
cert_file_name = key_cert_name(
cert_meta.name, cert_meta.subject_type,
encryption_algo)
fd_cert = StringIO(key_text + cert_text)
if cert_meta.cert_type == 'LE':
chain_file_name = key_cert_cacert_chain_name(
cert_meta.name,
cert_meta.subject_type,
encryption_algo)
fd_chain = StringIO(key_text +
cert_text +
cacert_text)
distribute_cert(
fd_chain, fqdn, dest_dir,
chain_file_name, place, jail)
elif place.cert_file_type == 'combine both':
cert_file_name = key_cert_cacert_name(
cert_meta.name, cert_meta.subject_type,
encryption_algo)
fd_cert = StringIO(key_text + cert_text +
cacert_text)
elif place.cert_file_type == 'combine cacert':
cert_file_name = cert_cacert_name(
cert_meta.name, cert_meta.subject_type,
encryption_algo)
fd_cert = StringIO(cert_text + cacert_text)
distribute_cert(fd_key, fqdn, dest_dir,
key_file_name, place, None)
# this may be redundant in case of LE, where the cert was in chained file
distribute_cert(fd_cert, fqdn, dest_dir,
cert_file_name, place, jail)
except IOError: # distribute_cert may error out
error_found = True
break # no cert - no TLSA
sli('')
if opts.sync_disk: # skip TLSA stuff if doing consolidate
continue
if not opts.no_TLSA:
distribute_tlsa_rrs(cert_meta, hashes)
if not host_omitted and not cert_meta.subject_type == 'CA':
ci.state = CertState('deployed')
cert_meta.save_instance(ci)
else:
sln('State of cert {} not promoted to DEPLOYED, '
'because hosts where limited or skipped'.format(
cert_meta.name))
# clear mail-sent-time if local cert.
if cert_meta.cert_type == CertType('local'):
cert_meta.update_authorized_until(None)
updateSOAofUpdatedZones()
return not error_found
def distribute_cert(fd, dest_host, dest_dir, file_name, place, jail):
"""
Distribute cert and key to a host, jail (if any) and place.
Optional reload the service.
If global opts.extract set, instead of distributing to a host,
certificat and key are written to the local work directory.
@param fd: file descriptor of memory stream
@type fd: io.StringIO
@param dest_host: fqdn of target host
@type dest_host: string
@param dest_dir: target directory
@type dest_dir: string
@param file_name: file name of key or cert file
@type file_name: string
@param place: place with details about setting mode and uid/gid of file
@type place: serverPKI.cert.Place instance
@param jail: name of jail for service to reload
@type jail: string or None
@rtype: not yet any
@exceptions: IOError
"""
sld('Handling dest_host {} and dest_dir "{}" in distribute_cert'.format(
dest_host, dest_dir))
with ssh_connection(dest_host) as client:
with client.open_sftp() as sftp:
try:
sftp.chdir(str(dest_dir))
except IOError:
sln('{}:{} does not exist - creating\n\t{}'.format(
dest_host, dest_dir,
sys.exc_info()[0].__name__))
try:
sftp.mkdir(str(dest_dir))
except IOError:
sle('Cant create {}:{}: Missing parent?\n\t{}'.format(
dest_host, dest_dir,
sys.exc_info()[0].__name__, str(sys.exc_info()[1])))
raise
sftp.chdir(str(dest_dir))
sli('{} => {}:{}'.format(file_name, dest_host, dest_dir))
fat = sftp.putfo(fd, file_name, confirm=True)
sld('size={}, uid={}, gid={}, mtime={}'.format(
fat.st_size, fat.st_uid, fat.st_gid, fat.st_mtime))
if 'key' in file_name:
sld('Setting mode to 0o400 of {}:{}/{}'.format(
dest_host, dest_dir, file_name))
mode = 0o400
if place.mode:
mode = place.mode
sld('Setting mode of key at target to {}'.format(
oct(place.mode)))
sftp.chmod(file_name, mode)
if place.pgLink:
try:
sftp.unlink('postgresql.key')
except IOError:
pass # none exists: ignore
sftp.symlink(file_name, 'postgresql.key')
sld('{} => postgresql.key'.format(file_name))
if 'key' in file_name or place.chownBoth:
uid = gid = 0
if place.uid: uid = place.uid
if place.gid: gid = place.gid
if uid != 0 or gid != 0:
sld('Setting uid/gid to {}:{} of {}:{}/{}'.format(
uid, gid, dest_host, dest_dir, file_name))
sftp.chown(file_name, uid, gid)
elif place.pgLink:
try:
sftp.unlink('postgresql.crt')
except IOError:
pass # none exists: ignore
sftp.symlink(file_name, 'postgresql.crt')
sld('{} => postgresql.crt'.format(file_name))
if jail and place.reload_command:
try:
cmd = str((place.reload_command).format(jail))
except: #No "{}" in reload command: means no jail
cmd = place.reload_command
sli('Executing "{}" on host {}'.format(cmd, dest_host))
with client.get_transport().open_session() as chan:
chan.settimeout(10.0)
chan.set_combine_stderr(True)
chan.exec_command(cmd)
remote_result_msg = ''
timed_out = False
while not chan.exit_status_ready():
if timed_out: break
if chan.recv_ready():
try:
data = chan.recv(1024)
except (timeout):
sle('Timeout on remote execution of "{}" on host {}'
.format(cmd, dest_host))
break
while data:
remote_result_msg += (data.decode('ascii'))
try:
data = chan.recv(1024)
except (timeout):
sle('Timeout on remote execution of "{}" on host {}'
.format(cmd, dest_host))
tmp = timed_out
timed_out = True
break
es = int(chan.recv_exit_status())
if es != 0:
sln('Remote execution failure of "{}" on host {}\texit={}, because:\n\r{}'
.format(cmd, dest_host, es, remote_result_msg))
else:
sli(remote_result_msg)
def scheduleCerts(db: db_conn, cert_metas: Dict[str, cert.Certificate]) -> None:
"""
Schedule state transitions and do related actions of CertInstances
:param db: Open Database connection
:param cert_metas: list of Cerificate instances to act on
:return:
"""
global ps_delete, to_be_deleted
(DBAccount, Misc, Pathes, X509atts) = get_config()
opts = get_options()
def issue(cm: cert.Certificate) -> Optional[cert.CertInstance]:
"""
If cert type is 'LE', issue a Letsencrypt cert
:param cm: cert meta
:return: ci of new cert or None
"""
if cm.cert_type == cert.CertType('local'):
return None
if opts.check_only:
sld('Would issue {}.'.format(cm.name))
return
if not cm.disabled:
sli('Requesting issue from LE for {}'.format(cm.name))
return issue_LE_cert(cm)
def prepublish(cm: cert.Certificate, active_ci: cert.CertInstance, new_ci: cert.CertInstance) -> None:
"""
Prepublish cert hashes per TLSA RRs in DNS
:param cm: Our cert meta data instance
:param active_ci: CertInstance currently in use
:param new_ci: CertInstance just created but not yet deployed
:return:
"""
if opts.check_only:
sld('Would prepublish {} {}.'.format(active_ci.row_id, new_ci.row_id))
return
# collect hashes for all certs in all algos
hashes = tuple(cm.TLSA_hashes(active_ci).values()) + tuple(cm.TLSA_hashes(new_ci).values())
sli('Prepublishing {}:{}:{}'.
format(cm.name, active_ci.row_id, new_ci.row_id))
distribute_tlsa_rrs(cm, hashes)
new_ci.state = cert.CertState('prepublished')
cm.save_instance(new_ci)
def distribute(cm: cert.Certificate, ci: cert.CertInstance, state: cert.CertState):
if opts.check_only:
sld('Would distribute {}.'.format(ci.row_id))
return
sli('Distributing {}:{}'.
format(cm.name, ci.row_id))
cm_dict = {cm.name: cm}
try:
deployCerts(cm_dict, (ci,), allowed_states=(state,))
except Exception:
sln('Skipping distribution of cert {} because {} [{}]'.format(
cm.name,
sys.exc_info()[0].__name__,
str(sys.exc_info()[1])))
def expire(cm, ci):
if opts.check_only:
sld('Would expire {}.'.format(ci.row_id))
return
sli('State transition from {} to EXPIRED of {}:{}'.
format(ci.state, cm.name, ci.row_id))
ci.state = cert.CertState('expired')
cm.save_instance(ci)
def archive(cm, ci):
if opts.check_only:
sld('Would archive {}.'.format(ci.row_id))
return
sli('State transition from {} to ARCHIVED of {}:{}'.
format(ci.state, cm.name, ci.row_id))
ci.state = cert.CertState('archived')
cm.save_instance(ci)
for cm in cert_metas.values():
sld('{} {} ------------------------------'.format(
cm.name,
'DISABLED' if cm.disabled else ''))
if cm.subject_type in (cert.SubjectType('CA'),cert.SubjectType('reserved')): continue
issued_ci = None
prepublished_ci = None
deployed_ci = None
surviving = _find_to_be_deleted(cm)
if not surviving:
ci = issue(cm)
if ci: distribute(cm, ci, cert.CertState('issued'))
continue
for ci in surviving:
if ci.state == cert.CertState('expired'):
archive(cm, ci)
continue
if datetime.utcnow() >= (ci.not_after + timedelta(days=1)):
if ci.state != cert.CertState('deployed'):
expire(cm, ci)
continue
elif ci.state == cert.CertState('issued'):
issued_ci = ci
elif ci.state == cert.CertState('prepublished'):
prepublished_ci = ci
elif ci.state == cert.CertState('deployed'):
deployed_ci = ci
else:
assert (ci.state in (cert.CertState('issued'), cert.CertState('prepublished'), cert.CertState('deployed'),))
if deployed_ci and issued_ci: # issued too old to replace deployed in future?
if issued_ci.not_after < (deployed_ci.not_after +
timedelta(days=Misc.LOCAL_ISSUE_MAIL_TIMEDELTA)):
to_be_deleted |= set((issued_ci,)) # yes: mark for delete
issued_ci = None
# request issue_mail if near to expiration
if (deployed_ci
and cm.cert_type == 'local'
and not cm.authorized_until
and datetime.utcnow() >= (deployed_ci.not_after -
timedelta(days=Misc.LOCAL_ISSUE_MAIL_TIMEDELTA))):
to_be_mailed.append(cm)
sld('schedule.to_be_mailed: ' + str(cm))
if cm.disabled:
continue
# deployed cert expired or no cert deployed?
if (not deployed_ci) or \
(datetime.utcnow() >= deployed_ci.not_after - timedelta(days=1)):
distributed = False
sld('scheduleCerts: no deployed cert or deployed cert'
'expired {}'.format(str(deployed_ci)))
if prepublished_ci: # yes - distribute prepublished
distribute(cm, prepublished_ci, cert.CertState('prepublished'))
distributed = True
elif issued_ci: # or issued cert?
distribute(cm, issued_ci, cert.CertState('issued')) # yes - distribute it
distributed = True
if deployed_ci:
expire(cm, deployed_ci) # and expire deployed cert
if not distributed:
ci = issue(cm)
if ci: distribute(cm, ci, cert.CertState('issued'))
continue
if cm.cert_type == 'local':
continue # no TLSAs with local certs
# We have an active LE cert deployed
if datetime.utcnow() >= \
(deployed_ci.not_after - timedelta(days=Misc.PRE_PUBLISH_TIMEDELTA)):
# pre-publishtime reached?
ci = issued_ci
if prepublished_ci: # yes: TLSA already pre-published?
continue # yes
elif not issued_ci: # do we have a cert handy?
ci = issue(cm) # no: create one
if not ci:
sln('Failed to issue cert for prepublishing of {}'.format(cm.name))
continue
sld('scheduleCerts will call prepublish with deployed_ci={}, ci={}'.format(
str(deployed_ci), str(ci)))
prepublish(cm, deployed_ci, ci) # and prepublish it
# end for name in cert_names
if opts.check_only:
sld('Would delete and mail..')
return
for ci in to_be_deleted:
sld('Deleting {}'.format(ci.row_id))
result = ci.cm.delete_instance(ci)
if result != 1:
sln('Failed to delete cert instance {}'.format(ci.row_id))
if to_be_mailed:
body = str('Following local Certificates must be issued prior to {}:\n'.
format(date.today() + timedelta(days=Misc.LOCAL_ISSUE_MAIL_TIMEDELTA)))
for cert_meta in to_be_mailed:
body += str('\t{} \t{}'.format(cert_meta.name,
'[DISABLED]' if cert_meta.disabled else ''))
cert_meta.update_authorized_until(datetime.utcnow())
msg = MIMEText(body)
msg['Subject'] = 'Local certificate issue reminder'
msg['From'] = Misc.MAIL_SENDER
msg['To'] = Misc.MAIL_RECIPIENT
s = smtplib.SMTP(Misc.MAIL_RELAY)
s.send_message(msg)
s.quit()
def _find_to_be_deleted(cm: cert.Certificate) -> Optional[set]:
"""
Create set of CertInstances to be deleted.
Keep most recent active Certinstance in state prepublished and deployed.
If only active in state issued and expired, keep theese.
:param cm: Certificate to act on
:return: Set of Certinstances to be deleted
"""
global to_be_deleted
surviving = set()
if not cm.cert_instances: return None
for ci in cm.cert_instances:
sld('{:04} Issued {}, expires: {}, state {}\t{}'.format(
ci.row_id,
shortDateTime(ci.not_before),
shortDateTime(ci.not_after),
ci.state,
cm.name)
)
if ci.state in (cert.CertState('reserved'), cert.CertState('archived')):
to_be_deleted.add(ci)
else:
surviving.add(ci)
sld('Before state loop: ' + str([i.__str__() for i in surviving]))
for state in (cert.CertState('issued'), cert.CertState('prepublished'), cert.CertState('deployed'), cert.CertState('expired')):
ci_list = []
for ci in surviving:
if ci.state == state:
ci_list.append(ci)
if not ci_list:
continue
# only the most recent (with highest row_id) survives from current state set
sorted_list = sorted(ci_list, key=lambda ci: ci.row_id)
to_be_added_to_be_deleted = set(sorted_list[:-1]) # all but last to be deleted
to_be_deleted.update(to_be_added_to_be_deleted) # add to to_be_deleted set
surviving = surviving - to_be_added_to_be_deleted # remove from surviving set
sld('{} surviving in state {}'.format(sorted_list[-1].row_id, state))
sld('to_be_deleted now: {}'.format(str([i.__str__() for i in to_be_deleted])))
sld('surviving now: {}'.format(str([i.__str__() for i in surviving])))
sld('---------------------------------------------------------------')
return surviving
def distribute_tlsa_rrs(cert_meta: Certificate,
hashes: Union[Tuple[str], List[str]]) -> None:
"""
Distribute TLSA RR.
Puts TLSA RR fqdn into DNS zone, by dynamic dns or editing zone file and updating zone cache.
If cert has altnames, one set of TLSA RRs is inserted per altname and per TLSA prefix.
:param cert_meta:
:param hashes: list of hashes, may include active and prepublishes hashes for all algos
:return:
"""
if len(cert_meta.tlsaprefixes) == 0: return
sli('Distributing TLSA RRs for DANE.')
if Pathes.tlsa_dns_master == '': # DNS master on local host
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for (zone, fqdn) in zone_and_FQDN_from_altnames(cert_meta):
filename = fqdn + '.tlsa'
dest = str(Pathes.zone_file_root / zone / filename)
sli('{} => {}'.format(filename, dest))
tlsa_lines = []
for prefix in cert_meta.tlsaprefixes.keys():
for hash in hashes:
tlsa_lines.append(
str(prefix.format(fqdn) + ' ' + hash + '\n'))
with open(dest, 'w') as fd:
fd.writelines(tlsa_lines)
updateZoneCache(zone)
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
tlsa_datatype = rdatatype.from_text('TLSA')
zones = {}
for (zone, fqdn) in cert_meta.zone_and_FQDN_from_altnames():
if zone in zones:
if fqdn not in zones[zone]: zones[zone].append(fqdn)
else:
zones[zone] = [fqdn]
for zone in zones:
the_update = ddns_update(zone)
for fqdn in zones[zone]:
for prefix in cert_meta.tlsaprefixes.keys():
pf_with_fqdn = str(prefix.format(fqdn))
fields = pf_with_fqdn.split(maxsplit=4)
sld('Deleting possible old TLSAs: {}'.format(
fields[0]))
the_update.delete(fields[0], tlsa_datatype)
for hash in hashes:
sld('Adding TLSA: {} {} {} {}'.format(
fields[0], int(fields[1]), fields[3],
fields[4] + ' ' + hash))
the_update.add(fields[0], int(fields[1]),
fields[3], fields[4] + ' ' + hash)
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
rc = response.rcode()
if rc != 0:
sle('DNS update failed for zone {} with rcode: {}:\n{}'.
format(zone, response.rcode.to_text(rc),
response.rcode))
raise Exception(
'DNS update add failed for zone {} with rcode: {}'.
format(zone, response.rcode.to_text(rc)))
else: # remote DNS master ( **INCOMPLETE**)
sle('Remote DNS master server is currently not supported. Must be on same host as this script.'
)
exit(1)
with ssh_connection(Pathes.tlsa_dns_master) as client:
with client.open_sftp() as sftp:
chdir(str(Pathes.work_tlsa))
p = Path('.')
sftp.chdir(str(Pathes.zone_file_root))
for child_dir in p.iterdir():
for child in child_dir.iterdir():
sli('{} => {}:{}'.format(child, Pathes.tlsa_dns_master,
child))
fat = sftp.put(str(child), str(child), confirm=True)
sld('size={}, uid={}, gid={}, mtime={}'.format(
fat.st_size, fat.st_uid, fat.st_gid, fat.st_mtime))
def _issue_cert_for_one_algo(encryption_algo: EncAlgoCKS,
cert_meta: Certificate,
account: Account) -> Optional[dict]:
"""
Try to issue a Letsencrypt certificate for one encryption algorithm.
Does authorization if necessary.
:param encryption_algo: encryption algo to use
:param cert_meta: description of cert
:param account: our account at Letsencrypt
:return: None or dict: dict layout as follows:
{'Cert': certificate, 'Key': certificate_key, 'Intermediate': intcert, 'Algo': encryption_algo}
"""
options = get_options()
alt_names = [
cert_meta.name,
]
if len(cert_meta.altnames) > 0:
alt_names.extend(cert_meta.altnames)
sli('Creating {} key and {} cert for {}'.format(
'rsa {} bits'.format(int(X509atts.bits)) if encryption_algo == 'rsa'
else 'ec', cert_meta.subject_type, cert_meta.name))
order = _authorize(cert_meta, account)
if not order:
return None
if encryption_algo == EncAlgoCKS('rsa'):
certificate_key = manuale_crypto.generate_rsa_key(X509atts.bits)
elif encryption_algo == EncAlgoCKS('ec'):
crypto_backend = default_backend()
certificate_key = ec.generate_private_key(ec.SECP384R1(),
crypto_backend)
else:
raise ValueError(
'Wrong encryption_algo {} in _issue_cert_for_one_algo for {}'.
format(encryption_algo, cert_meta.name))
order.key = manuale_crypto.export_private_key(certificate_key).decode(
'ascii')
csr = manuale_crypto.create_csr(certificate_key,
alt_names,
must_staple=cert_meta.ocsp_must_staple)
acme = AcmeV2(Misc.LE_SERVER, account)
try:
sli('Requesting certificate issuance from LE...')
if not order.contents['status'] == 'valid':
if order.contents['status'] == 'ready':
final_order = acme.finalize_order(order, csr)
order.contents = final_order
if order.contents['status'] == "valid":
sld('{}/{}: Order finalized. Certificate is being issued.'
.format(cert_meta.name, encryption_algo))
else:
sld("{}/{}: Checking order status.".format(
cert_meta.name, encryption_algo))
fulfillment = acme.await_for_order_fulfillment(order)
if fulfillment['status'] == "valid":
order.contents = fulfillment
else:
sle("{}: Order not ready or invalid after finalize. Status = {}. Giving up. \n Response = {}"
.format(cert_meta.name, final_order['status'],
final_order))
return None
if not order.certificate_uri:
sle("{}/{}: Order not valid after fulfillment: Missing certificate URI"
.format(cert_meta.name, encryption_algo))
return None
else:
sli("Downloading certificate for {}/{}.".format(
cert_meta.name, encryption_algo))
result = acme.download_order_certificate(order)
except manuale_errors.AcmeError as e:
if '(type urn:acme:error:unauthorized, HTTP 403)' in str(e):
sle('LetsEncrypt lost authorization for {}/{} [DOWNLOAD]. Giving up'
.format(cert_meta.name, encryption_algo))
else:
sle("Connection or service request failed for {}/{}[DOWNLOAD]. Aborting."
.format(cert_meta.name, encryption_algo))
raise manuale_errors.AutomatoesError(e)
try:
certificates = manuale_crypto.strip_certificates(result.content) # DER
certificate = manuale_crypto.load_pem_certificate(certificates[0])
except IOError as e:
sle("Failed to load new certificate for {}/{}. Aborting.".format(
cert_meta.name, encryption_algo))
raise manuale_errors.AutomatoesError(e)
intcert = manuale_crypto.load_pem_certificate(certificates[1])
return {
'Cert': certificate,
'Key': certificate_key,
'Intermediate': intcert,
'Algo': encryption_algo
}
def _authorize(cert_meta: Certificate, account: Account) -> Optional[Order]:
"""
Try to prove the control about a DNS object.
@param cert_meta: Cert meta instance to issue an certificate for
@type cert_meta: Cert meta instance
@param account: Our letsencrypt account
@type account: manuale_cli.load_account instance
@rtype: True if all fqdns could be authorized, False otherwise
@exceptions: manuale_errors.AutomatoesError on Network or other fatal error
"""
acme = AcmeV2(Misc.LE_SERVER, account)
FQDNS = dict()
FQDNS[cert_meta.name] = 0
for name in cert_meta.altnames:
if name not in FQDNS:
FQDNS[name] = 0
domains = list(FQDNS.keys())
try:
order: Order = acme.new_order(domains, 'dns')
except AcmeError as e:
print(e)
return None
returned_order = acme.query_order(order)
sld('new_order for {} returned\n{}'.format(cert_meta.name,
print_order(returned_order)))
if order.expired or order.invalid:
sle("{}: Order is {} {}. Giving up.".format(
cert_meta.name, 'invalid' if order.invalid else '',
'expired' if order.expired else ''))
return None
returned_fqdns = [idf['value'] for idf in order.contents['identifiers']]
if set(domains) != set(returned_fqdns):
sle("{}: List of FQDNS returned by order does not match ordered FQDNs:\n{}\n{}"
.format(cert_meta.name, returned_fqdns, domains))
return None
if order.contents['status'] != 'pending':
return order # all done, if now challenge pending
fqdn_challenges = {} # key = domain, value = chllenge
pending_challenges = acme.get_order_challenges(order)
for challenge in pending_challenges:
if challenge.status == 'valid':
sli(" {} is already authorized until {}.".format(
challenge.domain, challenge.expires))
continue
fqdn_challenges[challenge.domain] = challenge
# find zones by fqdn
zones = {}
sld('Calling zone_and_FQDN_from_altnames()')
for (zone, fqdn) in Certificate.zone_and_FQDN_from_altnames(cert_meta):
if fqdn in fqdn_challenges:
if zone in zones:
if fqdn not in zones[zone]: zones[zone].append(fqdn)
else:
zones[zone] = [fqdn]
sld('zones: {}'.format(zones))
if not fqdn_challenges:
server_order = acme.query_order(order)
order.contents = server_order.contents
sli("All Altnames of {} are already authorized.Order status = {}".
format(cert_meta.name, order.contents['status']))
return order
create_challenge_responses_in_dns(zones, fqdn_challenges)
sld('{} completed DNS setup on hidden primary for all pending FQDNs'.
format(datetime.datetime.utcnow().isoformat()))
# Validate challenges
authorized_until = None
sli('Waiting 15 seconds for dns propagation')
time.sleep(15)
for challenge in pending_challenges:
# wait maximum 2 minutes
sld('{} starting verification of {}'.format(
datetime.datetime.utcnow().isoformat(), challenge.domain))
response = acme.verify_order_challenge(challenge,
timeout=5,
retry_limit=5)
sld('{} acme.verify_order_challenge returned "{}"'.format(
datetime.datetime.utcnow().isoformat(), response['status']))
if response['status'] == "valid":
sld("{}: OK! Authorization lasts until {}.".format(
challenge.domain, challenge.expires))
authorized_until = challenge.expires
elif response['status'] == 'invalid':
sle("{}: Challenge failed, because: {} ({})".format(
challenge.domain, response['error']['detail'],
response['error']['type']))
# we need either all challenges or none: repeat with next cron cacle
return None
else:
sln("{}: Challenge returned status {}".format(
challenge.domain, response['status']))
# we need either all challenges or none: repeat with next cron cacle
return None
server_order = acme.query_order(order)
order.contents = server_order.contents
sld("All Altnames of {} authorized.Order status = {}".format(
cert_meta.name, order.contents['status']))
# remember new expiration date in DB
if authorized_until:
updates = cert_meta.update_authorized_until(
datetime.datetime.fromisoformat(re.sub('Z', '', authorized_until)))
if updates != 1:
sln('Failed to update DB with new authorized_until timestamp')
delete_challenge_responses_in_dns(zones)
sli("FQDNs authorized. Let's Encrypt!")
return order
def execute_from_command_line():
"""
Called from comand line
:return: sys.exit(0) on success, False or sys.exit(1) otherwise
"""
try: # try to contact IDE (PyCharm)
from serverPKI.settrace import connect_to_ide
connect_to_ide()
except Exception: # no IDE configured
pass
all_cert_names: List[str, ...] = []
our_cert_names: List[str, ...] = []
our_certs: Dict[str, Certificate] = {}
##util.log_to_file('sftp.log')
opts = parse_options() # globals not re-initialized
parse_config() # globals not re-initialized
(DBAccount, Misc, Pathes, X509atts) = get_config()
db_name = DBAccount.dbDatabase
sli('operateCA [{}-{}] started with options {}'.format(
db_name, get_version_string(), options_set()))
check_actions()
pe = dbc('serverpki')
db: db_conn = pe.open()
# in case of restarting serverPKI wo fresh imports (e.g. by pytest)
init_module_cert()
LocalCaCertCache.cert = None
LocalCaCertCache.key = None
LocalCaCertCache.ci = None
read_db_encryption_key(db)
all_cert_names = Certificate.names(db)
all_disthost_names = Certificate.disthost_names(db)
# preload CMs, CIs and CKSs of our CAs
cas = []
for name in (Misc.SUBJECT_LOCAL_CA, Misc.SUBJECT_LE_CA):
if name in all_cert_names: # does CA exist in DB?
cm = Certificate.create_or_load_cert_meta(db, name)
cas.append(name)
sli('{} certificates and CAs {} in DB'.format(len(all_cert_names), cas))
if opts.encrypt:
if encrypt_all_keys(db):
sys.exit(0)
sys.exit(1)
if opts.decrypt:
if decrypt_all_keys(db):
sys.exit(0)
sys.exit(1)
if opts.issue_local_cacert:
if issue_local_CAcert(db):
sys.exit(0)
sys.exit(1)
if opts.all:
our_cert_names = all_cert_names
elif opts.remaining_days:
our_cert_names = names_of_local_certs_to_be_renewed(
db, opts.remaining_days, opts.distribute)
if not opts.create:
opts.create = not opts.distribute
cert_name_set: set = set(our_cert_names)
error = False
if opts.only_cert:
error = False
for i in opts.only_cert:
if i not in all_cert_names:
sle("{} not in configuration. Can't be specified with --only".
format(i))
error = True
if not error:
cert_name_set = set(opts.only_cert)
else:
if opts.cert_to_be_included:
error = False
for i in opts.cert_to_be_included:
if i not in all_cert_names:
sle("{} not in configuration. Can't be included".format(i))
error = True
if not error:
cert_name_set = set(opts.cert_to_be_included)
if opts.cert_to_be_excluded:
error = False
for i in opts.cert_to_be_excluded:
if i not in all_cert_names:
sle("{} not in configuration. Can't be excluded".format(i))
error = True
if not error:
cert_name_set -= set(opts.cert_to_be_excluded)
if opts.only_host:
for i in opts.only_host:
if i not in all_disthost_names:
sle("{} not in configuration. Can't use in --limit-to-disthost"
.format(i))
error = True
if opts.skip_host:
for i in opts.skip_host:
if i not in all_disthost_names:
sle("{} not in configuration. Can't use in --skip-disthost".
format(i))
error = True
if error:
sle('Stopped due to command line errors')
sys.exit(1)
our_cert_names = sorted(list(cert_name_set))
for name in our_cert_names:
cm = Certificate.create_or_load_cert_meta(db, name)
if cm.in_db: our_certs[name] = cm
if opts.check_only and not opts.schedule:
sli('No syntax errors found in configuration.')
##sli('Selected certificates:\n\r{}'.format(our_cert_names))
print_certs(db, our_cert_names)
sys.exit(0)
sld('Selected certificates:\n\r{}'.format(our_cert_names))
if opts.schedule:
sli('Scheduling actions.')
scheduleCerts(db, our_certs)
else:
if opts.create:
sli('Creating certificates.')
for c in our_certs.values():
if c.cert_type == CertType('LE'):
if issue_LE_cert(c):
continue
elif c.cert_type == CertType('local'):
if issue_local_cert(c):
continue
else:
raise AssertionError('Invalid CertType in {}'.format(
c.name))
sle('Stopped due to error')
sys.exit(1)
if opts.distribute:
sli('Distributing certificates.')
deployCerts(our_certs)
if opts.sync_disk:
for c in our_certs.values():
consolidate_cert(c)
if opts.sync_tlsas:
for c in our_certs.values():
consolidate_TLSA(c)
updateSOAofUpdatedZones()
if opts.remove_tlsas:
for c in our_certs.values():
delete_TLSA(c)
updateSOAofUpdatedZones()
if opts.cert_serial:
sli('Exporting certificate instance.')
export_instance(db)
if opts.register:
sli('Registering a new Let\'s Encrypt Account.\n With URI:{}\n'
' and e-mail {}'.format(Misc.LE_SERVER, Misc.LE_EMAIL))
register(Misc.LE_SERVER, Pathes.le_account, Misc.LE_EMAIL, None)
