def _load_cakey(cakey_pem: bytes) -> Optional[rsa.RSAPrivateKey]:
"""
Load private key and query passphrase
:param cakey_pem: The PEM encoded key data as bytes
:return:
:exceptions: ValueError, TypeError
"""
cakey = None
def _load_it(cakey_pem, passphrase):
cakey = serialization.load_pem_private_key(data=cakey_pem,
password=passphrase,
backend=default_backend())
return cakey
try:
cakey = _load_it(cakey_pem, None)
except (TypeError): # needing passphrase
sli('Please enter passphrase to unlock key of CA cert.')
while not cakey:
pp1 = getpass.getpass(prompt='passphrase (empty to abort): ')
if pp1 == '':
return None
try:
cakey = _load_it(cakey_pem, pp1.encode('utf-8'))
except (TypeError, ValueError, UnicodeDecodeError):
sle('Wrong passphrase. Please retry')
return cakey
def open(self):
"""
Open the connection to the DB server
@rtype: DbConnection instance with state 'opened'
@exceptions: None, but does a exit(1) if connection can't be
established
"""
if not self.conn:
try:
if self.ssl_required:
if self.sslcrtfile:
self.conn = pg_open(host=self.host, port=self.port, user=self.user, database=self.database,
sslmode="require", sslcrtfile=self.sslcrtfile, sslkeyfile=self.sslkeyfile)
else:
self.conn = pg_open(host=self.host, port=self.port, user=self.user, database=self.database,
sslmode="require")
else:
self.conn = pg_open(host=self.host, port=self.port, user=self.user, database=self.database)
self.conn.settings['search_path'] = self.search_path
except:
sle('Unable to connect to database %s' % (self.dsn))
sys.exit(1)
return self.conn
def create_challenge_responses_in_dns(zones, fqdn_challenges):
"""
Create the expected challenge response in dns
@param zones: dict of zones, where each zone has a list of fqdns
as values
@type zones: dict()
@param fqdn_challenges: dict of zones, containing challenge response
(key) of zone
@type fqdn_challenges: dict()
@rtype: None
@exceptions Can''t parse ddns key or
DNS update failed for zone {} with rcode: {}
"""
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for zone in zones.keys():
dest = str(Pathes.zone_file_root / zone /
Pathes.zone_file_include_name)
lines = []
for fqdn in zones[zone]:
sld('fqdn: {}'.format(fqdn))
lines.append(
str('_acme-challenge.{}. IN TXT \"{}\"\n'.format(
fqdn, fqdn_challenges[fqdn].key)))
sli('Writing RRs: {}'.format(lines))
with open(dest, 'w') as file:
file.writelines(lines)
##os.chmod(file.fileno(), Pathes.zone_tlsa_inc_mode)
##os.chown(file.fileno(), pathes.zone_tlsa_inc_uid, pathes.zone_tlsa_inc_gid)
updateZoneCache(zone)
updateSOAofUpdatedZones()
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
txt_datatape = rdatatype.from_text('TXT')
for zone in zones.keys():
the_update = ddns_update(zone)
for fqdn in zones[zone]:
the_update.delete('_acme-challenge.{}.'.format(fqdn),
txt_datatape)
the_update.add('_acme-challenge.{}.'.format(fqdn), 60,
txt_datatape, fqdn_challenges[fqdn].key)
sld('DNS update of RR: {}'.format(
'_acme-challenge.{}. 60 TXT \"{}\"'.format(
fqdn, fqdn_challenges[fqdn].key)))
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
sld('DNS update delete/add returned response: {}'.format(response))
rc = response.rcode()
if rc != 0:
sle('DNS delete failed for zone {} with rcode: {}:\n{}'.format(
zone, rc.to_text(rc), rc))
raise Exception(
'DNS update failed for zone {} with rcode: {}'.format(
zone, rc.to_text(rc)))
def delete_TLSA(cert_meta: Certificate) -> None:
"""
Delete all TLSA RRs per fqdn of all altnames either in flatfile (make include file empty) or in dyn dns
:param cert_meta:
:return:
"""
if Pathes.tlsa_dns_master == '': # DNS master on local host
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for (zone, fqdn) in cert_meta.zone_and_FQDN_from_altnames():
filename = fqdn + '.tlsa'
dest = str(Pathes.zone_file_root / zone / filename)
#just open for write without writing, which makes file empty
with open(dest, 'w') as fd:
sli('Truncating {}'.format(dest))
updateZoneCache(zone)
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
zones = {}
for (zone, fqdn) in cert_meta.zone_and_FQDN_from_altnames():
if zone in zones:
if fqdn not in zones[zone]: zones[zone].append(fqdn)
else:
zones[zone] = [fqdn]
for zone in zones:
the_update = ddns_update(zone)
for fqdn in zones[zone]:
for prefix in cert_meta.tlsaprefixes.keys():
tag = str(prefix.format(fqdn)).split(maxsplit=1)[0]
sld('Deleting TLSA with tag {} an fqdn {} in zone {}'.
format(tag, fqdn, zone))
the_update.delete(tag)
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
rc = response.rcode()
if rc != 0:
sle('DNS update failed for zone {} with rcode: {}:\n{}'.
format(zone, response.rcode.to_text(rc),
response.rcode))
raise Exception(
'DNS update failed for zone {} with rcode: {}'.format(
zone, response.rcode.to_text(rc)))
def create_CAcert_meta(
db: db_conn,
name: str,
cert_type: Optional[CertType] = None) -> Optional[Certificate]:
"""
Lookup or create a CA cert meta in rows ceetificates and subjects
:param db: opened database connection in read/write transaction
:param name: name of CA cert (as configured in config: Misc.SUBJECT_LOCAL_CA or SUBJECT_LE_CA)
:cert_type: CertType
:return: cert meta or None
"""
if name not in (Misc.SUBJECT_LOCAL_CA, Misc.SUBJECT_LE_CA):
raise AssertionError(
'create_CAcert_meta: argument name "{} invalid"'.format(name))
cm = Certificate.ca_cert_meta(db, name, cert_type)
if not cm:
sle('Failed to create CA cert meta for {}'.format(name))
sys.exit(1)
return cm
def issue_local_CAcert(db: db_conn) -> bool:
"""
Issue a local CA cert and store it in DB.
:param db: Opened DB connection
:return: True, if CA cert created and stored in DB, False otherwise
"""
sli('Creating local CA certificate.')
cm = Certificate.create_or_load_cert_meta(db, name=Misc.SUBJECT_LOCAL_CA)
if not cm.in_db: # loaded from DB?
# CM not in DB, create rows in Subjects and Certificates for it
cm = create_CAcert_meta(db=db,
name=Misc.SUBJECT_LOCAL_CA,
cert_type=CertType('local'))
try:
create_local_ca_cert(db, cm)
except Exception as e:
sle('Failed to create local CA cert, because: {}'.format(str(e)))
return False
return True
def delete_challenge_responses_in_dns(zones):
"""
Delete the challenge response in dns, created by
create_challenge_responses_in_dns()
@param zones: dict of zones, where each zone has a list of fqdns
as values
@type zones: dict()
@rtype: None
@exceptions Can''t parse ddns key or
DNS update failed for zone {} with rcode: {}
"""
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for zone in zones.keys():
dest = str(Pathes.zone_file_root / zone /
Pathes.zone_file_include_name)
with open(dest, 'w') as file:
file.writelines(('', ))
updateZoneCache(zone)
updateSOAofUpdatedZones()
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
txt_datatape = rdatatype.from_text('TXT')
for zone in zones.keys():
the_update = ddns_update(zone)
for fqdn in zones[zone]:
the_update.delete('_acme-challenge.{}.'.format(fqdn),
txt_datatape)
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
sld('DNS update delete/add returned response: {}'.format(response))
rc = response.rcode()
if rc != 0:
sle('DNS update failed for zone {} with rcode: {}:\n{}'.format(
zone, rc.to_text(rc), rc))
raise Exception(
'DNS update failed for zone {} with rcode: {}'.format(
zone, rc.to_text(rc)))
def __init__(self, service):
"""
Create a DbConnection instance
@param service: service name
@type service: string
@rtype: DbConnection instance
@exceptions: None, but does a exit(1) if connection can't be
established
"""
self.lock = None
self.conn = None
self.sslcrtfile = None
try:
self.host = DBAccount.dbHost
self.port = str(DBAccount.dbPort)
self.user = DBAccount.dbUser
self.dba_user = DBAccount.dbDbaUser
self.ssl_required = DBAccount.dbSslRequired
self.database = DBAccount.dbDatabase
self.search_path = DBAccount.dbSearchPath
ssl_option = (', sslmode=' + '"require"') if self.ssl_required else ''
self.dsn = str('host=' + self.host + ', port=' + self.port +
', user='******', database=' + self.database + ssl_option)
if DBAccount.dbCert:
self.sslcrtfile = DBAccount.dbCert
self.sslkeyfile = DBAccount.dbCertKey
self.dsn = self.dsn + str(', sslcrtfile={}, sslkeyfile={}'.format(self.sslcrtfile, self.sslkeyfile))
except:
sle('Config error: Missing or wrong keyword in DBAccount.\n' +
'Must be dbHost, dbPort, dbUser, , dbDbaUser, dbDatabase, dbSslrequired and dbSearchPath.')
raise(BaseException())
def create_local_ca_cert(
db: db_conn, cm: Certificate
) -> Tuple[x509.Certificate, rsa.RSAPrivateKeyWithSerialization, CertInstance]:
"""
Create a new local CA cert (use an existing one, if one in db)
Make it available in globals local_cacert, local_cakey and local_cacert_instance
:param db: Opened DB connection
:param cm: CA cert Certifcate meta instance
:return: Tuple of cacert, cakey and cacert instance and globals
local_cacert, local_cakey, local_cacert_instance setup
"""
global local_cacert, local_cakey, local_cacert_instance
# Read pass phrase from user
match = False
while not match:
sli('Please enter passphrase for new CA cert (ASCII only).')
try:
pp1 = getpass.getpass(prompt='passphrase: ')
except UnicodeDecodeError:
sle('None-ASCII character found.')
continue
if pp1 == '':
sln('Passphrases must not be empty.')
continue
sli('Please enter it again.')
try:
pp2 = getpass.getpass(prompt='passphrase: ')
except UnicodeDecodeError:
pass
if pp1 == pp2:
match = True
else:
sln('Both passphrases differ.')
# Generate our key
cakey = rsa.generate_private_key(public_exponent=65537,
key_size=Misc.LOCAL_CA_BITS,
backend=default_backend())
# compose cert
# Various details about who we are. For a self-signed certificate the
# subject and issuer are always the same.
serial = randbits(32)
name_dict = X509atts.names
subject = issuer = x509.Name([
x509.NameAttribute(x509.NameOID.COUNTRY_NAME, name_dict['C']),
x509.NameAttribute(x509.NameOID.LOCALITY_NAME, name_dict['L']),
x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, name_dict['O']),
x509.NameAttribute(x509.NameOID.COMMON_NAME, name_dict['CN']),
])
not_after = datetime.datetime.utcnow() + datetime.timedelta(
days=Misc.LOCAL_CA_LIFETIME)
not_before = datetime.datetime.utcnow() - datetime.timedelta(days=1)
ci = cm.create_instance(not_before=not_before,
not_after=not_after,
state=CertState('issued'))
ski = x509.SubjectKeyIdentifier.from_public_key(cakey.public_key())
cacert = x509.CertificateBuilder().subject_name(subject).issuer_name(
issuer).public_key(cakey.public_key()).serial_number(
serial).not_valid_before(not_before).not_valid_after(
# Our certificate will be valid for 10 days
not_after).add_extension(
# CA and no intermediate CAs
x509.BasicConstraints(ca=True, path_length=0),
critical=True).add_extension(
ski,
critical=False,
# Sign our certificate with our private key
).add_extension(x509.KeyUsage(digital_signature=True,
key_cert_sign=True,
crl_sign=True,
key_encipherment=False,
content_commitment=False,
data_encipherment=False,
key_agreement=False,
encipher_only=False,
decipher_only=False),
critical=True).sign(
cakey, hashes.SHA512(),
default_backend())
sli('CA cert serial {} with {} bit key, valid until {} created.'.format(
serial, Misc.LOCAL_CA_BITS, not_after.isoformat()))
ci.store_cert_key(algo='rsa', cert=cacert, key=cakey)
cm.save_instance(ci)
# CA cert loaded, cache it and return
cache_cacert(cacert, cakey, ci)
return (cacert, cakey, ci)
def get_cacert_and_key(
db: db_conn
) -> Tuple[x509.Certificate, rsa.RSAPrivateKeyWithSerialization, CertInstance]:
"""
Return a valid local CA certificate and a loaded private key.
Use globals local_cacert, local_cakey and local_cacert_instance if available
If necessary, create a local CAcert or read a historical one from disk.
Store it in DB, creating necessary rows in Subjects, Certificates
and Certinstances and store them in LocalCaCertCache
Does exit(1) if CA key could not be loaded.
:param db: Opened DB connection
:return: Tuple of cacert, cakey and cacert instance and globals
local_cacert, local_cakey, local_cacert_instance setup
"""
if LocalCaCertCache.cert and LocalCaCertCache.key and LocalCaCertCache.ci:
return (LocalCaCertCache.cert, LocalCaCertCache.key,
LocalCaCertCache.ci)
cacert = None
cakey = None
ci = None
cm = Certificate.create_or_load_cert_meta(db, name=Misc.SUBJECT_LOCAL_CA)
if cm.in_db: # loaded from DB?
ci = cm.most_recent_active_instance # YES: load its active CI
if ci: # one there, which is valid today?
cks = list(ci.the_cert_key_stores.values())[
0] # expecting only one algorithm
cacert_pem = cks.cert_for_ca
cakey_pem = cks.key_for_ca
algo = cks.algo
cacert = x509.load_pem_x509_certificate(data=cacert_pem,
backend=default_backend())
cakey = _load_cakey(cakey_pem)
if not cakey: # operator aborted pass phrase input
sle('Can' 't create certificates without passphrase')
exit(1)
# CA cert loaded, cache it and return
cache_cacert(cacert, cakey, ci)
return (cacert, cakey, ci)
sln('Missed active cacert instance or store in db, where it should be: {}'
.format(cm.name))
else: # CM not in DB, create rows in Subjects and Certificates for it
cm = create_CAcert_meta(db=db,
name=Misc.SUBJECT_LOCAL_CA,
cert_type=CertType('local'))
# Try to load active historical CA cert from flat file
result = load_historical_cert_from_flatfile(cm)
if result: # loaded, cached and stored in DB
return result # globals already setup
else: # none found in flat file - issue a new one
result = create_local_ca_cert(db, cm)
if result: # issued, cached and stored in DB
return result # globals already setup
else:
sle('Failed to issue CA cert')
sys.exit(1)
if not ci: # no most_recent_active_instance found above, issue one
result = create_local_ca_cert(db, cm)
if result: # issued, cached and stored in DB
return result
else:
sle('Failed to issue CA cert')
sys.exit(1)
def issue_local_cert(cert_meta: Certificate) -> Optional[CertInstance]:
"""
Ask local CA to issue a certificate.
Will ask for a passphrase to access the CA key.
On success, inserts a row into CertInstances.
If this is the first local instance, additional rows are inserted
into Subjects, Certificates and CertInstances for local CA cert.
Additional Certinstances may also be inserted if the local CA cert
changes.
FIXME: Currently only supports rsa keys
:param cert_meta: Cert meta instance to issue an certificate for
:rtype: cert instance id in DB of new cert or None
"""
cacert, cakey, cacert_ci = get_cacert_and_key(cert_meta.db)
sli('Creating key ({} bits) and cert for {} {}. Using CA cert {}'.format(
int(X509atts.bits), cert_meta.subject_type, cert_meta.name,
cacert_ci.row_id))
serial = x509.random_serial_number()
# Generate our key
key = rsa.generate_private_key(public_exponent=65537,
key_size=int(X509atts.bits),
backend=default_backend())
builder = x509.CertificateBuilder()
builder = builder.subject_name(
x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, cert_meta.name),
]))
builder = builder.issuer_name(x509.Name(cacert.subject, ))
not_valid_before = datetime.datetime.utcnow() - datetime.timedelta(days=1)
not_valid_after = datetime.datetime.utcnow() + datetime.timedelta(
days=X509atts.lifetime)
builder = builder.not_valid_before(not_valid_before)
builder = builder.not_valid_after(not_valid_after)
builder = builder.serial_number(serial)
public_key = key.public_key()
builder = builder.public_key(public_key)
builder = builder.add_extension(
x509.BasicConstraints(ca=False, path_length=None),
critical=True,
)
ski = x509.SubjectKeyIdentifier.from_public_key(key.public_key())
builder = builder.add_extension(
ski,
critical=False,
)
try:
builder = builder.add_extension(
x509.AuthorityKeyIdentifier.from_issuer_subject_key_identifier(
x509.SubjectKeyIdentifier.from_public_key(cakey.public_key())),
critical=False,
)
except x509.extensions.ExtensionNotFound:
sle('Could not add a AuthorityKeyIdentifier, because CA has no SubjectKeyIdentifier'
)
if cert_meta.subject_type == 'client':
alt_names = [
x509.RFC822Name(cert_meta.name),
]
else:
alt_names = [
x509.DNSName(cert_meta.name),
]
for n in cert_meta.altnames:
if cert_meta.subject_type == 'client':
alt_names.append(x509.RFC822Name(n))
else:
alt_names.append(x509.DNSName(n))
builder = builder.add_extension(
x509.SubjectAlternativeName(alt_names),
critical=False,
)
builder = builder.add_extension(
x509.KeyUsage(digital_signature=True,
key_encipherment=True
if cert_meta.subject_type == 'server' else False,
content_commitment=False,
data_encipherment=False,
key_agreement=False,
key_cert_sign=False,
crl_sign=False,
encipher_only=False,
decipher_only=False),
critical=True,
)
eku = None
if cert_meta.subject_type == 'server':
eku = x509.oid.ExtendedKeyUsageOID.SERVER_AUTH
elif cert_meta.subject_type == 'client':
eku = x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH
if eku:
builder = builder.add_extension(
x509.ExtendedKeyUsage((eku, )),
critical=True,
)
cert = builder.sign(private_key=cakey,
algorithm=hashes.SHA384(),
backend=default_backend())
ci = cert_meta.create_instance(state=CertState('issued'),
not_before=not_valid_before,
not_after=not_valid_after,
ca_cert_ci=cacert_ci)
ci.store_cert_key(algo=cert_meta.encryption_algo, cert=cert, key=key)
cert_meta.save_instance(ci)
sli('Certificate for {} {}, serial {}, valid until {} created.'.format(
cert_meta.subject_type, cert_meta.name, serial,
not_valid_after.isoformat()))
return ci
def distribute_cert(fd, dest_host, dest_dir, file_name, place, jail):
"""
Distribute cert and key to a host, jail (if any) and place.
Optional reload the service.
If global opts.extract set, instead of distributing to a host,
certificat and key are written to the local work directory.
@param fd: file descriptor of memory stream
@type fd: io.StringIO
@param dest_host: fqdn of target host
@type dest_host: string
@param dest_dir: target directory
@type dest_dir: string
@param file_name: file name of key or cert file
@type file_name: string
@param place: place with details about setting mode and uid/gid of file
@type place: serverPKI.cert.Place instance
@param jail: name of jail for service to reload
@type jail: string or None
@rtype: not yet any
@exceptions: IOError
"""
sld('Handling dest_host {} and dest_dir "{}" in distribute_cert'.format(
dest_host, dest_dir))
with ssh_connection(dest_host) as client:
with client.open_sftp() as sftp:
try:
sftp.chdir(str(dest_dir))
except IOError:
sln('{}:{} does not exist - creating\n\t{}'.format(
dest_host, dest_dir,
sys.exc_info()[0].__name__))
try:
sftp.mkdir(str(dest_dir))
except IOError:
sle('Cant create {}:{}: Missing parent?\n\t{}'.format(
dest_host, dest_dir,
sys.exc_info()[0].__name__, str(sys.exc_info()[1])))
raise
sftp.chdir(str(dest_dir))
sli('{} => {}:{}'.format(file_name, dest_host, dest_dir))
fat = sftp.putfo(fd, file_name, confirm=True)
sld('size={}, uid={}, gid={}, mtime={}'.format(
fat.st_size, fat.st_uid, fat.st_gid, fat.st_mtime))
if 'key' in file_name:
sld('Setting mode to 0o400 of {}:{}/{}'.format(
dest_host, dest_dir, file_name))
mode = 0o400
if place.mode:
mode = place.mode
sld('Setting mode of key at target to {}'.format(
oct(place.mode)))
sftp.chmod(file_name, mode)
if place.pgLink:
try:
sftp.unlink('postgresql.key')
except IOError:
pass # none exists: ignore
sftp.symlink(file_name, 'postgresql.key')
sld('{} => postgresql.key'.format(file_name))
if 'key' in file_name or place.chownBoth:
uid = gid = 0
if place.uid: uid = place.uid
if place.gid: gid = place.gid
if uid != 0 or gid != 0:
sld('Setting uid/gid to {}:{} of {}:{}/{}'.format(
uid, gid, dest_host, dest_dir, file_name))
sftp.chown(file_name, uid, gid)
elif place.pgLink:
try:
sftp.unlink('postgresql.crt')
except IOError:
pass # none exists: ignore
sftp.symlink(file_name, 'postgresql.crt')
sld('{} => postgresql.crt'.format(file_name))
if jail and place.reload_command:
try:
cmd = str((place.reload_command).format(jail))
except: #No "{}" in reload command: means no jail
cmd = place.reload_command
sli('Executing "{}" on host {}'.format(cmd, dest_host))
with client.get_transport().open_session() as chan:
chan.settimeout(10.0)
chan.set_combine_stderr(True)
chan.exec_command(cmd)
remote_result_msg = ''
timed_out = False
while not chan.exit_status_ready():
if timed_out: break
if chan.recv_ready():
try:
data = chan.recv(1024)
except (timeout):
sle('Timeout on remote execution of "{}" on host {}'
.format(cmd, dest_host))
break
while data:
remote_result_msg += (data.decode('ascii'))
try:
data = chan.recv(1024)
except (timeout):
sle('Timeout on remote execution of "{}" on host {}'
.format(cmd, dest_host))
tmp = timed_out
timed_out = True
break
es = int(chan.recv_exit_status())
if es != 0:
sln('Remote execution failure of "{}" on host {}\texit={}, because:\n\r{}'
.format(cmd, dest_host, es, remote_result_msg))
else:
sli(remote_result_msg)
def issue_LE_cert(cert_meta: Certificate) -> Optional[CertInstance]:
"""
Try to issue a Letsencrypt certificate.
Does authorization if necessary.
On success, inserts a row into CertInstances.
If this is the first Letsencrypt instance, additional rows are inserted
into Subjects, Certificates and CertInstances for LE intermediate cert.
Additional Certinstances may also be inserted if the intermediate cert
from LE changes.
:param cert_meta: Description of certificate to issue
:return: Instance of new cert
"""
# Set up logging
root = logging.getLogger('automatoes')
root.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(logging.Formatter("%(message)s"))
root.addHandler(handler)
os.chdir(str(Pathes.work)) ##FIXME## remove this ?
sli('Creating certificate for {} and crypto algo {}'.format(
cert_meta.name, cert_meta.encryption_algo))
try:
account: Account = manuale_cli.load_account(str(Pathes.le_account))
except:
sle('Problem with Lets Encrypt account data at {}'.format(
Pathes.le_account))
return None
if cert_meta.encryption_algo == EncAlgo('rsa plus ec'):
encryption_algos = (EncAlgoCKS('rsa'), EncAlgoCKS('ec'))
else:
encryption_algos = (cert_meta.encryption_algo, )
results = []
for encryption_algo in encryption_algos:
result = _issue_cert_for_one_algo(encryption_algo, cert_meta, account)
if not result:
return None
else:
results.append(result)
# loop ensures to store either all cks in DB or none:
ci = None
for result in results:
if not ci:
cacert_ci = _get_intermediate_instance(
db=cert_meta.db, int_cert=result['Intermediate'])
ci = cert_meta.create_instance(
state=CertState('issued'),
not_before=result['Cert'].not_valid_before,
not_after=result['Cert'].not_valid_after,
ca_cert_ci=cacert_ci)
cks = ci.store_cert_key(algo=result['Algo'],
cert=result['Cert'],
key=result['Key'])
sli('Certificate issued for {} . Valid until {}'.format(
cert_meta.name, result['Cert'].not_valid_after.isoformat()))
sli('Hash is: {}, algo is {}'.format(cks.hash, result['Algo']))
cert_meta.save_instance(ci)
return ci
def _authorize(cert_meta: Certificate, account: Account) -> Optional[Order]:
"""
Try to prove the control about a DNS object.
@param cert_meta: Cert meta instance to issue an certificate for
@type cert_meta: Cert meta instance
@param account: Our letsencrypt account
@type account: manuale_cli.load_account instance
@rtype: True if all fqdns could be authorized, False otherwise
@exceptions: manuale_errors.AutomatoesError on Network or other fatal error
"""
acme = AcmeV2(Misc.LE_SERVER, account)
FQDNS = dict()
FQDNS[cert_meta.name] = 0
for name in cert_meta.altnames:
if name not in FQDNS:
FQDNS[name] = 0
domains = list(FQDNS.keys())
try:
order: Order = acme.new_order(domains, 'dns')
except AcmeError as e:
print(e)
return None
returned_order = acme.query_order(order)
sld('new_order for {} returned\n{}'.format(cert_meta.name,
print_order(returned_order)))
if order.expired or order.invalid:
sle("{}: Order is {} {}. Giving up.".format(
cert_meta.name, 'invalid' if order.invalid else '',
'expired' if order.expired else ''))
return None
returned_fqdns = [idf['value'] for idf in order.contents['identifiers']]
if set(domains) != set(returned_fqdns):
sle("{}: List of FQDNS returned by order does not match ordered FQDNs:\n{}\n{}"
.format(cert_meta.name, returned_fqdns, domains))
return None
if order.contents['status'] != 'pending':
return order # all done, if now challenge pending
fqdn_challenges = {} # key = domain, value = chllenge
pending_challenges = acme.get_order_challenges(order)
for challenge in pending_challenges:
if challenge.status == 'valid':
sli(" {} is already authorized until {}.".format(
challenge.domain, challenge.expires))
continue
fqdn_challenges[challenge.domain] = challenge
# find zones by fqdn
zones = {}
sld('Calling zone_and_FQDN_from_altnames()')
for (zone, fqdn) in Certificate.zone_and_FQDN_from_altnames(cert_meta):
if fqdn in fqdn_challenges:
if zone in zones:
if fqdn not in zones[zone]: zones[zone].append(fqdn)
else:
zones[zone] = [fqdn]
sld('zones: {}'.format(zones))
if not fqdn_challenges:
server_order = acme.query_order(order)
order.contents = server_order.contents
sli("All Altnames of {} are already authorized.Order status = {}".
format(cert_meta.name, order.contents['status']))
return order
create_challenge_responses_in_dns(zones, fqdn_challenges)
sld('{} completed DNS setup on hidden primary for all pending FQDNs'.
format(datetime.datetime.utcnow().isoformat()))
# Validate challenges
authorized_until = None
sli('Waiting 15 seconds for dns propagation')
time.sleep(15)
for challenge in pending_challenges:
# wait maximum 2 minutes
sld('{} starting verification of {}'.format(
datetime.datetime.utcnow().isoformat(), challenge.domain))
response = acme.verify_order_challenge(challenge,
timeout=5,
retry_limit=5)
sld('{} acme.verify_order_challenge returned "{}"'.format(
datetime.datetime.utcnow().isoformat(), response['status']))
if response['status'] == "valid":
sld("{}: OK! Authorization lasts until {}.".format(
challenge.domain, challenge.expires))
authorized_until = challenge.expires
elif response['status'] == 'invalid':
sle("{}: Challenge failed, because: {} ({})".format(
challenge.domain, response['error']['detail'],
response['error']['type']))
# we need either all challenges or none: repeat with next cron cacle
return None
else:
sln("{}: Challenge returned status {}".format(
challenge.domain, response['status']))
# we need either all challenges or none: repeat with next cron cacle
return None
server_order = acme.query_order(order)
order.contents = server_order.contents
sld("All Altnames of {} authorized.Order status = {}".format(
cert_meta.name, order.contents['status']))
# remember new expiration date in DB
if authorized_until:
updates = cert_meta.update_authorized_until(
datetime.datetime.fromisoformat(re.sub('Z', '', authorized_until)))
if updates != 1:
sln('Failed to update DB with new authorized_until timestamp')
delete_challenge_responses_in_dns(zones)
sli("FQDNs authorized. Let's Encrypt!")
return order
def _issue_cert_for_one_algo(encryption_algo: EncAlgoCKS,
cert_meta: Certificate,
account: Account) -> Optional[dict]:
"""
Try to issue a Letsencrypt certificate for one encryption algorithm.
Does authorization if necessary.
:param encryption_algo: encryption algo to use
:param cert_meta: description of cert
:param account: our account at Letsencrypt
:return: None or dict: dict layout as follows:
{'Cert': certificate, 'Key': certificate_key, 'Intermediate': intcert, 'Algo': encryption_algo}
"""
options = get_options()
alt_names = [
cert_meta.name,
]
if len(cert_meta.altnames) > 0:
alt_names.extend(cert_meta.altnames)
sli('Creating {} key and {} cert for {}'.format(
'rsa {} bits'.format(int(X509atts.bits)) if encryption_algo == 'rsa'
else 'ec', cert_meta.subject_type, cert_meta.name))
order = _authorize(cert_meta, account)
if not order:
return None
if encryption_algo == EncAlgoCKS('rsa'):
certificate_key = manuale_crypto.generate_rsa_key(X509atts.bits)
elif encryption_algo == EncAlgoCKS('ec'):
crypto_backend = default_backend()
certificate_key = ec.generate_private_key(ec.SECP384R1(),
crypto_backend)
else:
raise ValueError(
'Wrong encryption_algo {} in _issue_cert_for_one_algo for {}'.
format(encryption_algo, cert_meta.name))
order.key = manuale_crypto.export_private_key(certificate_key).decode(
'ascii')
csr = manuale_crypto.create_csr(certificate_key,
alt_names,
must_staple=cert_meta.ocsp_must_staple)
acme = AcmeV2(Misc.LE_SERVER, account)
try:
sli('Requesting certificate issuance from LE...')
if not order.contents['status'] == 'valid':
if order.contents['status'] == 'ready':
final_order = acme.finalize_order(order, csr)
order.contents = final_order
if order.contents['status'] == "valid":
sld('{}/{}: Order finalized. Certificate is being issued.'
.format(cert_meta.name, encryption_algo))
else:
sld("{}/{}: Checking order status.".format(
cert_meta.name, encryption_algo))
fulfillment = acme.await_for_order_fulfillment(order)
if fulfillment['status'] == "valid":
order.contents = fulfillment
else:
sle("{}: Order not ready or invalid after finalize. Status = {}. Giving up. \n Response = {}"
.format(cert_meta.name, final_order['status'],
final_order))
return None
if not order.certificate_uri:
sle("{}/{}: Order not valid after fulfillment: Missing certificate URI"
.format(cert_meta.name, encryption_algo))
return None
else:
sli("Downloading certificate for {}/{}.".format(
cert_meta.name, encryption_algo))
result = acme.download_order_certificate(order)
except manuale_errors.AcmeError as e:
if '(type urn:acme:error:unauthorized, HTTP 403)' in str(e):
sle('LetsEncrypt lost authorization for {}/{} [DOWNLOAD]. Giving up'
.format(cert_meta.name, encryption_algo))
else:
sle("Connection or service request failed for {}/{}[DOWNLOAD]. Aborting."
.format(cert_meta.name, encryption_algo))
raise manuale_errors.AutomatoesError(e)
try:
certificates = manuale_crypto.strip_certificates(result.content) # DER
certificate = manuale_crypto.load_pem_certificate(certificates[0])
except IOError as e:
sle("Failed to load new certificate for {}/{}. Aborting.".format(
cert_meta.name, encryption_algo))
raise manuale_errors.AutomatoesError(e)
intcert = manuale_crypto.load_pem_certificate(certificates[1])
return {
'Cert': certificate,
'Key': certificate_key,
'Intermediate': intcert,
'Algo': encryption_algo
}
def distribute_tlsa_rrs(cert_meta: Certificate,
hashes: Union[Tuple[str], List[str]]) -> None:
"""
Distribute TLSA RR.
Puts TLSA RR fqdn into DNS zone, by dynamic dns or editing zone file and updating zone cache.
If cert has altnames, one set of TLSA RRs is inserted per altname and per TLSA prefix.
:param cert_meta:
:param hashes: list of hashes, may include active and prepublishes hashes for all algos
:return:
"""
if len(cert_meta.tlsaprefixes) == 0: return
sli('Distributing TLSA RRs for DANE.')
if Pathes.tlsa_dns_master == '': # DNS master on local host
if Misc.LE_ZONE_UPDATE_METHOD == 'zone_file':
for (zone, fqdn) in zone_and_FQDN_from_altnames(cert_meta):
filename = fqdn + '.tlsa'
dest = str(Pathes.zone_file_root / zone / filename)
sli('{} => {}'.format(filename, dest))
tlsa_lines = []
for prefix in cert_meta.tlsaprefixes.keys():
for hash in hashes:
tlsa_lines.append(
str(prefix.format(fqdn) + ' ' + hash + '\n'))
with open(dest, 'w') as fd:
fd.writelines(tlsa_lines)
updateZoneCache(zone)
elif Misc.LE_ZONE_UPDATE_METHOD == 'ddns':
tlsa_datatype = rdatatype.from_text('TLSA')
zones = {}
for (zone, fqdn) in cert_meta.zone_and_FQDN_from_altnames():
if zone in zones:
if fqdn not in zones[zone]: zones[zone].append(fqdn)
else:
zones[zone] = [fqdn]
for zone in zones:
the_update = ddns_update(zone)
for fqdn in zones[zone]:
for prefix in cert_meta.tlsaprefixes.keys():
pf_with_fqdn = str(prefix.format(fqdn))
fields = pf_with_fqdn.split(maxsplit=4)
sld('Deleting possible old TLSAs: {}'.format(
fields[0]))
the_update.delete(fields[0], tlsa_datatype)
for hash in hashes:
sld('Adding TLSA: {} {} {} {}'.format(
fields[0], int(fields[1]), fields[3],
fields[4] + ' ' + hash))
the_update.add(fields[0], int(fields[1]),
fields[3], fields[4] + ' ' + hash)
response = dns_query.tcp(the_update, '127.0.0.1', timeout=10)
rc = response.rcode()
if rc != 0:
sle('DNS update failed for zone {} with rcode: {}:\n{}'.
format(zone, response.rcode.to_text(rc),
response.rcode))
raise Exception(
'DNS update add failed for zone {} with rcode: {}'.
format(zone, response.rcode.to_text(rc)))
else: # remote DNS master ( **INCOMPLETE**)
sle('Remote DNS master server is currently not supported. Must be on same host as this script.'
)
exit(1)
with ssh_connection(Pathes.tlsa_dns_master) as client:
with client.open_sftp() as sftp:
chdir(str(Pathes.work_tlsa))
p = Path('.')
sftp.chdir(str(Pathes.zone_file_root))
for child_dir in p.iterdir():
for child in child_dir.iterdir():
sli('{} => {}:{}'.format(child, Pathes.tlsa_dns_master,
child))
fat = sftp.put(str(child), str(child), confirm=True)
sld('size={}, uid={}, gid={}, mtime={}'.format(
fat.st_size, fat.st_uid, fat.st_gid, fat.st_mtime))
def execute_from_command_line():
"""
Called from comand line
:return: sys.exit(0) on success, False or sys.exit(1) otherwise
"""
try: # try to contact IDE (PyCharm)
from serverPKI.settrace import connect_to_ide
connect_to_ide()
except Exception: # no IDE configured
pass
all_cert_names: List[str, ...] = []
our_cert_names: List[str, ...] = []
our_certs: Dict[str, Certificate] = {}
##util.log_to_file('sftp.log')
opts = parse_options() # globals not re-initialized
parse_config() # globals not re-initialized
(DBAccount, Misc, Pathes, X509atts) = get_config()
db_name = DBAccount.dbDatabase
sli('operateCA [{}-{}] started with options {}'.format(
db_name, get_version_string(), options_set()))
check_actions()
pe = dbc('serverpki')
db: db_conn = pe.open()
# in case of restarting serverPKI wo fresh imports (e.g. by pytest)
init_module_cert()
LocalCaCertCache.cert = None
LocalCaCertCache.key = None
LocalCaCertCache.ci = None
read_db_encryption_key(db)
all_cert_names = Certificate.names(db)
all_disthost_names = Certificate.disthost_names(db)
# preload CMs, CIs and CKSs of our CAs
cas = []
for name in (Misc.SUBJECT_LOCAL_CA, Misc.SUBJECT_LE_CA):
if name in all_cert_names: # does CA exist in DB?
cm = Certificate.create_or_load_cert_meta(db, name)
cas.append(name)
sli('{} certificates and CAs {} in DB'.format(len(all_cert_names), cas))
if opts.encrypt:
if encrypt_all_keys(db):
sys.exit(0)
sys.exit(1)
if opts.decrypt:
if decrypt_all_keys(db):
sys.exit(0)
sys.exit(1)
if opts.issue_local_cacert:
if issue_local_CAcert(db):
sys.exit(0)
sys.exit(1)
if opts.all:
our_cert_names = all_cert_names
elif opts.remaining_days:
our_cert_names = names_of_local_certs_to_be_renewed(
db, opts.remaining_days, opts.distribute)
if not opts.create:
opts.create = not opts.distribute
cert_name_set: set = set(our_cert_names)
error = False
if opts.only_cert:
error = False
for i in opts.only_cert:
if i not in all_cert_names:
sle("{} not in configuration. Can't be specified with --only".
format(i))
error = True
if not error:
cert_name_set = set(opts.only_cert)
else:
if opts.cert_to_be_included:
error = False
for i in opts.cert_to_be_included:
if i not in all_cert_names:
sle("{} not in configuration. Can't be included".format(i))
error = True
if not error:
cert_name_set = set(opts.cert_to_be_included)
if opts.cert_to_be_excluded:
error = False
for i in opts.cert_to_be_excluded:
if i not in all_cert_names:
sle("{} not in configuration. Can't be excluded".format(i))
error = True
if not error:
cert_name_set -= set(opts.cert_to_be_excluded)
if opts.only_host:
for i in opts.only_host:
if i not in all_disthost_names:
sle("{} not in configuration. Can't use in --limit-to-disthost"
.format(i))
error = True
if opts.skip_host:
for i in opts.skip_host:
if i not in all_disthost_names:
sle("{} not in configuration. Can't use in --skip-disthost".
format(i))
error = True
if error:
sle('Stopped due to command line errors')
sys.exit(1)
our_cert_names = sorted(list(cert_name_set))
for name in our_cert_names:
cm = Certificate.create_or_load_cert_meta(db, name)
if cm.in_db: our_certs[name] = cm
if opts.check_only and not opts.schedule:
sli('No syntax errors found in configuration.')
##sli('Selected certificates:\n\r{}'.format(our_cert_names))
print_certs(db, our_cert_names)
sys.exit(0)
sld('Selected certificates:\n\r{}'.format(our_cert_names))
if opts.schedule:
sli('Scheduling actions.')
scheduleCerts(db, our_certs)
else:
if opts.create:
sli('Creating certificates.')
for c in our_certs.values():
if c.cert_type == CertType('LE'):
if issue_LE_cert(c):
continue
elif c.cert_type == CertType('local'):
if issue_local_cert(c):
continue
else:
raise AssertionError('Invalid CertType in {}'.format(
c.name))
sle('Stopped due to error')
sys.exit(1)
if opts.distribute:
sli('Distributing certificates.')
deployCerts(our_certs)
if opts.sync_disk:
for c in our_certs.values():
consolidate_cert(c)
if opts.sync_tlsas:
for c in our_certs.values():
consolidate_TLSA(c)
updateSOAofUpdatedZones()
if opts.remove_tlsas:
for c in our_certs.values():
delete_TLSA(c)
updateSOAofUpdatedZones()
if opts.cert_serial:
sli('Exporting certificate instance.')
export_instance(db)
if opts.register:
sli('Registering a new Let\'s Encrypt Account.\n With URI:{}\n'
' and e-mail {}'.format(Misc.LE_SERVER, Misc.LE_EMAIL))
register(Misc.LE_SERVER, Pathes.le_account, Misc.LE_EMAIL, None)
def deployCerts(
cert_metas: Dict[str, Certificate],
cert_instances: Optional[Tuple[CertInstance]] = None,
allowed_states: Tuple[CertState] = (CertState('issued'), )
) -> bool:
"""
Deploy a list of (certificates. keys and TLSA RRs, using paramiko/sftp) and dyn DNS (or zone files).
Restart service at target host and reload nameserver (if using zone files).
:param cert_metas: Dict of cert metas, telling which certs to deploy, key is cert subject name
:param cert_instances: Optional list of CertInstance instances
:param allowed_states: States describing CertInstance states to act on
:return: True if successfully deployed certs
"""
error_found = False
limit_hosts = False
opts = get_options()
only_host = []
if opts.only_host: only_host = opts.only_host
if len(only_host) > 0: limit_hosts = True
skip_host = []
if opts.skip_host: skip_host = opts.skip_host
sld('limit_hosts={}, only_host={}, skip_host={}'.format(
limit_hosts, only_host, skip_host))
for cert_meta in cert_metas.values():
if len(cert_meta.disthosts) == 0: continue
the_instances = []
hashes = []
##FIXME## highly speculative!
insts = cert_instances if cert_instances else [
y for (x, y) in cert_meta.active_instances.items()
]
for ci in insts:
if ci.state in allowed_states:
the_instances.append(ci)
if len(the_instances) == 0:
etxt = 'No valid cerificate for {} in DB - create it first\n' \
'States being considered are {}'. \
format(cert_meta.name, [state for state in allowed_states])
sli(etxt)
if cert_instances: # let caller handle this error, if we have explicit inst ids
raise MyException(etxt)
else:
continue
# more than 1 member of the_instances only expected with cert.instance(i).encryption_algo == 'both'
for ci in the_instances:
state = ci.state
cacert_text = cert_meta.cacert_PEM(ci)
host_omitted = False
cksd = ci.the_cert_key_stores
for encryption_algo in cksd.keys():
cks = cksd[encryption_algo]
cert_text = cks.cert
key_text = cks.key
TLSA_text = cks.hash
hashes.append(TLSA_text)
for fqdn, dh in cert_meta.disthosts.items():
if fqdn in skip_host:
host_omitted = True
continue
if limit_hosts and (fqdn not in only_host):
host_omitted = True
continue
dest_path = PurePath('/')
sld('{}: {}'.format(cert_meta.name, fqdn))
for jail in (dh['jails'].keys()
or ('', )): # jail is empty if no jails
if '/' in jail:
sle('"/" in jail name "{}" not allowed with subject {}.'
.format(jail, cert_meta.name))
error_found = True
return False
jailroot = dh[
'jailroot'] if jail != '' else '' # may also be empty
dest_path = PurePath('/', jailroot, jail)
sld('{}: {}: {}'.format(cert_meta.name, fqdn,
dest_path))
the_jail = dh['jails'][jail]
if len(the_jail['places']) == 0:
sle('{} subject has no place attribute.'.format(
cert_meta.name))
error_found = True
return False
for place in the_jail['places'].values():
sld('Handling jail "{}" and place {}'.format(
jail, place.name))
fd_key = StringIO(key_text)
fd_cert = StringIO(cert_text)
key_file_name = key_name(cert_meta.name,
cert_meta.subject_type,
encryption_algo)
cert_file_name = cert_name(cert_meta.name,
cert_meta.subject_type,
encryption_algo)
pcp = place.cert_path
if '{}' in pcp: # we have a home directory named like the subject
pcp = pcp.format(cert_meta.name)
# make sure pcb does not start with '/', which would ignore dest_path:
if PurePath(pcp).is_absolute():
dest_dir = PurePath(
dest_path,
PurePath(pcp).relative_to('/'))
else:
dest_dir = PurePath(dest_path, PurePath(pcp))
sld('Handling fqdn {} and dest_dir "{}" in deployCerts'
.format(fqdn, dest_dir))
try:
if place.key_path:
key_dest_dir = PurePath(
dest_path, place.key_path)
distribute_cert(fd_key, fqdn, key_dest_dir,
key_file_name, place, None)
elif place.cert_file_type == 'separate':
distribute_cert(fd_key, fqdn, dest_dir,
key_file_name, place, None)
if cert_meta.cert_type == 'LE':
chain_file_name = cert_cacert_chain_name(
cert_meta.name,
cert_meta.subject_type,
encryption_algo)
fd_chain = StringIO(cert_text +
cacert_text)
distribute_cert(
fd_chain, fqdn, dest_dir,
chain_file_name, place, jail)
elif place.cert_file_type == 'combine key':
cert_file_name = key_cert_name(
cert_meta.name, cert_meta.subject_type,
encryption_algo)
fd_cert = StringIO(key_text + cert_text)
if cert_meta.cert_type == 'LE':
chain_file_name = key_cert_cacert_chain_name(
cert_meta.name,
cert_meta.subject_type,
encryption_algo)
fd_chain = StringIO(key_text +
cert_text +
cacert_text)
distribute_cert(
fd_chain, fqdn, dest_dir,
chain_file_name, place, jail)
elif place.cert_file_type == 'combine both':
cert_file_name = key_cert_cacert_name(
cert_meta.name, cert_meta.subject_type,
encryption_algo)
fd_cert = StringIO(key_text + cert_text +
cacert_text)
elif place.cert_file_type == 'combine cacert':
cert_file_name = cert_cacert_name(
cert_meta.name, cert_meta.subject_type,
encryption_algo)
fd_cert = StringIO(cert_text + cacert_text)
distribute_cert(fd_key, fqdn, dest_dir,
key_file_name, place, None)
# this may be redundant in case of LE, where the cert was in chained file
distribute_cert(fd_cert, fqdn, dest_dir,
cert_file_name, place, jail)
except IOError: # distribute_cert may error out
error_found = True
break # no cert - no TLSA
sli('')
if opts.sync_disk: # skip TLSA stuff if doing consolidate
continue
if not opts.no_TLSA:
distribute_tlsa_rrs(cert_meta, hashes)
if not host_omitted and not cert_meta.subject_type == 'CA':
ci.state = CertState('deployed')
cert_meta.save_instance(ci)
else:
sln('State of cert {} not promoted to DEPLOYED, '
'because hosts where limited or skipped'.format(
cert_meta.name))
# clear mail-sent-time if local cert.
if cert_meta.cert_type == CertType('local'):
cert_meta.update_authorized_until(None)
updateSOAofUpdatedZones()
return not error_found
