def task_config():
'''
write config.yml -> .config.yml
'''
log_level = 'WARNING'
filename = '{0}/LOG_LEVEL'.format(os.path.dirname(__file__))
if os.path.isfile(filename):
log_level = open(filename).read().strip()
log_level = get_var('LOG_LEVEL', log_level)
if log_level not in LOG_LEVELS:
raise UnknownLogLevelError(log_level)
punch = fmt('''
logging:
loggers:
api:
level: {log_level}
handlers:
console:
level: {log_level}
''')
return {
'actions': [
fmt('echo "cp {CONFIG_YML}\n-> {DOT_CONFIG_YML}"'),
fmt('echo "setting LOG_LEVEL={log_level}"'),
fmt('cp {CONFIG_YML} {DOT_CONFIG_YML}'),
lambda: _update_config(DOT_CONFIG_YML, yaml.safe_load(punch)),
]
}
def _get_installed_certificates_details(self, certs, *dests):
app.logger.debug(fmt('_get_installed_certificates_details:\n{locals}'))
summary = self._get_installed_summary(certs, *dests)
details = {}
if summary:
common_names, paths, dests = zip(*summary)
calls = self.gets(paths=paths,
dests=dests,
product=False,
verify_ssl=False)
for common_name, path, dest, call in zip(common_names, paths,
dests, calls):
try:
crt = windows2unix(
call.recv.json.properties.basic.get(
'public', 'missing'))
csr = windows2unix(
call.recv.json.properties.basic.get(
'request', 'missing'))
key = windows2unix(
call.recv.json.properties.basic.get(
'private', 'missing'))
note = call.recv.json.properties.basic.get('note', '')
except Exception as ex:
app.logger.debug(fmt('call.send.url={0}', call.send.url))
app.logger.debug(
fmt('call.recv.json=\n{0}', call.recv.json))
raise ex
details[(common_name, crt[:40])] = details.get(
(common_name, crt[:40]), {})
details[(common_name, crt[:40])][dest] = (key, csr, crt, note)
return details
def _validate_domains(self,
organization_id,
container_id,
domains,
whois_check=False):
app.logger.debug(fmt('_validate_domains:\n{locals}'))
active_domains = self._get_domains(organization_id, container_id)
active_domains = [ad.name for ad in active_domains]
def _is_validated(domain_to_check):
matched_domains = [
ad for ad in active_domains if domain_to_check == ad
]
if matched_domains:
domain = matched_domains[0]
else:
matched_subdomains = [
ad for ad in active_domains
if domain_to_check.endswith('.' + ad)
]
if matched_subdomains:
domain = matched_subdomains[0]
else:
return False
return True
def _whois_email(domain_to_check):
app.logger.debug(fmt('_whois_email:\n{locals}'))
try:
emails = whois(domain_to_check)['emails']
app.logger.debug(fmt('emails={emails}'))
return '*****@*****.**' in emails
except Exception as ex:
app.logger.debug('WHOIS_ERROR')
app.logger.debug(ex)
return False
return False
app.logger.debug(fmt('domains={domains}'))
domains = list(set([get_tld('http://' + domain)
for domain in domains]))
app.logger.debug(fmt('domains={domains}'))
not_whois_domains = []
if whois_check:
app.logger.info(
'the whois check was enabled with --whois-check flag for this run'
)
not_whois_domains = [
domain for domain in domains if not _whois_email(domain)
]
if not_whois_domains:
raise WhoisDoesntMatchError(not_whois_domains)
denied_domains = [
domain for domain in domains if not _is_validated(domain)
]
if denied_domains:
raise NotValidatedDomainError(denied_domains, active_domains)
return True
def _get_domains(self, organization_id, container_id):
app.logger.debug(fmt('_get_domains:\n{locals}'))
call = self.get(fmt('domain?container_id={container_id}'))
if call.recv.status != 200:
raise DigicertError(call)
return [
domain for domain in call.recv.json.domains
if domain.is_active and domain.organization.id == organization_id
]
def _approve_certificate(self, request_id):
app.logger.info(fmt('_approve_certificate:\n{locals}'))
path = fmt('request/{request_id}/status')
json = dict(status='approved', processor_comment='autocert')
app.logger.debug(
fmt('calling digicert api with path={path} and json={json}'))
call = self.put(path=path, json=json)
if call.recv.status == 204:
return True
raise ApproveCertificateError(call)
def _whois_email(domain_to_check):
app.logger.debug(fmt('_whois_email:\n{locals}'))
try:
emails = whois(domain_to_check)['emails']
app.logger.debug(fmt('emails={emails}'))
return '*****@*****.**' in emails
except Exception as ex:
app.logger.debug('WHOIS_ERROR')
app.logger.debug(ex)
return False
return False
def _prepare_paths_jsons_for_revocations(self, certs, bug):
app.logger.debug(
fmt('_prepare_paths_jsons_for_revocations:\n{locals}'))
order_ids = [cert.authority['digicert']['order_id'] for cert in certs]
calls = self._get_certificate_order_detail(order_ids)
certificate_ids = [call.recv.json.certificate.id for call in calls]
paths = [
fmt('certificate/{certificate_id}/revoke')
for certificate_id in certificate_ids
]
jsons = [dict(comments=str(bug))]
return paths, jsons
def task_zeus():
'''
launch zeus containers
'''
image = 'zeus17.3'
for container in [ fmt('{image}_test{num}') for num in (1, 2)]:
yield {
'task_dep': ['prune'],
'name': container,
'actions': [fmt('docker run -d --name {container} {image}')],
'uptodate': [fmt('[ -n "`docker ps -q -f name={container}`" ] && exit 0 || exit 1')]
}
def task_pull():
'''
do a safe git pull
'''
test = '`git diff-index --quiet HEAD --`'
pull = 'git pull --rebase'
dirty = fmt('echo "refusing to \'{pull}\' because the tree is dirty"')
return {
'actions': [
fmt('if {test}; then {pull}; else {dirty}; exit 1; fi'),
],
}
def task_savelogs():
'''
save the logs to a timestamped file
'''
timestamp = datetime2int(utcnow())
return {
'task_dep': ['checkreqs', 'dockercompose'],
'actions': [
fmt('mkdir -p {LOGDIR}'),
fmt('docker-compose logs > {LOGDIR}/{timestamp}.log'),
]
}
def task_rmvolumes():
'''
remove dangling docker volumes
'''
query = '`docker volume ls -q -f dangling=true`'
return {
'actions': [
fmt('docker volume rm {query}'),
],
'uptodate': [
fmt('[ -z "{query}" ] && exit 0 || exit 1'),
],
}
def _update_requests_status(self, request_ids, status, bug):
app.logger.debug(fmt('_update_requests_status:\n{locals}'))
paths = [
fmt('request/{request_id}/status') for request_id in request_ids
]
jsons = [dict(status=status, processor_comment=bug)]
app.logger.debug(
fmt('calling digicert api with paths={paths} and jsons={jsons}'))
calls = self.puts(paths=paths, jsons=jsons)
for call in calls:
if call.recv.status != 204:
if call.recv.json.errors[0].code != 'request_already_processed':
raise ApproveCertificateError(call)
return True
def _order_certificate(self, common_name, csr, sans=None):
app.logger.info(fmt('_order_certificate:\n{locals}'))
path = 'order/certificate/ssl_plus'
json = merge(self.cfg.template,
dict(certificate=dict(common_name=common_name, csr=csr)))
if sans:
path = 'order/certificate/ssl_multi_domain'
json = merge(json, dict(certificate=dict(dns_names=sans)))
app.logger.debug(
fmt('calling digicert api with path={path} and json={json}'))
call = self.post(path=path, json=json)
if call.recv.status == 201:
return call.recv.json.id, call.recv.json.requests[0].id
raise OrderCertificateError(call)
def generate_content(self):
if_predicate = fmt('if hash {0} 2> /dev/null; then\n', self.program)
else_clause = fmt('else\n echo "missing {0}"\nfi', self.program)
clone_github = 'git clone https://github.com'
body = ['mkdir -p ' + self.reporoot]
for reponame in self.reponames:
repopath = os.path.join(self.reporoot, reponame)
body += [
fmt('[ -d "{repopath}" ] || {clone_github}/{reponame} {repopath}'
)
]
body += [fmt('(cd {repopath} && git pull && git checkout HEAD)')]
return if_predicate + join(body, prefix=' ', sepby='\n',
tail='\n') + else_clause
def get_destinations(destinations=None, **kwargs):
d = []
if destinations is not None:
for k, v in destinations.items():
for i in v.keys():
d += [fmt('{k}:{i}')]
return d
def create_endpoint(method, cfg, args):
if cfg is None:
cfg = CFG
endpoint = command2endpoint[args['command']]
app.logger.debug(
fmt('create_endpoint: endpoint={0} args={1}', endpoint, args))
return endpoint(cfg, args)
def __init__(self, dests, paths):
len_dests = len(dests) if isinstance(dests, list) else None
len_paths = len(paths) if isinstance(paths, list) else None
message = fmt(
'len(dests) -> {len_dests} != len(paths) -> {len_paths}; dests={dests}, paths={paths}'
)
super(DestsDontMatchPathsError, self).__init__(message)
def create_certificate(self,
organization_name,
common_name,
validity_years,
csr,
bug,
sans=None,
repeat_delta=None,
whois_check=False):
app.logger.info(fmt('create_certificate:\n{locals}'))
if not sans:
sans = []
organization_id, container_id = self._get_organization_container_ids(
organization_name)
path, json = self._prepare_path_json(organization_id,
container_id,
common_name,
validity_years,
csr,
bug,
sans=sans,
whois_check=whois_check)
crts, expiries, order_ids = self._create_certificates([path], [json],
bug,
repeat_delta)
authority = dict(digicert=dict(order_id=order_ids[0]))
return crts[0], expiries[0], authority
def __init__(self, jsons, paths):
len_jsons = len(jsons) if isinstance(jsons, list) else None
len_paths = len(paths) if isinstance(paths, list) else None
msg = fmt(
'len(jsons) -> {len_jsons} != len(paths) -> {len_paths}; jsons={jsons}, paths={paths}'
)
super(JsonsDontMatchPathsError, self).__init__(msg)
def check_hash(program):
from subprocess import check_call, CalledProcessError, PIPE
try:
check_call(fmt('hash {program}'), shell=True, stdout=PIPE, stderr=PIPE)
return True
except CalledProcessError:
return False
def _prepare_path_json(self,
organization_id,
container_id,
common_name,
validity_years,
csr,
bug,
sans=None,
whois_check=False,
renewal_of_order_id=None):
app.logger.debug(fmt('_prepare_path_json:\n{locals}'))
domains_to_check = self._domains_to_check(common_name, sans)
self._validate_domains(organization_id, container_id, domains_to_check,
whois_check)
path = 'order/certificate/ssl_plus'
json = merge(
self.cfg.template,
dict(validity_years=validity_years,
certificate=dict(common_name=common_name, csr=csr),
organization=dict(id=organization_id),
comments=bug))
if common_name.startswith('*.'):
path = 'order/certificate/ssl_wildcard'
elif sans:
path = 'order/certificate/ssl_multi_domain'
json = merge(json, dict(certificate=dict(dns_names=sans)))
if renewal_of_order_id:
json = merge(json, dict(renewal_of_order_id=renewal_of_order_id))
return path, json
def _prepare_paths_jsons_for_renewals(self,
certs,
organization_id,
container_id,
bug,
validity_years,
sans_to_add,
whois_check=False):
app.logger.debug(fmt('_prepare_paths_jsons_for_renewals:\n{locals}'))
order_ids = [cert.authority['digicert']['order_id'] for cert in certs]
calls = self._get_certificate_order_detail(order_ids)
paths = []
jsons = []
for cert, call in zip(certs, calls):
cert.sans = combine_sans(cert.sans, sans_to_add)
path, json = self._prepare_path_json(
organization_id,
container_id,
cert.common_name,
validity_years,
cert.csr,
bug,
sans=cert.sans,
whois_check=whois_check,
renewal_of_order_id=cert.authority['digicert']['order_id'])
paths += [path]
jsons += [json]
return paths, jsons
def _domains_to_check(self, common_name, sans):
app.logger.debug(fmt('_domains_to_check:\n{locals}'))
domains_to_check = [
(common_name[2:] if common_name.startswith('*.') else common_name)
]
domains_to_check += sans if sans else []
return list(set(domains_to_check))
def install_certificates(self, note, certs, *dests):
paths, jsons = zip(*[(ZEUS_PATH+cert.friendly_common_name, compose_json(cert.key, cert.csr, cert.crt, note)) for cert in certs])
app.logger.info(fmt('install_certificates:\n{locals}'))
calls = self.puts(paths=paths, dests=dests, jsons=jsons, verify_ssl=False)
certs = self.fetch_certificates(certs, *dests)
return certs
def _create_modhash(key):
modulus_int = key.private_numbers().public_numbers.n
modulus_hex = hex(modulus_int).rstrip('L').lstrip('0x').upper()
modulus_bytes = fmt('Modulus={modulus_hex}\n').encode('utf-8')
md5 = hashlib.md5()
md5.update(modulus_bytes)
return md5.hexdigest()
def _order_certificates(self, paths, jsons):
app.logger.debug(fmt('_order_certificates:\n{locals}'))
calls = self.posts(paths=paths, jsons=jsons)
for call in calls:
if call.recv.status != 201:
raise OrderCertificateError(call)
return zip(*[(call.recv.json.id, call.recv.json.requests[0].id)
for call in calls])
def _revoke_certificates(self, paths, jsons, bug):
app.logger.debug(fmt('_revoke_certificates:\n{locals}'))
calls = self.puts(paths=paths, jsons=jsons)
for call in calls:
if call.recv.status != 201:
raise RevokeCertificateError(call)
request_ids = [call.recv.json.id for call in calls]
self._update_requests_status(request_ids, 'approved', bug)
def create_endpoint(method, cfg, verbosity):
if cfg is None:
cfg = CFG
endpoint = method2endpoint[method]
app.logger.debug(
fmt('create_endpoint: verbosity={0} endpoint={1}', verbosity,
endpoint))
return endpoint(cfg, verbosity)
def initialize():
from logging.config import dictConfig
from config import CFG
if sys.argv[0] != 'venv/bin/pytest':
dictConfig(CFG.logging) #3
PID = os.getpid()
PPID = os.getppid()
USER = pwd.getpwuid(os.getuid())[0]
app.logger.info(fmt('starting api with pid={PID}, ppid={PPID} by user={USER}'))
def _get_organization_container_ids(self, organization_name):
app.logger.debug(fmt('_get_organization_container_ids:\n{locals}'))
path = 'organization'
call = self.get(path)
if call.recv.status != 200:
raise DigicertError(call)
for organization in call.recv.json.organizations:
if organization.name == organization_name:
return organization.id, organization.container.id
raise OrganizationNameNotFoundError(organization_name)
