def rotate_all_pending_endpoints():
"""."""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
logger = logging.getLogger(function)
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
logger.debug("Skipping task: Task is already active", extra=log_data)
return
pending_endpoints = endpoint_service.get_all_pending_rotation()
rotate_endpoint_tasks = []
for endpoint in pending_endpoints:
new_cert = endpoint.certificate.replaced[0]
# verify that the certificate has been uploaded before rotating the endpoints
if not all([
dest.plugin.verify(new_cert.name, dest.options)
for dest in new_cert.destinations
]):
logger.warning(
"Certificate has not been uploaded to all destinations, skipping rotate"
)
continue
project, lb, old_cert_name = endpoint.name.split("/")
task = rotate_endpoint.s(
endpoint.id,
lb=lb,
project=project,
new_cert=new_cert.name,
old_cert=old_cert_name,
)
rotate_endpoint_tasks.append(task)
logger.info(
f"Creating tasks to rotate {len(rotate_endpoint_tasks)} endpoints.")
# create task group and skew execution
skew_config = current_app.config.get("CELERY_ROTATE_ENDPOINT_SKEW", {})
start = skew_config.get("start",
0) # seconds until execution of first task
stop = skew_config.get(
"stop", None) # maximum number of seconds to wait for a task
step = skew_config.get(
"step", 10
) # number of seconds to wait until next task is executed (stop might override this)
logger.debug(
f"Scheduling endpoint rotations (start={start}, stop={stop}, step={step})"
)
g = group(rotate_endpoint_tasks).skew(start, stop, step)
g.apply_async()
def rotate(endpoint_name, new_certificate_name, old_certificate_name, message,
commit):
"""
Rotates an endpoint and reissues it if it has not already been replaced. If it has
been replaced, will use the replacement certificate for the rotation.
"""
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting endpoint rotation.")
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print("[+] Rotating endpoint: {0} to certificate {1}".format(
endpoint.name, new_cert.name))
request_rotation(endpoint, new_cert, message, commit)
elif old_cert and new_cert:
print("[+] Rotating all endpoints from {0} to {1}".format(
old_cert.name, new_cert.name))
for endpoint in old_cert.endpoints:
print("[+] Rotating {0}".format(endpoint.name))
request_rotation(endpoint, new_cert, message, commit)
else:
print(
"[+] Rotating all endpoints that have new certificates available"
)
for endpoint in endpoint_service.get_all_pending_rotation():
if len(endpoint.certificate.replaced) == 1:
print("[+] Rotating {0} to {1}".format(
endpoint.name, endpoint.certificate.replaced[0].name))
request_rotation(endpoint,
endpoint.certificate.replaced[0], message,
commit)
else:
metrics.send('endpoint_rotation_failure', 'counter', 1)
print(
"[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found."
.format(endpoint.name))
print("[+] Done!")
except Exception as e:
sentry.captureException()
def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, commit):
"""
Rotates an endpoint and reissues it if it has not already been replaced. If it has
been replaced, will use the replacement certificate for the rotation.
"""
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting endpoint rotation.")
status = FAILURE_METRIC_STATUS
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
request_rotation(endpoint, new_cert, message, commit)
elif old_cert and new_cert:
print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
for endpoint in old_cert.endpoints:
print("[+] Rotating {0}".format(endpoint.name))
request_rotation(endpoint, new_cert, message, commit)
else:
print("[+] Rotating all endpoints that have new certificates available")
for endpoint in endpoint_service.get_all_pending_rotation():
if len(endpoint.certificate.replaced) == 1:
print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
else:
metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
endpoint.name
))
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
except Exception as e:
sentry.captureException()
metrics.send('endpoint_rotation_job', 'counter', 1, metric_tags={'status': status})
def rotate_region(endpoint_name, new_certificate_name, old_certificate_name,
message, commit, region):
"""
Rotates an endpoint in a defined region it if it has not already been replaced. If it has
been replaced, will use the replacement certificate for the rotation.
:param old_certificate_name: Name of the certificate you wish to rotate.
:param new_certificate_name: Name of the certificate you wish to rotate to.
:param endpoint_name: Name of the endpoint you wish to rotate.
:param message: Send a rotation notification to the certificates owner.
:param commit: Persist changes.
:param region: Region in which to rotate the endpoint.
"""
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting endpoint rotation.")
status = FAILURE_METRIC_STATUS
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"region": region,
}
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = new_cert.name
request_rotation_region(endpoint, new_cert, message, commit,
log_data, region)
elif old_cert and new_cert:
log_data["certificate"] = new_cert.name
log_data["certificate_old"] = old_cert.name
log_data["message"] = "Rotating endpoint from old to new cert"
print(log_data)
current_app.logger.info(log_data)
for endpoint in old_cert.endpoints:
log_data["endpoint"] = endpoint.dnsname
request_rotation_region(endpoint, new_cert, message, commit,
log_data, region)
else:
log_data[
"message"] = "Rotating all endpoints that have new certificates available"
print(log_data)
current_app.logger.info(log_data)
all_pending_rotation_endpoints = endpoint_service.get_all_pending_rotation(
)
for endpoint in all_pending_rotation_endpoints:
log_data["endpoint"] = endpoint.dnsname
if region not in endpoint.dnsname:
log_data["message"] = "Skipping rotation, region mismatch"
print(log_data)
current_app.logger.info(log_data)
metrics.send(
"endpoint_rotation_region_skipped",
"counter",
1,
metric_tags={
"region":
region,
"old_certificate_name":
str(old_cert),
"new_certificate_name":
str(endpoint.certificate.replaced[0].name),
"endpoint_name":
str(endpoint.dnsname),
},
)
if len(endpoint.certificate.replaced) == 1:
log_data["certificate"] = endpoint.certificate.replaced[
0].name
log_data["message"] = "Rotating all endpoints in region"
print(log_data)
current_app.logger.info(log_data)
request_rotation(endpoint,
endpoint.certificate.replaced[0], message,
commit)
status = SUCCESS_METRIC_STATUS
else:
status = FAILURE_METRIC_STATUS
log_data[
"message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
print(log_data)
current_app.logger.info(log_data)
metrics.send(
"endpoint_rotation_region",
"counter",
1,
metric_tags={
"status":
FAILURE_METRIC_STATUS,
"old_certificate_name":
str(old_cert),
"new_certificate_name":
str(endpoint.certificate.replaced[0].name),
"endpoint_name":
str(endpoint.dnsname),
"message":
str(message),
"region":
str(region),
},
)
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
except Exception as e:
sentry.captureException(
extra={
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint": str(endpoint_name),
"message": str(message),
"region": str(region),
})
metrics.send(
"endpoint_rotation_region_job",
"counter",
1,
metric_tags={
"status": status,
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
"endpoint": str(globals().get("endpoint")),
"region": str(region),
},
)
def rotate(endpoint_name, new_certificate_name, old_certificate_name, message,
commit):
"""
Rotates an endpoint and reissues it if it has not already been replaced. If it has
been replaced, will use the replacement certificate for the rotation.
"""
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting endpoint rotation.")
status = FAILURE_METRIC_STATUS
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
}
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
log_data["message"] = "Rotating endpoint"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = new_cert.name
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
elif old_cert and new_cert:
print(
f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}"
)
log_data["message"] = "Rotating all endpoints"
log_data["certificate"] = new_cert.name
log_data["certificate_old"] = old_cert.name
log_data["message"] = "Rotating endpoint from old to new cert"
for endpoint in old_cert.endpoints:
print(f"[+] Rotating {endpoint.name}")
log_data["endpoint"] = endpoint.dnsname
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
else:
print(
"[+] Rotating all endpoints that have new certificates available"
)
log_data[
"message"] = "Rotating all endpoints that have new certificates available"
for endpoint in endpoint_service.get_all_pending_rotation():
log_data["endpoint"] = endpoint.dnsname
if len(endpoint.certificate.replaced) == 1:
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
log_data["certificate"] = endpoint.certificate.replaced[
0].name
request_rotation(endpoint,
endpoint.certificate.replaced[0], message,
commit)
current_app.logger.info(log_data)
else:
log_data[
"message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
print(log_data)
metrics.send(
"endpoint_rotation",
"counter",
1,
metric_tags={
"status":
FAILURE_METRIC_STATUS,
"old_certificate_name":
str(old_cert),
"new_certificate_name":
str(endpoint.certificate.replaced[0].name),
"endpoint_name":
str(endpoint.name),
"message":
str(message),
},
)
print(
f"[!] Failed to rotate endpoint {endpoint.name} reason: "
"Multiple replacement certificates found.")
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
except Exception as e:
sentry.captureException(
extra={
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
})
metrics.send(
"endpoint_rotation_job",
"counter",
1,
metric_tags={
"status": status,
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
"endpoint": str(globals().get("endpoint")),
},
)
def rotate(endpoint_name, new_certificate_name, old_certificate_name, message,
commit):
"""
Rotates an endpoint and reissues it if it has not already been replaced. If it has
been replaced, will use the replacement certificate for the rotation.
"""
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting endpoint rotation.")
status = FAILURE_METRIC_STATUS
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
request_rotation(endpoint, new_cert, message, commit)
elif old_cert and new_cert:
print(
f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}"
)
for endpoint in old_cert.endpoints:
print(f"[+] Rotating {endpoint.name}")
request_rotation(endpoint, new_cert, message, commit)
else:
print(
"[+] Rotating all endpoints that have new certificates available"
)
for endpoint in endpoint_service.get_all_pending_rotation():
if len(endpoint.certificate.replaced) == 1:
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
request_rotation(endpoint,
endpoint.certificate.replaced[0], message,
commit)
else:
metrics.send(
"endpoint_rotation",
"counter",
1,
metric_tags={
"status":
FAILURE_METRIC_STATUS,
"old_certificate_name":
str(old_cert),
"new_certificate_name":
str(endpoint.certificate.replaced[0].name),
"endpoint_name":
str(endpoint.name),
"message":
str(message),
},
)
print(
f"[!] Failed to rotate endpoint {endpoint.name} reason: "
"Multiple replacement certificates found.")
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
except Exception as e:
sentry.captureException(
extra={
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
})
metrics.send(
"endpoint_rotation_job",
"counter",
1,
metric_tags={
"status": status,
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
"endpoint": str(globals().get("endpoint")),
},
)
def rotate(endpoint_name, new_certificate_name, old_certificate_name, message,
commit):
"""
Rotates an endpoint and reissues it if it has not already been replaced. If it has
been replaced, will use the replacement certificate for the rotation.
"""
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting endpoint rotation.")
status = FAILURE_METRIC_STATUS
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
}
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
log_data["message"] = "Rotating one endpoint"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = new_cert.name
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
elif old_cert and new_cert:
print(
f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}"
)
log_data["certificate"] = new_cert.name
log_data["certificate_old"] = old_cert.name
log_data["message"] = "Rotating endpoint from old to new cert"
for endpoint in old_cert.endpoints:
print(f"[+] Rotating {endpoint.name}")
log_data["endpoint"] = endpoint.dnsname
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
else:
# No certificate name or endpoint is provided. We will now fetch all endpoints,
# which are associated with a certificate that has been replaced
print(
"[+] Rotating all endpoints that have new certificates available"
)
for endpoint in endpoint_service.get_all_pending_rotation():
log_data["message"] = "Rotating endpoint from old to new cert"
if len(endpoint.certificate.replaced) > 1:
log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
f"{len(endpoint.certificate.replaced)}"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = endpoint.certificate.replaced[0].name
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
request_rotation(endpoint, endpoint.certificate.replaced[0],
message, commit)
current_app.logger.info(log_data)
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
except Exception as e:
capture_exception(
extra={
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
})
metrics.send(
"endpoint_rotation_job",
"counter",
1,
metric_tags={
"status": status,
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
"endpoint": str(globals().get("endpoint")),
},
)
