def fetch(ids): """ Attempt to get full certificates for each pending certificate listed. Args: ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) `python manager.py pending_certs fetch -i 123 321 all` """ pending_certs = pending_certificate_service.get_pending_certs(ids) user = user_service.get_by_username('lemur') new = 0 failed = 0 for cert in pending_certs: authority = plugins.get(cert.authority.plugin_name) real_cert = authority.get_ordered_certificate(cert) if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and delete # the pending certificate pending_certificate_service.create_certificate( cert, real_cert, user) pending_certificate_service.delete(cert) # add metrics to metrics extension new += 1 else: pending_certificate_service.increment_attempt(cert) failed += 1 print("[+] Certificates: New: {new} Failed: {failed}".format( new=new, failed=failed, ))
def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { "function": function, "message": "Starting job.", "task_id": task_id, } if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return pending_certs = pending_certificate_service.get_pending_certs("all") # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): log_data["pending_cert_id"] = cert.id log_data["pending_cert_name"] = cert.name log_data["message"] = "Deleting pending certificate" current_app.logger.debug(log_data) pending_certificate_service.delete(cert) metrics.send(f"{function}.success", "counter", 1) return log_data
def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" pending_certs = pending_certificate_service.get_pending_certs('all') # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': if cert.last_updated == cert.date_created or datetime.datetime.now( timezone.utc) - cert.last_updated > datetime.timedelta( minutes=3): fetch_acme_cert.delay(cert.id)
def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" log_data = {"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)} pending_certs = pending_certificate_service.get_pending_certs("all") # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): log_data["pending_cert_id"] = cert.id log_data["pending_cert_name"] = cert.name log_data["message"] = "Deleting pending certificate" current_app.logger.debug(log_data) pending_certificate_service.delete(cert)
def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } pending_certs = pending_certificate_service.get_pending_certs('all') # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): log_data['pending_cert_id'] = cert.id log_data['pending_cert_name'] = cert.name log_data['message'] = "Deleting pending certificate" current_app.logger.debug(log_data) pending_certificate_service.delete(cert.id)
def fetch_all_acme(): """ Attempt to get full certificates for each pending certificate listed with the acme-issuer. This is more efficient for acme-issued certificates because it will configure all of the DNS challenges prior to resolving any certificates. """ pending_certs = pending_certificate_service.get_pending_certs('all') user = user_service.get_by_username('lemur') new = 0 failed = 0 wrong_issuer = 0 acme_certs = [] # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': acme_certs.append(cert) else: wrong_issuer += 1 authority = plugins.get("acme-issuer") resolved_certs = authority.get_ordered_certificates(acme_certs) for cert in resolved_certs: real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get( cert.get("pending_cert").id) if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and delete # the pending certificate pending_certificate_service.create_certificate( pending_cert, real_cert, user) pending_certificate_service.delete_by_id(pending_cert.id) # add metrics to metrics extension new += 1 else: pending_certificate_service.increment_attempt(pending_cert) failed += 1 print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}" .format(new=new, failed=failed, wrong_issuer=wrong_issuer))
def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "Starting job.", } pending_certs = pending_certificate_service.get_pending_certs("all") # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): log_data["pending_cert_id"] = cert.id log_data["pending_cert_name"] = cert.name log_data["message"] = "Deleting pending certificate" current_app.logger.debug(log_data) pending_certificate_service.delete(cert) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1)
def fetch(ids): """ Attempt to get full certificate for each pending certificate listed. Args: ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) `python manager.py pending_certs fetch -i 123 321 all` """ pending_certs = pending_certificate_service.get_pending_certs(ids) new = 0 failed = 0 for cert in pending_certs: authority = plugins.get(cert.authority.plugin_name) real_cert = authority.get_ordered_certificate(cert) if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and mark # the pending certificate as resolved final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user) pending_certificate_service.update( cert.id, resolved=True ) pending_certificate_service.update( cert.id, resolved_cert_id=final_cert.id ) # add metrics to metrics extension new += 1 else: pending_certificate_service.increment_attempt(cert) failed += 1 print( "[+] Certificates: New: {new} Failed: {failed}".format( new=new, failed=failed, ) )
def fetch_acme_cert(id): """ Attempt to get the full certificate for the pending certificate listed. Args: id: an id of a PendingCertificate """ task_id = None if celery.current_task: task_id = celery.current_task.request.id function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "Resolving pending certificate {}".format(id), "task_id": task_id, "id": id, } current_app.logger.debug(log_data) if task_id and is_task_active(log_data["function"], task_id, (id, )): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 wrong_issuer = 0 acme_certs = [] # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == "acme-issuer": acme_certs.append(cert) else: wrong_issuer += 1 authority = plugins.get("acme-issuer") resolved_certs = authority.get_ordered_certificates(acme_certs) for cert in resolved_certs: real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get( cert.get("pending_cert").id) if not pending_cert or pending_cert.resolved: # pending_cert is cleared or it was resolved by another process log_data[ "message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" current_app.logger.error(log_data) continue if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and mark # the pending certificate as resolved final_cert = pending_certificate_service.create_certificate( pending_cert, real_cert, pending_cert.user) pending_certificate_service.update(cert.get("pending_cert").id, resolved_cert_id=final_cert.id) pending_certificate_service.update(cert.get("pending_cert").id, resolved=True) # add metrics to metrics extension new += 1 else: failed += 1 error_log = copy.deepcopy(log_data) error_log["message"] = "Pending certificate creation failure" error_log["pending_cert_id"] = pending_cert.id error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify) # Mark the pending cert as resolved pending_certificate_service.update(cert.get("pending_cert").id, resolved=True) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update(cert.get("pending_cert").id, status=str( cert.get("last_error"))) # Add failed pending cert task back to queue fetch_acme_cert.delay(id) current_app.logger.error(error_log) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) metrics.send(f"{function}.resolved", "gauge", new) metrics.send(f"{function}.failed", "gauge", failed) metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}" .format(new=new, failed=failed, wrong_issuer=wrong_issuer)) return log_data
def fetch_acme_cert(id): """ Attempt to get the full certificate for the pending certificate listed. Args: id: an id of a PendingCertificate """ log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), "message": "Resolving pending certificate {}".format(id) } current_app.logger.debug(log_data) pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 wrong_issuer = 0 acme_certs = [] # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': acme_certs.append(cert) else: wrong_issuer += 1 authority = plugins.get("acme-issuer") resolved_certs = authority.get_ordered_certificates(acme_certs) for cert in resolved_certs: real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) if not pending_cert: log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" current_app.logger.error(log_data) continue if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and mark # the pending certificate as resolved final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) pending_certificate_service.update( cert.get("pending_cert").id, resolved=True ) pending_certificate_service.update( cert.get("pending_cert").id, resolved_cert_id=final_cert.id ) # add metrics to metrics extension new += 1 else: failed += 1 error_log = copy.deepcopy(log_data) error_log["message"] = "Pending certificate creation failure" error_log["pending_cert_id"] = pending_cert.id error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) # Mark the pending cert as resolved pending_certificate_service.update( cert.get("pending_cert").id, resolved=True ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( cert.get("pending_cert").id, status=str(cert.get("last_error")) ) # Add failed pending cert task back to queue fetch_acme_cert.delay(id) current_app.logger.error(error_log) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, failed=failed, wrong_issuer=wrong_issuer ) )
def fetch_cert(id): """ Attempt to get the full certificate for the pending certificate listed. Args: id: an id of a PendingCertificate """ task_id = None if celery.current_task: task_id = celery.current_task.request.id function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "Resolving pending certificate {}".format(id), "task_id": task_id, "id": id, } current_app.logger.debug(log_data) if task_id and is_task_active(log_data["function"], task_id, (id, )): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == "acme-issuer": current_app.logger.warning( "Skipping acme cert (use `fetch_acme_cert()` instead).") continue plugin = plugins.get(cert_authority.plugin_name) real_cert = plugin.get_ordered_certificate(cert) if real_cert: # If a real certificate was returned from issuer, then create it in # Lemur and mark the pending certificate as resolved # Ideally, this should be a db transaction that would check resolved status # before creating a new one final_cert = pending_certificate_service.create_certificate( cert, real_cert, cert.user) pending_certificate_service.update(cert.id, resolved_cert_id=final_cert.id) pending_certificate_service.update(cert.id, resolved=True) # add metrics to metrics extension new += 1 else: # Double check the pending certificate still exists in the database # before updating the failed attempt counter if not pending_certificate_service.get(cert.id): current_app.logger.info( f"Not incrementing failed attempt counter because {cert.name} was cancelled before." ) else: pending_certificate_service.increment_attempt(cert) failed += 1 log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed current_app.logger.debug(log_data) metrics.send(f"{function}.resolved", "gauge", new) metrics.send(f"{function}.failed", "gauge", failed) print(f"[+] Certificates: New: {new} Failed: {failed}") return log_data
def fetch_all_acme(): """ Attempt to get full certificates for each pending certificate listed with the acme-issuer. This is more efficient for acme-issued certificates because it will configure all of the DNS challenges prior to resolving any certificates. """ log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } pending_certs = pending_certificate_service.get_pending_certs('all') user = user_service.get_by_username('lemur') new = 0 failed = 0 wrong_issuer = 0 acme_certs = [] # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': acme_certs.append(cert) else: wrong_issuer += 1 authority = plugins.get("acme-issuer") resolved_certs = authority.get_ordered_certificates(acme_certs) for cert in resolved_certs: real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get( cert.get("pending_cert").id) if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and delete # the pending certificate pending_certificate_service.create_certificate( pending_cert, real_cert, user) pending_certificate_service.delete_by_id(pending_cert.id) # add metrics to metrics extension new += 1 else: failed += 1 error_log = copy.deepcopy(log_data) error_log["message"] = "Pending certificate creation failure" error_log["pending_cert_id"] = pending_cert.id error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify) pending_certificate_service.delete_by_id(pending_cert.id) current_app.logger.error(error_log) pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( cert.get("pending_cert").id, status=str(cert.get("last_error"))[0:128]) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}" .format(new=new, failed=failed, wrong_issuer=wrong_issuer))
def fetch_acme_cert(id): """ Attempt to get the full certificate for the pending certificate listed. Args: id: an id of a PendingCertificate """ log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 wrong_issuer = 0 acme_certs = [] # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': acme_certs.append(cert) else: wrong_issuer += 1 authority = plugins.get("acme-issuer") resolved_certs = authority.get_ordered_certificates(acme_certs) for cert in resolved_certs: real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get( cert.get("pending_cert").id) if not pending_cert: log_data[ "message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" current_app.logger.error(log_data) continue if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and mark # the pending certificate as resolved final_cert = pending_certificate_service.create_certificate( pending_cert, real_cert, pending_cert.user) pending_certificate_service.update(cert.get("pending_cert").id, resolved=True) pending_certificate_service.update(cert.get("pending_cert").id, resolved_cert_id=final_cert.id) # add metrics to metrics extension new += 1 else: failed += 1 error_log = copy.deepcopy(log_data) error_log["message"] = "Pending certificate creation failure" error_log["pending_cert_id"] = pending_cert.id error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify) # Mark the pending cert as resolved pending_certificate_service.update(cert.get("pending_cert").id, resolved=True) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update(cert.get("pending_cert").id, status=str( cert.get("last_error"))) # Add failed pending cert task back to queue fetch_acme_cert.delay(id) current_app.logger.error(error_log) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}" .format(new=new, failed=failed, wrong_issuer=wrong_issuer))