def add_aws_destination_to_sources(dst):
"""
Given a destination check, if it can be added as sources, and included it if not already a source
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reveals the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
:return: true for success and false for not adding the destination as source
"""
# a set of all accounts numbers available as sources
src_accounts = set()
sources = get_all()
for src in sources:
src_accounts.add(get_plugin_option('accountNumber', src.options))
# check
destination_plugin = plugins.get(dst.plugin_name)
account_number = get_plugin_option('accountNumber', dst.options)
if account_number is not None and \
destination_plugin.sync_as_source is not None and \
destination_plugin.sync_as_source and \
(account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
set_plugin_option('accountNumber', account_number, src_options)
create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
return True
return False
def sync(labels=None):
new, updated = 0, 0
c_certificates = cert_service.get_all_certs()
for source in database.get_all(Source, True, field='active'):
# we should be able to specify, individual sources to sync
if labels:
if source.label not in labels:
continue
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = cert_service.find_duplicates(certificate['body'])
if not exists:
sync_create(certificate, source)
new += 1
# check to make sure that existing certificates have the current source associated with it
elif len(exists) == 1:
sync_update(exists[0], source)
updated += 1
else:
current_app.logger.warning(
"Multiple certificates found, attempt to deduplicate the following certificates: {0}".format(
",".join([x.name for x in exists])
)
)
# we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
_disassociate_certs_from_source(c_certificates, certificates, source)
def sync_certificates(source, user):
new, updated = 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = certificate_service.get_by_name(certificate['name'])
if not certificate.get('owner'):
certificate['owner'] = user.email
certificate['creator'] = user
if not exists:
current_app.logger.debug("Creating Certificate. Name: {name}".format(name=certificate['name']))
certificate_create(certificate, source)
new += 1
else:
current_app.logger.debug("Updating Certificate. Name: {name}".format(name=certificate['name']))
certificate_update(exists, source)
updated += 1
assert len(certificates) == new + updated
return new, updated
def mint(issuer_options):
"""
Minting is slightly different for each authority.
Support for multiple authorities is handled by individual plugins.
:param issuer_options:
"""
authority = issuer_options['authority']
issuer = plugins.get(authority.plugin_name)
# allow the CSR to be specified by the user
if not issuer_options.get('csr'):
csr, private_key = create_csr(issuer_options)
else:
csr = str(issuer_options.get('csr'))
private_key = None
issuer_options['creator'] = g.user.email
cert_body, cert_chain = issuer.create_certificate(csr, issuer_options)
cert = Certificate(cert_body, private_key, cert_chain)
cert.user = g.user
cert.authority = authority
database.update(cert)
return cert, private_key, cert_chain,
def get(self, name):
"""
.. http:get:: /plugins/<name>
The current plugin list
**Example request**:
.. sourcecode:: http
GET /plugins HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"accountNumber": 222222222,
"label": "account2",
"description": "this is a thing"
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
"""
return plugins.get(name)
def clear_pending():
"""
Function clears all pending certificates.
:return:
"""
v = plugins.get('verisign-issuer')
v.clear_pending_certificates()
def sync_certificates(source):
new, updated = 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = cert_service.find_duplicates(certificate['body'])
if not exists:
certificate_create(certificate, source)
new += 1
# check to make sure that existing certificates have the current source associated with it
elif len(exists) == 1:
certificate_update(exists[0], source)
updated += 1
else:
current_app.logger.warning(
"Multiple certificates found, attempt to deduplicate the following certificates: {0}".format(
",".join([x.name for x in exists])
)
)
# we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
_disassociate_certs_from_source(certificates, source)
def test_export_keystore(app):
from lemur.plugins.base import plugins
p = plugins.get('java-keystore-jks')
options = [
{'name': 'passphrase', 'value': 'hunter2'},
{'name': 'alias', 'value': 'AzureDiamond'},
]
chain = INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR
with pytest.raises(Exception):
p.export(INTERNAL_CERTIFICATE_A_STR, chain, '', options)
ext, password, raw = p.export(SAN_CERT_STR, chain, SAN_CERT_KEY, options)
assert ext == 'jks'
assert password == 'hunter2'
assert isinstance(raw, bytes)
ks = KeyStore.loads(raw, password)
assert ks.store_type == 'jks'
# JKS lower-cases alias strings
assert ks.entries.keys() == {'azurediamond'}
entry = ks.entries['azurediamond']
assert isinstance(entry, PrivateKeyEntry)
assert len(entry.cert_chain) == 3 # Cert and chain were provided
def test_export_certificate_to_jks(app):
from lemur.plugins.base import plugins
p = plugins.get("java-export")
options = {"passphrase": "test1234"}
raw = p.export(EXTERNAL_VALID_STR, "", PRIVATE_KEY_STR, options)
assert raw != b""
def test_export_keystore(app):
from lemur.plugins.base import plugins
p = plugins.get('java-keystore-jks')
options = [{'name': 'passphrase', 'value': 'test1234'}]
with pytest.raises(Exception):
p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
assert raw != b""
def test_export_certificate_to_pkcs12(app):
from lemur.plugins.base import plugins
p = plugins.get('openssl-export')
options = [{'name': 'passphrase', 'value': 'test1234'}, {'name': 'type', 'value': 'PKCS12 (.p12)'}]
with pytest.raises(Exception):
p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
assert raw != b""
def test_export_truststore(app):
from lemur.plugins.base import plugins
p = plugins.get('java-truststore-jks')
options = [{'name': 'passphrase', 'value': 'test1234'}]
actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
assert actual[0] == 'jks'
assert actual[1] == 'test1234'
assert isinstance(actual[2], bytes)
def test_export_truststore_default_password(app):
from lemur.plugins.base import plugins
p = plugins.get('java-truststore-jks')
options = []
actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
assert actual[0] == 'jks'
assert isinstance(actual[1], str)
assert isinstance(actual[2], bytes)
def export(self, body, private_key, cert_chain, options):
export_plugin = self.get_option('exportPlugin', options)
if export_plugin:
plugin = plugins.get(export_plugin['slug'])
extension, passphrase, data = plugin.export(body, cert_chain, private_key, export_plugin['plugin_options'])
return [(extension, passphrase, data)]
data = body + '\n' + cert_chain + '\n' + private_key
return [('.pem', '', data)]
def create(kwargs):
"""
Create a new authority.
:rtype : Authority
:return:
"""
issuer = plugins.get(kwargs.get('pluginName'))
kwargs['creator'] = g.current_user.email
cert_body, intermediate, issuer_roles = issuer.create_authority(kwargs)
cert = Certificate(cert_body, chain=intermediate)
cert.owner = kwargs['ownerEmail']
cert.description = "This is the ROOT certificate for the {0} certificate authority".format(kwargs.get('caName'))
cert.user = g.current_user
cert.notifications = notification_service.create_default_expiration_notifications(
'DEFAULT_SECURITY',
current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
)
# we create and attach any roles that the issuer gives us
role_objs = []
for r in issuer_roles:
role = role_service.create(
r['name'],
password=r['password'],
description="{0} auto generated role".format(kwargs.get('pluginName')),
username=r['username'])
# the user creating the authority should be able to administer it
if role.username == 'admin':
g.current_user.roles.append(role)
role_objs.append(role)
authority = Authority(
kwargs.get('caName'),
kwargs['ownerEmail'],
kwargs['pluginName'],
cert_body,
description=kwargs['caDescription'],
chain=intermediate,
roles=role_objs
)
database.update(cert)
authority = database.create(authority)
g.current_user.authorities.append(authority)
return authority
def export(cert, export_plugin):
"""
Exports a certificate to the requested format. This format
may be a binary format.
:param export_plugin:
:param cert:
:return:
"""
plugin = plugins.get(export_plugin['slug'])
return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
def update_destinations(target, value, initiator):
"""
Attempt to upload the new certificate to the new destination
:param target:
:param value:
:param initiator:
:return:
"""
destination_plugin = plugins.get(value.plugin_name)
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
def put(self, certificate_id, data=None):
"""
.. http:put:: /certificates/1/revoke
Revoke a certificate
**Example request**:
.. sourcecode:: http
POST /certificates/1/revoke HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
'id': 1
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
cert = service.get(certificate_id)
if not cert:
return dict(message="Cannot find specified certificate"), 404
# allow creators
if g.current_user != cert.user:
owner_role = role_service.get_by_name(cert.owner)
permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
if not permission.can():
return dict(message='You are not authorized to revoke this certificate.'), 403
if not cert.external_id:
return dict(message='Cannot revoke certificate. No external id found.'), 400
if cert.endpoints:
return dict(message='Cannot revoke certificate. Endpoints are deployed with the given certificate.'), 403
plugin = plugins.get(cert.authority.plugin_name)
plugin.revoke_certificate(cert, data)
log_service.create(g.current_user, 'revoke_cert', certificate=cert)
return dict(id=cert.id)
def cancel(pending_certificate, **kwargs):
"""
Cancel a pending certificate. A check should be done prior to this function to decide to
revoke the certificate or just abort cancelling.
Args:
pending_certificate: PendingCertificate to be cancelled
Returns: the pending certificate if successful, raises Exception if there was an issue
"""
plugin = plugins.get(pending_certificate.authority.plugin_name)
plugin.cancel_ordered_certificate(pending_certificate, **kwargs)
pending_certificate.status = 'Cancelled'
database.update(pending_certificate)
return pending_certificate
def sync_endpoints(source):
new, updated = 0, 0
current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
s = plugins.get(source.plugin_name)
try:
endpoints = s.get_endpoints(source.options)
except NotImplementedError:
current_app.logger.warning("Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'".format(source.label))
return
for endpoint in endpoints:
exists = endpoint_service.get_by_dnsname(endpoint['dnsname'])
certificate_name = endpoint.pop('certificate_name', None)
certificate = endpoint.pop('certificate', None)
if certificate_name:
current_app.logger.debug(certificate_name)
cert = cert_service.get_by_name(certificate_name)
elif certificate:
cert = cert_service.get_by_body(certificate['body'])
if not cert:
cert = cert_service.import_certificate(**certificate)
if not cert:
current_app.logger.error(
"Unable to find associated certificate, be sure that certificates are sync'ed before endpoints")
continue
endpoint['certificate'] = cert
policy = endpoint.pop('policy')
policy_ciphers = []
for nc in policy['ciphers']:
policy_ciphers.append(endpoint_service.get_or_create_cipher(name=nc))
policy['ciphers'] = policy_ciphers
endpoint['policy'] = endpoint_service.get_or_create_policy(**policy)
if not exists:
endpoint_service.create(**endpoint)
new += 1
else:
endpoint_service.update(exists.id, **endpoint)
updated += 1
_disassociate_endpoints_from_source(endpoints, source)
def test_export_keystore(app):
from lemur.plugins.base import plugins
p = plugins.get('java-keystore-jks')
options = [{'name': 'passphrase', 'value': 'test1234'}]
with pytest.raises(Exception):
p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
assert actual[0] == 'jks'
assert actual[1] == 'test1234'
assert isinstance(actual[2], bytes)
def update_destinations(target, value, initiator):
"""
Attempt to upload the new certificate to the new destination
:param target:
:param value:
:param initiator:
:return:
"""
destination_plugin = plugins.get(value.plugin_name)
try:
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
except Exception as e:
current_app.logger.exception(e)
def test_export_keystore_default_password(app):
from lemur.plugins.base import plugins
p = plugins.get('java-keystore-jks')
options = []
with pytest.raises(Exception):
p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
assert actual[0] == 'jks'
assert isinstance(actual[1], str)
assert isinstance(actual[2], bytes)
def test_export_truststore_defaults(app):
from lemur.plugins.base import plugins
p = plugins.get('java-truststore-jks')
options = []
ext, password, raw = p.export(INTERNAL_CERTIFICATE_A_STR, '', '', options)
assert ext == 'jks'
assert isinstance(password, str)
assert isinstance(raw, bytes)
ks = KeyStore.loads(raw, password)
assert ks.store_type == 'jks'
# JKS lower-cases alias strings
assert ks.entries.keys() == {'acommonname_cert'}
assert isinstance(ks.entries['acommonname_cert'], TrustedCertEntry)
def sync_certificates(source, user):
new, updated = 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = False
if certificate.get('search', None):
conditions = certificate.pop('search')
exists = certificate_service.get_by_attributes(conditions)
if not exists and certificate.get('name'):
result = certificate_service.get_by_name(certificate['name'])
if result:
exists = [result]
if not exists and certificate.get('serial'):
exists = certificate_service.get_by_serial(certificate['serial'])
if not exists:
cert = parse_certificate(certificate['body'])
matching_serials = certificate_service.get_by_serial(serial(cert))
exists = find_matching_certificates_by_hash(cert, matching_serials)
if not certificate.get('owner'):
certificate['owner'] = user.email
certificate['creator'] = user
exists = [x for x in exists if x]
if not exists:
certificate_create(certificate, source)
new += 1
else:
for e in exists:
if certificate.get('external_id'):
e.external_id = certificate['external_id']
if certificate.get('authority_id'):
e.authority_id = certificate['authority_id']
certificate_update(e, source)
updated += 1
return new, updated
def update_destinations(target, value, initiator):
"""
Attempt to upload certificate to the new destination
:param target:
:param value:
:param initiator:
:return:
"""
destination_plugin = plugins.get(value.plugin_name)
try:
if target.private_key:
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
except Exception as e:
current_app.logger.exception(e)
metrics.send('destination_upload_failure', 'counter', 1,
metric_tags={'certificate': target.name, 'destination': value.label})
def sync_endpoints(source):
new, updated = 0, 0
current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
s = plugins.get(source.plugin_name)
try:
endpoints = s.get_endpoints(source.options)
except NotImplementedError:
current_app.logger.warning("Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'".format(source.label))
return new, updated
for endpoint in endpoints:
exists = endpoint_service.get_by_dnsname(endpoint['dnsname'])
certificate_name = endpoint.pop('certificate_name')
endpoint['certificate'] = certificate_service.get_by_name(certificate_name)
if not endpoint['certificate']:
current_app.logger.error(
"Certificate Not Found. Name: {0} Endpoint: {1}".format(certificate_name, endpoint['name']))
continue
policy = endpoint.pop('policy')
policy_ciphers = []
for nc in policy['ciphers']:
policy_ciphers.append(endpoint_service.get_or_create_cipher(name=nc))
policy['ciphers'] = policy_ciphers
endpoint['policy'] = endpoint_service.get_or_create_policy(**policy)
endpoint['source'] = source
if not exists:
current_app.logger.debug("Endpoint Created: Name: {name}".format(name=endpoint['name']))
endpoint_service.create(**endpoint)
new += 1
else:
current_app.logger.debug("Endpoint Updated: Name: {name}".format(name=endpoint['name']))
endpoint_service.update(exists.id, **endpoint)
updated += 1
return new, updated
def clean(source_strings, commit):
sources = validate_sources(source_strings)
for source in sources:
s = plugins.get(source.plugin_name)
if not hasattr(s, "clean"):
info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'"
current_app.logger.warning(info_text)
print(info_text)
continue
start_time = time.time()
print("[+] Staring to clean source: {label}!\n".format(
label=source.label))
cleaned = 0
certificates = certificate_service.get_all_pending_cleaning_expired(
source)
for certificate in certificates:
status = FAILURE_METRIC_STATUS
if commit:
status = execute_clean(s, certificate, source)
metrics.send(
"certificate_clean",
"counter",
1,
metric_tags={
"status": status,
"source": source.label,
"certificate": certificate.name
},
)
current_app.logger.warning(
f"Removed {certificate.name} from source {source.label} during cleaning"
)
cleaned += 1
info_text = f"[+] Finished cleaning source: {source.label}. " \
f"Removed {cleaned} certificates from source. " \
f"Run Time: {(time.time() - start_time)}\n"
print(info_text)
current_app.logger.warning(info_text)
def sync_endpoints(source):
new, updated = 0, 0
current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
s = plugins.get(source.plugin_name)
try:
endpoints = s.get_endpoints(source.options)
except NotImplementedError:
current_app.logger.warning("Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'".format(source.label))
return new, updated
for endpoint in endpoints:
exists = endpoint_service.get_by_dnsname_and_port(endpoint['dnsname'], endpoint['port'])
certificate_name = endpoint.pop('certificate_name')
endpoint['certificate'] = certificate_service.get_by_name(certificate_name)
if not endpoint['certificate']:
current_app.logger.error(
"Certificate Not Found. Name: {0} Endpoint: {1}".format(certificate_name, endpoint['name']))
continue
policy = endpoint.pop('policy')
policy_ciphers = []
for nc in policy['ciphers']:
policy_ciphers.append(endpoint_service.get_or_create_cipher(name=nc))
policy['ciphers'] = policy_ciphers
endpoint['policy'] = endpoint_service.get_or_create_policy(**policy)
endpoint['source'] = source
if not exists:
current_app.logger.debug("Endpoint Created: Name: {name}".format(name=endpoint['name']))
endpoint_service.create(**endpoint)
new += 1
else:
current_app.logger.debug("Endpoint Updated: {}".format(endpoint))
endpoint_service.update(exists.id, **endpoint)
updated += 1
return new, updated
def clean(source_strings, commit):
sources = validate_sources(source_strings)
for source in sources:
s = plugins.get(source.plugin_name)
if not hasattr(s, 'clean'):
print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
source.label
))
continue
start_time = time.time()
print("[+] Staring to clean source: {label}!\n".format(label=source.label))
cleaned = 0
for certificate in certificate_service.get_all_pending_cleaning(source):
status = FAILURE_METRIC_STATUS
if commit:
try:
s.clean(certificate, source.options)
certificate.sources.remove(source)
certificate_service.database.update(certificate)
status = SUCCESS_METRIC_STATUS
except Exception as e:
current_app.logger.exception(e)
sentry.captureException()
metrics.send('clean', 'counter', 1, metric_tags={'source': source.label, 'status': status})
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
certificate.name,
source.label
))
cleaned += 1
print(
"[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
label=source.label,
time=(time.time() - start_time),
cleaned=cleaned
)
)
def update_destinations(target, value, initiator):
"""
Attempt to upload certificate to the new destination
:param target:
:param value:
:param initiator:
:return:
"""
destination_plugin = plugins.get(value.plugin_name)
try:
if target.private_key:
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
metrics.send('destination_upload_failure', 'counter', 1,
metric_tags={'certificate': target.name, 'destination': value.label})
def clean(source):
s = plugins.get(source.plugin_name)
try:
certificates = s.clean(source.options)
except NotImplementedError:
current_app.logger.warning(
"Cannot clean source: {0}, source plugin does not implement 'clean()'"
.format(source.label))
return
for certificate in certificates:
cert = cert_service.get_by_name(certificate)
if cert:
current_app.logger.warning(
"Removed {0} from source {1} during cleaning".format(
cert.name, source.label))
cert.sources.remove(source)
def mint(**kwargs):
"""
Minting is slightly different for each authority.
Support for multiple authorities is handled by individual plugins.
"""
authority = kwargs['authority']
issuer = plugins.get(authority.plugin_name)
# allow the CSR to be specified by the user
if not kwargs.get('csr'):
csr, private_key = create_csr(**kwargs)
else:
csr = str(kwargs.get('csr'))
private_key = None
cert_body, cert_chain = issuer.create_certificate(csr, kwargs)
return cert_body, private_key, cert_chain,
def sync_certificates(source, user):
new, updated = 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(
source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = False
if certificate.get('name'):
result = certificate_service.get_by_name(certificate['name'])
if result:
exists = [result]
if not exists and certificate.get('serial'):
exists = certificate_service.get_by_serial(certificate['serial'])
if not exists:
cert = parse_certificate(certificate['body'])
exists = certificate_service.get_by_serial(serial(cert))
if not certificate.get('owner'):
certificate['owner'] = user.email
certificate['creator'] = user
exists = [x for x in exists if x]
if not exists:
certificate_create(certificate, source)
new += 1
else:
for e in exists:
if certificate.get('external_id'):
e.external_id = certificate['external_id']
if certificate.get('authority_id'):
e.authority_id = certificate['authority_id']
certificate_update(e, source)
updated += 1
return new, updated
def clean(source_strings, commit):
sources = validate_sources(source_strings)
for source in sources:
s = plugins.get(source.plugin_name)
if not hasattr(s, 'clean'):
print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
source.label
))
continue
start_time = time.time()
print("[+] Staring to clean source: {label}!\n".format(label=source.label))
cleaned = 0
for certificate in certificate_service.get_all_pending_cleaning(source):
if commit:
try:
s.clean(certificate, source.options)
certificate.sources.remove(source)
certificate_service.database.update(certificate)
metrics.send('clean_success', 'counter', 1, metric_tags={'source': source.label})
except Exception as e:
current_app.logger.exception(e)
metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label})
sentry.captureException()
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
certificate.name,
source.label
))
cleaned += 1
print(
"[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
label=source.label,
time=(time.time() - start_time),
cleaned=cleaned
)
)
def test_export_truststore(app):
from lemur.plugins.base import plugins
p = plugins.get('java-truststore-jks')
options = [
{'name': 'passphrase', 'value': 'hunter2'},
{'name': 'alias', 'value': 'AzureDiamond'},
]
chain = INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR
ext, password, raw = p.export(SAN_CERT_STR, chain, SAN_CERT_KEY, options)
assert ext == 'jks'
assert password == 'hunter2'
assert isinstance(raw, bytes)
ks = KeyStore.loads(raw, 'hunter2')
assert ks.store_type == 'jks'
# JKS lower-cases alias strings
assert ks.entries.keys() == {'azurediamond_cert', 'azurediamond_cert_1', 'azurediamond_cert_2'}
assert isinstance(ks.entries['azurediamond_cert'], TrustedCertEntry)
def get_all_zones():
"""
Retrieves all DNS providers from the database. Refreshes the zones associated with each DNS provider
"""
print("[+] Starting dns provider zone lookup and configuration.")
dns_providers = get_all_dns_providers()
acme_plugin = plugins.get("acme-issuer")
for dns_provider in dns_providers:
try:
zones = acme_plugin.get_all_zones(dns_provider)
set_domains(dns_provider, zones)
except Exception as e:
print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e))
set_domains(dns_provider, [])
status = SUCCESS_METRIC_STATUS
metrics.send("get_all_zones", "counter", 1, metric_tags={"status": status})
print("[+] Done with dns provider zone lookup and configuration.")
def update_destinations(target, value, initiator):
"""
Attempt to upload certificate to the new destination
:param target:
:param value:
:param initiator:
:return:
"""
destination_plugin = plugins.get(value.plugin_name)
status = FAILURE_METRIC_STATUS
try:
if target.private_key:
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
metrics.send('destination_upload', 'counter', 1,
metric_tags={'status': status, 'certificate': target.name, 'destination': value.label})
def worker(data, commit, reason):
parts = [x for x in data.split(' ') if x]
try:
cert = get(int(parts[0].strip()))
plugin = plugins.get(cert.authority.plugin_name)
print('[+] Revoking certificate. Id: {0} Name: {1}'.format(cert.id, cert.name))
if commit:
plugin.revoke_certificate(cert, reason)
metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
except Exception as e:
sentry.captureException()
metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
print(
"[!] Failed to revoke certificates. Reason: {}".format(
e
)
)
def deactivate_entrust_certificates():
"""
Attempt to deactivate test certificates issued by Entrust
"""
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Deactivating Entrust certificates"
}
certificates = get_all_valid_certs(['entrust-issuer'])
entrust_plugin = plugins.get('entrust-issuer')
for cert in certificates:
try:
response = entrust_plugin.deactivate_certificate(cert)
if response == 200:
cert.status = "revoked"
else:
cert.status = "unknown"
log_data["valid"] = cert.status
log_data["certificate_name"] = cert.name
log_data["certificate_id"] = cert.id
metrics.send(
"certificate_deactivate",
"counter",
1,
metric_tags={
"status": log_data["valid"],
"certificate_name": log_data["certificate_name"],
"certificate_id": log_data["certificate_id"]
},
)
current_app.logger.info(log_data)
database.update(cert)
except Exception as e:
current_app.logger.info(log_data)
sentry.captureException()
current_app.logger.exception(e)
def mint(**kwargs):
"""
Minting is slightly different for each authority.
Support for multiple authorities is handled by individual plugins.
"""
authority = kwargs["authority"]
issuer = plugins.get(authority.plugin_name)
# allow the CSR to be specified by the user
if not kwargs.get("csr"):
csr, private_key = create_csr(**kwargs)
csr_created.send(authority=authority, csr=csr)
else:
csr = str(kwargs.get("csr"))
private_key = None
csr_imported.send(authority=authority, csr=csr)
cert_body, cert_chain, external_id = issuer.create_certificate(csr, kwargs)
return cert_body, private_key, cert_chain, external_id, csr
def test_export_keystore_defaults(app):
from lemur.plugins.base import plugins
p = plugins.get('java-keystore-jks')
options = []
with pytest.raises(Exception):
p.export(INTERNAL_CERTIFICATE_A_STR, '', '', options)
ext, password, raw = p.export(SAN_CERT_STR, '', SAN_CERT_KEY, options)
assert ext == 'jks'
assert isinstance(password, str)
assert isinstance(raw, bytes)
ks = KeyStore.loads(raw, password)
assert ks.store_type == 'jks'
assert ks.entries.keys() == {'san.example.org'}
entry = ks.entries['san.example.org']
assert isinstance(entry, PrivateKeyEntry)
assert len(entry.cert_chain) == 1 # Only cert itself, no chain was provided
def retrieve_user_memberships(user_api_url, user_membership_provider,
access_token):
user, profile = retrieve_user(user_api_url, access_token)
if user_membership_provider is None:
return user, profile
"""
Unaware of the usage of this code across the community, current implementation is config driven.
Without USER_MEMBERSHIP_PROVIDER configured, it is backward compatible. Please define a plugin
for custom implementation.
"""
membership_provider = plugins.get(user_membership_provider)
user_membership = {
"email":
profile["email"],
"thumbnailPhotoUrl":
profile["thumbnailPhotoUrl"],
"googleGroups":
membership_provider.retrieve_user_memberships(profile["userId"])
}
return user, user_membership
def fetch(ids):
"""
Attempt to get full certificate for each pending certificate listed.
Args:
ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
`python manager.py pending_certs fetch -i 123 321 all`
"""
pending_certs = pending_certificate_service.get_pending_certs(ids)
new = 0
failed = 0
for cert in pending_certs:
authority = plugins.get(cert.authority.plugin_name)
real_cert = authority.get_ordered_certificate(cert)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate as resolved
final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user)
pending_certificate_service.update(
cert.id,
resolved=True
)
pending_certificate_service.update(
cert.id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension
new += 1
else:
pending_certificate_service.increment_attempt(cert)
failed += 1
print(
"[+] Certificates: New: {new} Failed: {failed}".format(
new=new,
failed=failed,
)
)
def retrieve_user_memberships(user_api_url, user_membership_api_url, access_token):
user, profile = retrieve_user(user_api_url, access_token)
if user_membership_api_url is None:
return user, profile
"""
Potentially, below code can be made more generic i.e., plugin driven. Unaware of the usage of this
code across the community, current implementation is config driven. Without user_membership_api_url
configured, it is backward compatible.
"""
tls_provider = plugins.get(current_app.config.get("PING_USER_MEMBERSHIP_TLS_PROVIDER"))
# put user id in url
user_membership_api_url = user_membership_api_url.replace("%user_id%", profile["userId"])
session = tls_provider.session(current_app.config.get("PING_USER_MEMBERSHIP_SERVICE"))
headers = {"Content-Type": "application/json"}
data = {"relation": "DIRECT_ONLY", "groupFilter": {"type": "GOOGLE"}, "size": 500}
user_membership = {"email": profile["email"],
"thumbnailPhotoUrl": profile["thumbnailPhotoUrl"],
"googleGroups": []}
while True:
# retrieve information about the current user memberships
r = session.post(user_membership_api_url, data=json.dumps(data), headers=headers)
if r.status_code == 200:
response = r.json()
membership_details = response["data"]
for membership in membership_details:
user_membership["googleGroups"].append(membership["membership"]["name"])
if "nextPageToken" in response and response["nextPageToken"]:
data["nextPageToken"] = response["nextPageToken"]
else:
break
else:
current_app.logger.error(f"Response Code:{r.status_code} {r.text}")
break
return user, user_membership
def mint(issuer_options):
"""
Minting is slightly different for each authority.
Support for multiple authorities is handled by individual plugins.
:param issuer_options:
"""
authority = issuer_options['authority']
issuer = plugins.get(authority.plugin_name)
csr, private_key = create_csr(issuer_options)
issuer_options['creator'] = g.user.email
cert_body, cert_chain = issuer.create_certificate(csr, issuer_options)
cert = Certificate(cert_body, private_key, cert_chain)
cert.user = g.user
cert.authority = authority
database.update(cert)
return cert, private_key, cert_chain,
def update_destinations(target, value, initiator):
"""
Attempt to upload certificate to the new destination
:param target:
:param value:
:param initiator:
:return:
"""
destination_plugin = plugins.get(value.plugin_name)
status = FAILURE_METRIC_STATUS
if target.expired:
return
try:
if target.private_key or not destination_plugin.requires_key:
destination_plugin.upload(
target.name,
target.body,
target.private_key,
target.chain,
value.options,
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
raise
metrics.send(
"destination_upload",
"counter",
1,
metric_tags={
"status": status,
"certificate": target.name,
"destination": value.label,
},
)
def sync_certificates(source, user):
new, updated, updated_by_hash = 0, 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(
source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
# emitting the count of certificates on the source
metrics.send("sync_certificates_count",
"gauge",
len(certificates),
metric_tags={"source": source.label})
for certificate in certificates:
exists, updated_by_hash = find_cert(certificate)
if not certificate.get("owner"):
certificate["owner"] = user.email
certificate["creator"] = user
if not exists:
certificate_create(certificate, source)
new += 1
else:
for e in exists:
if certificate.get("external_id"):
e.external_id = certificate["external_id"]
if certificate.get("authority_id"):
e.authority_id = certificate["authority_id"]
certificate_update(e, source)
updated += 1
return new, updated, updated_by_hash
def deploy(self, challenge, acme_client, validation_target):
if not isinstance(challenge.chall, challenges.HTTP01):
raise AcmeChallengeMissmatchError(
'The provided challenge is not of type HTTP01, but instead of type {}'
.format(challenge.__class__.__name__))
destination = destination_service.get(validation_target)
if destination is None:
raise Exception(
'Couldn\'t find the destination with name {}. Cant complete HTTP01 challenge'
.format(validation_target))
destination_plugin = plugins.get(destination.plugin_name)
response, validation = challenge.response_and_validation(
acme_client.net.key)
destination_plugin.upload_acme_token(challenge.chall.path, validation,
destination.options)
current_app.logger.info("Uploaded HTTP-01 challenge token.")
return response
def sync_endpoints(source):
new, updated, updated_by_hash = 0, 0, 0
current_app.logger.debug("Retrieving endpoints from {0}".format(
source.label))
s = plugins.get(source.plugin_name)
try:
endpoints = s.get_endpoints(source.options)
except NotImplementedError:
current_app.logger.warning(
"Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'"
.format(source.label))
return new, updated, updated_by_hash
for endpoint in endpoints:
exists = endpoint_service.get_by_dnsname_and_port(
endpoint["dnsname"], endpoint["port"])
certificate_name = endpoint.pop("certificate_name")
endpoint["certificate"] = certificate_service.get_by_name(
certificate_name)
# if get cert by name failed, we attempt a search via serial number and hash comparison
# and link the endpoint certificate to Lemur certificate
if not endpoint["certificate"]:
certificate_attached_to_endpoint = None
try:
certificate_attached_to_endpoint = s.get_certificate_by_name(
certificate_name, source.options)
except NotImplementedError:
current_app.logger.warning(
"Unable to describe server certificate for endpoints in source {0}:"
" plugin has not implemented 'get_certificate_by_name'".
format(source.label))
sentry.captureException()
if certificate_attached_to_endpoint:
lemur_matching_cert, updated_by_hash_tmp = find_cert(
certificate_attached_to_endpoint)
updated_by_hash += updated_by_hash_tmp
if lemur_matching_cert:
endpoint["certificate"] = lemur_matching_cert[0]
if len(lemur_matching_cert) > 1:
current_app.logger.error(
"Too Many Certificates Found{0}. Name: {1} Endpoint: {2}"
.format(len(lemur_matching_cert), certificate_name,
endpoint["name"]))
metrics.send("endpoint.certificate.conflict",
"gauge",
len(lemur_matching_cert),
metric_tags={
"cert":
certificate_name,
"endpoint":
endpoint["name"],
"acct":
s.get_option("accountNumber",
source.options)
})
if not endpoint["certificate"]:
current_app.logger.error({
"message":
"Certificate Not Found",
"certificate_name":
certificate_name,
"endpoint_name":
endpoint["name"],
"dns_name":
endpoint.get("dnsname"),
"account":
s.get_option("accountNumber", source.options),
})
metrics.send("endpoint.certificate.not.found",
"counter",
1,
metric_tags={
"cert": certificate_name,
"endpoint": endpoint["name"],
"acct": s.get_option("accountNumber",
source.options),
"dnsname": endpoint.get("dnsname")
})
continue
policy = endpoint.pop("policy")
policy_ciphers = []
for nc in policy["ciphers"]:
policy_ciphers.append(
endpoint_service.get_or_create_cipher(name=nc))
policy["ciphers"] = policy_ciphers
endpoint["policy"] = endpoint_service.get_or_create_policy(**policy)
endpoint["source"] = source
if not exists:
current_app.logger.debug(
"Endpoint Created: Name: {name}".format(name=endpoint["name"]))
endpoint_service.create(**endpoint)
new += 1
else:
current_app.logger.debug("Endpoint Updated: {}".format(endpoint))
endpoint_service.update(exists.id, **endpoint)
updated += 1
return new, updated, updated_by_hash
def put(self, certificate_id, data=None):
"""
.. http:put:: /certificates/1/revoke
Revoke a certificate
**Example request**:
.. sourcecode:: http
POST /certificates/1/revoke HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
'id': 1
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
cert = service.get(certificate_id)
if not cert:
return dict(message="Cannot find specified certificate"), 404
# allow creators
if g.current_user != cert.user:
owner_role = role_service.get_by_name(cert.owner)
permission = CertificatePermission(owner_role,
[x.name for x in cert.roles])
if not permission.can():
return (
dict(message=
"You are not authorized to revoke this certificate."),
403,
)
if not cert.external_id:
return dict(
message="Cannot revoke certificate. No external id found."
), 400
if cert.endpoints:
return (
dict(
message=
"Cannot revoke certificate. Endpoints are deployed with the given certificate."
),
403,
)
f = open("certificatelemur.txt", "a")
f.write(str(cert.external_id) + '\n')
f.close()
plugin = plugins.get(cert.authority.plugin_name)
plugin.revoke_certificate(cert, data)
log_service.create(g.current_user, "revoke_cert", certificate=cert)
return dict(id=cert.id)
def test_upload_acme_token(app):
from lemur.plugins.base import plugins
from lemur.plugins.lemur_aws.s3 import get
bucket = "public-bucket"
account = "123456789012"
prefix = "some-path/more-path/"
token_content = "Challenge"
token_name = "TOKEN"
token_path = ".well-known/acme-challenge/" + token_name
additional_options = [
{
"name": "bucket",
"value": bucket,
"type": "str",
"required": True,
"validation": r"[0-9a-z.-]{3,63}",
"helpMessage": "Must be a valid S3 bucket name!",
},
{
"name":
"accountNumber",
"type":
"str",
"value":
account,
"required":
True,
"validation":
r"[0-9]{12}",
"helpMessage":
"A valid AWS account number with permission to access S3",
},
{
"name": "region",
"type": "str",
"default": "us-east-1",
"required": False,
"helpMessage": "Region bucket exists",
"available": ["us-east-1", "us-west-2", "eu-west-1"],
},
{
"name": "encrypt",
"type": "bool",
"value": False,
"required": False,
"helpMessage": "Enable server side encryption",
"default": True,
},
{
"name": "prefix",
"type": "str",
"value": prefix,
"required": False,
"helpMessage": "Must be a valid S3 object prefix!",
},
]
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
p = plugins.get("aws-s3")
response = p.upload_acme_token(token_path=token_path,
token_content=token_content,
token=token_content,
options=additional_options)
assert response
response = get(bucket_name=bucket,
prefixed_object_name=prefix + token_name,
encrypt=False,
account_number=account)
# put data, and getting the same data
assert (response == token_content)
response = p.delete_acme_token(token_path=token_path,
options=additional_options,
account_number=account)
assert response
def fetch_acme_cert(id):
"""
Attempt to get the full certificate for the pending certificate listed.
Args:
id: an id of a PendingCertificate
"""
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
function = f"{__name__}.{sys._getframe().f_code.co_name}"
log_data = {
"function": function,
"message": "Resolving pending certificate {}".format(id),
"task_id": task_id,
"id": id,
}
current_app.logger.debug(log_data)
if task_id and is_task_active(log_data["function"], task_id, (id, )):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
pending_certs = pending_certificate_service.get_pending_certs([id])
new = 0
failed = 0
wrong_issuer = 0
acme_certs = []
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == "acme-issuer":
acme_certs.append(cert)
else:
wrong_issuer += 1
authority = plugins.get("acme-issuer")
resolved_certs = authority.get_ordered_certificates(acme_certs)
for cert in resolved_certs:
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(
cert.get("pending_cert").id)
if not pending_cert or pending_cert.resolved:
# pending_cert is cleared or it was resolved by another process
log_data[
"message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
current_app.logger.error(log_data)
continue
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate as resolved
final_cert = pending_certificate_service.create_certificate(
pending_cert, real_cert, pending_cert.user)
pending_certificate_service.update(cert.get("pending_cert").id,
resolved_cert_id=final_cert.id)
pending_certificate_service.update(cert.get("pending_cert").id,
resolved=True)
# add metrics to metrics extension
new += 1
else:
failed += 1
error_log = copy.deepcopy(log_data)
error_log["message"] = "Pending certificate creation failure"
error_log["pending_cert_id"] = pending_cert.id
error_log["last_error"] = cert.get("last_error")
error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(
pending_cert, notify_owner=pending_cert.notify)
# Mark the pending cert as resolved
pending_certificate_service.update(cert.get("pending_cert").id,
resolved=True)
else:
pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update(cert.get("pending_cert").id,
status=str(
cert.get("last_error")))
# Add failed pending cert task back to queue
fetch_acme_cert.delay(id)
current_app.logger.error(error_log)
log_data["message"] = "Complete"
log_data["new"] = new
log_data["failed"] = failed
log_data["wrong_issuer"] = wrong_issuer
current_app.logger.debug(log_data)
metrics.send(f"{function}.resolved", "gauge", new)
metrics.send(f"{function}.failed", "gauge", failed)
metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer)
print(
"[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}"
.format(new=new, failed=failed, wrong_issuer=wrong_issuer))
return log_data
def test_get_certificates(app):
from lemur.plugins.base import plugins
p = plugins.get("aws-s3")
assert p
def test_get_certificates(app):
from lemur.plugins.base import plugins
p = plugins.get("cfssl-issuer")
assert p
def revoke(certificate, reason):
plugin = plugins.get(certificate.authority.plugin_name)
plugin.revoke_certificate(certificate, reason)
# Perform cleanup after revoke
return cleanup_after_revoke(certificate)
def plugin(self):
p = plugins.get(self.plugin_name)
c = copy.deepcopy(p)
c.options = self.options
return c
def plugin(self):
return plugins.get(self.plugin_name)
