def convert(file_in, file_out):
import base64
from dataset import Key
from cryptography.x509.base import load_der_x509_certificate
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
results = {"rsa": 0, "all": 0, "errors": 0}
with open(file_in) as fp:
with open(file_out, "w") as fop:
for cnt, line in enumerate(fp):
results["all"] += 1
try:
cert64 = line.split(",")[1]
cert_bin = base64.b64decode(cert64)
cert = load_der_x509_certificate(cert_bin, default_backend())
pub = cert.public_key()
if isinstance(pub, RSAPublicKey):
not_before = cert.not_valid_before
cname = Rapid7.Converter.try_get_cname(cert)
pub_num = pub.public_numbers()
key = Key([cname, not_before.strftime('%Y-%m-%d')], pub_num.n, pub_num.e, 1)
fop.write(key.get_as_string() + "\n")
results["rsa"] += 1
except Exception as e:
results["errors"] += 1
logging.warning('Processing of dataset %s: %s, line %d' % (file_in, e, cnt))
return results
def process_ldiff(self, data, name):
"""
Processes LDAP output
field;binary::blob
:param data:
:param name:
:return:
"""
from cryptography.x509.base import load_der_x509_certificate
reg = re.compile(r'binary::\s*([0-9a-zA-Z+/=\s\t\r\n]{20,})$', re.MULTILINE | re.DOTALL)
matches = re.findall(reg, data)
num_certs_found = 0
for idx, match in enumerate(matches):
match = re.sub('[\r\t\n\s]', '', match)
try:
bindata = base64.b64decode(match)
x509 = load_der_x509_certificate(bindata, self.get_backend())
self.num_ldiff_cert += 1
self.process_x509(x509, name=name, pem=False, source='ldiff-cert')
except Exception as e:
logger.debug('Error in line ldiff file processing %s, idx %s, matchlen %s : %s'
% (name, idx, len(match), e))
self.trace_logger.log(e)
def process_jks(self, data, name):
"""
Processes Java Key Store file
:param data:
:param name:
:return:
"""
if self.jks_file_passwords is None and self.args.jks_pass_file is not None:
self.jks_file_passwords = []
if not os.path.exists(self.args.jks_pass_file):
logger.warning('JKS password file %s does not exist' % self.args.jks_pass_file)
with open(self.args.jks_pass_file) as fh:
self.jks_file_passwords = sorted(list(set([x.strip() for x in fh])))
ks = self.try_open_jks(data, name)
if ks is None:
logger.warning('Could not open JKS file: %s, password not valid, '
'try specify passwords in --jks-pass-file' % name)
return
# certs
from cryptography.x509.base import load_der_x509_certificate
for alias, cert in ks.certs.items():
try:
x509 = load_der_x509_certificate(cert.cert, self.get_backend())
self.num_jks_cert += 1
self.process_x509(x509, name=name, pem=False, source='jks-cert', aux='cert-%s' % alias)
except Exception as e:
logger.debug('Error in JKS cert processing %s, alias %s : %s' % (name, alias, e))
self.trace_logger.log(e)
# priv key chains
for alias, pk in ks.private_keys.items():
for idx, cert in enumerate(pk.cert_chain):
try:
x509 = load_der_x509_certificate(cert[1], self.get_backend())
self.num_jks_cert += 1
self.process_x509(x509, name=name, pem=False, source='jks-cert-chain',
aux='cert-chain-%s-%s' % (alias, idx))
except Exception as e:
logger.debug('Error in JKS priv key cert-chain processing %s, alias %s %s : %s'
% (name, alias, idx, e))
self.trace_logger.log(e)
def _get_cert_and_protocol_version(self, sock):
if sock is None:
self.log.debug(
"Could not validate certificate because there is no connection"
)
return None, None
# Get the cert & TLS version from the connection
with closing(sock):
self.log.debug('Getting cert and TLS protocol version')
try:
with closing(self.agent_check.get_tls_context().wrap_socket(
sock, server_hostname=self.agent_check._server_hostname
)) as secure_sock:
der_cert = secure_sock.getpeercert(binary_form=True)
protocol_version = secure_sock.version()
self.log.debug(
'Received serialized peer certificate and TLS protocol version %s',
protocol_version)
except Exception as e:
# https://docs.python.org/3/library/ssl.html#ssl.SSLCertVerificationError
err_code = getattr(e, 'verify_code', None)
message = getattr(e, 'verify_message', str(e))
self.log.debug(
'Error occurred while getting cert and TLS version from connection: %s',
str(e))
self.agent_check.service_check(SERVICE_CHECK_VALIDATION,
self.agent_check.CRITICAL,
tags=self.agent_check._tags,
message=message)
# There's no sane way to tell it to not validate just the expiration
# This only works on Python 3.7+, see: https://bugs.python.org/issue28182
# https://github.com/openssl/openssl/blob/0b45d8eec051fd9816b6bf46a975fa461ffc983d/include/openssl/x509_vfy.h#L109
if err_code == 10:
self.agent_check.service_check(
SERVICE_CHECK_EXPIRATION,
self.agent_check.CRITICAL,
tags=self.agent_check._tags,
message='Certificate has expired',
)
return None, None
# Load https://cryptography.io/en/latest/x509/reference/#cryptography.x509.Certificate
try:
self.log.debug('Deserializing peer certificate')
cert = load_der_x509_certificate(der_cert, default_backend())
self.log.debug('Deserialized peer certificate: %s', cert)
return cert, protocol_version
except Exception as e:
self.log.debug('Error while deserializing peer certificate: %s',
str(e))
self.agent_check.service_check(
SERVICE_CHECK_VALIDATION,
self.agent_check.CRITICAL,
tags=self.agent_check._tags,
message='Unable to parse the certificate: {}'.format(e),
)
return None, None
def load_intermediate_certs(self, der_cert):
# https://tools.ietf.org/html/rfc3280#section-4.2.2.1
# https://tools.ietf.org/html/rfc5280#section-5.2.7
try:
cert = load_der_x509_certificate(der_cert, default_backend())
except Exception as e:
self.log.error(
'Error while deserializing peer certificate to discover intermediate certificates: %s',
e)
return
try:
authority_information_access = cert.extensions.get_extension_for_oid(
ExtensionOID.AUTHORITY_INFORMATION_ACCESS)
except ExtensionNotFound:
self.log.debug(
'No Authority Information Access extension found, skipping discovery of intermediate certificates'
)
return
for access_description in authority_information_access.value:
if access_description.access_method != AuthorityInformationAccessOID.CA_ISSUERS:
continue
uri = access_description.access_location.value
if (uri in self.agent_check._intermediate_cert_uri_cache
and get_timestamp() -
self.agent_check._intermediate_cert_uri_cache[uri] <
self._intermediate_cert_refresh_interval):
continue
# Assume HTTP for now
try:
response = self.agent_check.http.get(
uri) # SKIP_HTTP_VALIDATION
response.raise_for_status()
except Exception as e:
self.log.error(
'Error fetching intermediate certificate from `%s`: %s',
uri, e)
continue
else:
access_time = get_timestamp()
intermediate_cert = response.content
cert_id = sha256(intermediate_cert).digest()
if cert_id not in self.agent_check._intermediate_cert_id_cache:
self.agent_check.get_tls_context().load_verify_locations(
cadata=intermediate_cert)
self.agent_check._intermediate_cert_id_cache.add(cert_id)
self.agent_check._intermediate_cert_uri_cache[uri] = access_time
self.load_intermediate_certs(intermediate_cert)
def process_der(self, data, name):
"""
DER processing
:param data:
:param name:
:return:
"""
from cryptography.x509.base import load_der_x509_certificate
try:
x509 = load_der_x509_certificate(data, self.get_backend())
self.num_der_certs += 1
self.process_x509(x509, name=name, pem=False, source='der-cert')
except Exception as e:
logger.debug('DER processing failed: %s : %s' % (name, e))
self.trace_logger.log(e)
def process_der(self, data, js, dn, serial, desc, idx):
"""
DER processing
:param data:
:param js:
:param serial:
:return:
"""
from cryptography.x509.base import load_der_x509_certificate
try:
x509 = load_der_x509_certificate(data, self.get_backend())
self.num_der_certs += 1
return self.process_x509(x509, js=js, dn=dn, serial=serial, desc=desc, idx=idx)
except Exception as e:
logger.debug('DER processing failed: %s : %s' % (js['id'], e))
return None
def process_js_certs(self, data, name, idx, sub_idx):
"""
Process one certificate from JSON
:param data:
:param name:
:param idx:
:param sub_idx:
:return:
"""
from cryptography.x509.base import load_der_x509_certificate
for crt_hex in data:
try:
bindata = base64.b64decode(crt_hex)
x509 = load_der_x509_certificate(bindata, self.get_backend())
self.num_ldiff_cert += 1
self.process_x509(x509, name=name, pem=False, source='ldiff-cert')
except Exception as e:
logger.debug('Error in line JSON cert file processing %s, idx %s, subidx %s : %s'
% (name, idx, sub_idx, e))
self.trace_logger.log(e)
def process_pem_cert(data, name, idx):
from cryptography.x509.base import load_der_x509_certificate
x509 = load_der_x509_certificate(pem_to_der(data), get_backend())
return process_x509(x509, name=name, idx=idx, data=data, pem=True, source='pem-cert')
def roots(self, fname):
"""
One root file processing
:param fname:
:return:
"""
self.cur_file = fname
before_file_certs_size = len(self.all_certs)
with open(fname) as fh:
for line in fh:
try:
if '"ca": false' in line:
continue
js = json.loads(line)
fprint = None
raw = None
rawb = None
if 'fprint' in js:
fprint = js['fprint']
if 'ca' in js and not js['ca']:
continue
fprint_requires_raw = fprint is None or len(fprint) != 40
if fprint_requires_raw and 'raw' not in js:
self.num_no_fprint_raw += 1
continue
if fprint_requires_raw:
raw = js['raw']
rawb = base64.b64decode(raw)
fprint = hashlib.sha1(rawb).hexdigest()
# Already seen in this round, may become valid in the next round.
if fprint in self.chain_cert_db:
continue
# Already assigned to a trust category
if fprint in self.assigned_fprints:
continue
if 'raw' not in js:
self.num_no_raw += 1
continue
if rawb is None:
raw = js['raw']
rawb = base64.b64decode(raw)
self.chain_cert_db.add(fprint)
crypt_cert = load_der_x509_certificate(rawb, get_backend())
if not utils.try_is_ca(crypt_cert):
if self.num_not_ca % 1000 == 0:
logger.debug('Cert is not CA: %s (%d)' %
(fprint, self.num_not_ca))
self.num_not_ca += 1
continue
# Verify
ossl_cert = load_certificate(FILETYPE_ASN1, rawb)
self.cur_store.set_flags(0x200000)
store_ctx = X509StoreContext(self.cur_store, ossl_cert)
try:
store_ctx.verify_certificate()
self.interms[self.cur_depth].append(js)
self.assigned_fprints.add(fprint)
self.all_certs.append(ossl_cert)
self.test_cert(crypt_cert, js)
except X509StoreContextError as cex:
self.trace_logger.log(cex,
custom_msg='Exc in verification')
if isinstance(cex.message,
(types.ListType, types.TupleType)):
if cex.message[0] == 10:
self.num_expired += 1
self.test_cert(crypt_cert, js, 'Expired')
except Exception as e:
self.trace_logger.log(
e, custom_msg='General Exc in verification')
self.report()
except Exception as e:
logger.error('Exception in processing certs %s' % e)
self.trace_logger.log(e)
self.num_errs += 1
new_certs_size = len(self.all_certs) - before_file_certs_size
logger.info('File %s contributed with %s certificates' %
(fname, new_certs_size))
def load_x509_der(data, backend=None):
return load_der_x509_certificate(data, get_backend(backend))
args = parser.parse_args()
for file_name in args.files:
with open(file_name, 'rb') as hnd:
crt = hnd.read()
is_pem = file_name.endswith('.pem') or crt.startswith('-----BEGIN')
if is_pem or args.pem:
parts = re.split('-{5,}BEGIN', crt)
if len(parts) == 0:
continue
if len(parts[0]) == 0:
parts.pop(0)
crt_arr = ['-----BEGIN' + x for x in parts]
for key in crt_arr:
try:
x509 = load_pem_x509_certificate(key, get_backend())
print_mod_hex(x509, print_e=args.exponent)
except Exception as e:
traceback.print_exc()
sys.stderr.write('Exception in parsing key: %s\n' % e)
if not is_pem or args.der:
try:
x509 = load_der_x509_certificate(crt, get_backend())
print_mod_hex(x509, print_e=args.exponent)
except Exception as e:
traceback.print_exc()
sys.stderr.write('Exception in parsing key: %s\n' % e)
action='store_const',
const=True,
help='Print also public exponent')
args = parser.parse_args()
for file_name in args.files:
reg = re.compile(r'::\s*([0-9a-zA-Z+/=\s\t\r\n]{20,})$',
re.MULTILINE | re.DOTALL)
with open(file_name, 'r') as hnd:
data = hnd.read()
num_certs = data.count('userCertificate;')
matches = re.findall(reg, data)
num_certs_found = 0
for idx, match in enumerate(matches):
match = re.sub('[\r\t\n\s]', '', match)
try:
bindata = base64.b64decode(match)
x509 = load_der_x509_certificate(bindata, get_backend())
print_mod_hex(x509, print_e=args.exponent)
num_certs_found += 1
except Exception as e:
traceback.print_exc()
sys.stderr.write('Exception in parsing key idx %d: %s\n' %
(idx, e))
sys.stderr.write('Finished, #of certs: %d, # of certs found: %d\n' %
(num_certs, num_certs_found))