def convert(file_in, file_out): import base64 from dataset import Key from cryptography.x509.base import load_der_x509_certificate from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey results = {"rsa": 0, "all": 0, "errors": 0} with open(file_in) as fp: with open(file_out, "w") as fop: for cnt, line in enumerate(fp): results["all"] += 1 try: cert64 = line.split(",")[1] cert_bin = base64.b64decode(cert64) cert = load_der_x509_certificate(cert_bin, default_backend()) pub = cert.public_key() if isinstance(pub, RSAPublicKey): not_before = cert.not_valid_before cname = Rapid7.Converter.try_get_cname(cert) pub_num = pub.public_numbers() key = Key([cname, not_before.strftime('%Y-%m-%d')], pub_num.n, pub_num.e, 1) fop.write(key.get_as_string() + "\n") results["rsa"] += 1 except Exception as e: results["errors"] += 1 logging.warning('Processing of dataset %s: %s, line %d' % (file_in, e, cnt)) return results
def process_ldiff(self, data, name): """ Processes LDAP output field;binary::blob :param data: :param name: :return: """ from cryptography.x509.base import load_der_x509_certificate reg = re.compile(r'binary::\s*([0-9a-zA-Z+/=\s\t\r\n]{20,})$', re.MULTILINE | re.DOTALL) matches = re.findall(reg, data) num_certs_found = 0 for idx, match in enumerate(matches): match = re.sub('[\r\t\n\s]', '', match) try: bindata = base64.b64decode(match) x509 = load_der_x509_certificate(bindata, self.get_backend()) self.num_ldiff_cert += 1 self.process_x509(x509, name=name, pem=False, source='ldiff-cert') except Exception as e: logger.debug('Error in line ldiff file processing %s, idx %s, matchlen %s : %s' % (name, idx, len(match), e)) self.trace_logger.log(e)
def process_jks(self, data, name): """ Processes Java Key Store file :param data: :param name: :return: """ if self.jks_file_passwords is None and self.args.jks_pass_file is not None: self.jks_file_passwords = [] if not os.path.exists(self.args.jks_pass_file): logger.warning('JKS password file %s does not exist' % self.args.jks_pass_file) with open(self.args.jks_pass_file) as fh: self.jks_file_passwords = sorted(list(set([x.strip() for x in fh]))) ks = self.try_open_jks(data, name) if ks is None: logger.warning('Could not open JKS file: %s, password not valid, ' 'try specify passwords in --jks-pass-file' % name) return # certs from cryptography.x509.base import load_der_x509_certificate for alias, cert in ks.certs.items(): try: x509 = load_der_x509_certificate(cert.cert, self.get_backend()) self.num_jks_cert += 1 self.process_x509(x509, name=name, pem=False, source='jks-cert', aux='cert-%s' % alias) except Exception as e: logger.debug('Error in JKS cert processing %s, alias %s : %s' % (name, alias, e)) self.trace_logger.log(e) # priv key chains for alias, pk in ks.private_keys.items(): for idx, cert in enumerate(pk.cert_chain): try: x509 = load_der_x509_certificate(cert[1], self.get_backend()) self.num_jks_cert += 1 self.process_x509(x509, name=name, pem=False, source='jks-cert-chain', aux='cert-chain-%s-%s' % (alias, idx)) except Exception as e: logger.debug('Error in JKS priv key cert-chain processing %s, alias %s %s : %s' % (name, alias, idx, e)) self.trace_logger.log(e)
def _get_cert_and_protocol_version(self, sock): if sock is None: self.log.debug( "Could not validate certificate because there is no connection" ) return None, None # Get the cert & TLS version from the connection with closing(sock): self.log.debug('Getting cert and TLS protocol version') try: with closing(self.agent_check.get_tls_context().wrap_socket( sock, server_hostname=self.agent_check._server_hostname )) as secure_sock: der_cert = secure_sock.getpeercert(binary_form=True) protocol_version = secure_sock.version() self.log.debug( 'Received serialized peer certificate and TLS protocol version %s', protocol_version) except Exception as e: # https://docs.python.org/3/library/ssl.html#ssl.SSLCertVerificationError err_code = getattr(e, 'verify_code', None) message = getattr(e, 'verify_message', str(e)) self.log.debug( 'Error occurred while getting cert and TLS version from connection: %s', str(e)) self.agent_check.service_check(SERVICE_CHECK_VALIDATION, self.agent_check.CRITICAL, tags=self.agent_check._tags, message=message) # There's no sane way to tell it to not validate just the expiration # This only works on Python 3.7+, see: https://bugs.python.org/issue28182 # https://github.com/openssl/openssl/blob/0b45d8eec051fd9816b6bf46a975fa461ffc983d/include/openssl/x509_vfy.h#L109 if err_code == 10: self.agent_check.service_check( SERVICE_CHECK_EXPIRATION, self.agent_check.CRITICAL, tags=self.agent_check._tags, message='Certificate has expired', ) return None, None # Load https://cryptography.io/en/latest/x509/reference/#cryptography.x509.Certificate try: self.log.debug('Deserializing peer certificate') cert = load_der_x509_certificate(der_cert, default_backend()) self.log.debug('Deserialized peer certificate: %s', cert) return cert, protocol_version except Exception as e: self.log.debug('Error while deserializing peer certificate: %s', str(e)) self.agent_check.service_check( SERVICE_CHECK_VALIDATION, self.agent_check.CRITICAL, tags=self.agent_check._tags, message='Unable to parse the certificate: {}'.format(e), ) return None, None
def load_intermediate_certs(self, der_cert): # https://tools.ietf.org/html/rfc3280#section-4.2.2.1 # https://tools.ietf.org/html/rfc5280#section-5.2.7 try: cert = load_der_x509_certificate(der_cert, default_backend()) except Exception as e: self.log.error( 'Error while deserializing peer certificate to discover intermediate certificates: %s', e) return try: authority_information_access = cert.extensions.get_extension_for_oid( ExtensionOID.AUTHORITY_INFORMATION_ACCESS) except ExtensionNotFound: self.log.debug( 'No Authority Information Access extension found, skipping discovery of intermediate certificates' ) return for access_description in authority_information_access.value: if access_description.access_method != AuthorityInformationAccessOID.CA_ISSUERS: continue uri = access_description.access_location.value if (uri in self.agent_check._intermediate_cert_uri_cache and get_timestamp() - self.agent_check._intermediate_cert_uri_cache[uri] < self._intermediate_cert_refresh_interval): continue # Assume HTTP for now try: response = self.agent_check.http.get( uri) # SKIP_HTTP_VALIDATION response.raise_for_status() except Exception as e: self.log.error( 'Error fetching intermediate certificate from `%s`: %s', uri, e) continue else: access_time = get_timestamp() intermediate_cert = response.content cert_id = sha256(intermediate_cert).digest() if cert_id not in self.agent_check._intermediate_cert_id_cache: self.agent_check.get_tls_context().load_verify_locations( cadata=intermediate_cert) self.agent_check._intermediate_cert_id_cache.add(cert_id) self.agent_check._intermediate_cert_uri_cache[uri] = access_time self.load_intermediate_certs(intermediate_cert)
def process_der(self, data, name): """ DER processing :param data: :param name: :return: """ from cryptography.x509.base import load_der_x509_certificate try: x509 = load_der_x509_certificate(data, self.get_backend()) self.num_der_certs += 1 self.process_x509(x509, name=name, pem=False, source='der-cert') except Exception as e: logger.debug('DER processing failed: %s : %s' % (name, e)) self.trace_logger.log(e)
def process_der(self, data, js, dn, serial, desc, idx): """ DER processing :param data: :param js: :param serial: :return: """ from cryptography.x509.base import load_der_x509_certificate try: x509 = load_der_x509_certificate(data, self.get_backend()) self.num_der_certs += 1 return self.process_x509(x509, js=js, dn=dn, serial=serial, desc=desc, idx=idx) except Exception as e: logger.debug('DER processing failed: %s : %s' % (js['id'], e)) return None
def process_js_certs(self, data, name, idx, sub_idx): """ Process one certificate from JSON :param data: :param name: :param idx: :param sub_idx: :return: """ from cryptography.x509.base import load_der_x509_certificate for crt_hex in data: try: bindata = base64.b64decode(crt_hex) x509 = load_der_x509_certificate(bindata, self.get_backend()) self.num_ldiff_cert += 1 self.process_x509(x509, name=name, pem=False, source='ldiff-cert') except Exception as e: logger.debug('Error in line JSON cert file processing %s, idx %s, subidx %s : %s' % (name, idx, sub_idx, e)) self.trace_logger.log(e)
def process_pem_cert(data, name, idx): from cryptography.x509.base import load_der_x509_certificate x509 = load_der_x509_certificate(pem_to_der(data), get_backend()) return process_x509(x509, name=name, idx=idx, data=data, pem=True, source='pem-cert')
def roots(self, fname): """ One root file processing :param fname: :return: """ self.cur_file = fname before_file_certs_size = len(self.all_certs) with open(fname) as fh: for line in fh: try: if '"ca": false' in line: continue js = json.loads(line) fprint = None raw = None rawb = None if 'fprint' in js: fprint = js['fprint'] if 'ca' in js and not js['ca']: continue fprint_requires_raw = fprint is None or len(fprint) != 40 if fprint_requires_raw and 'raw' not in js: self.num_no_fprint_raw += 1 continue if fprint_requires_raw: raw = js['raw'] rawb = base64.b64decode(raw) fprint = hashlib.sha1(rawb).hexdigest() # Already seen in this round, may become valid in the next round. if fprint in self.chain_cert_db: continue # Already assigned to a trust category if fprint in self.assigned_fprints: continue if 'raw' not in js: self.num_no_raw += 1 continue if rawb is None: raw = js['raw'] rawb = base64.b64decode(raw) self.chain_cert_db.add(fprint) crypt_cert = load_der_x509_certificate(rawb, get_backend()) if not utils.try_is_ca(crypt_cert): if self.num_not_ca % 1000 == 0: logger.debug('Cert is not CA: %s (%d)' % (fprint, self.num_not_ca)) self.num_not_ca += 1 continue # Verify ossl_cert = load_certificate(FILETYPE_ASN1, rawb) self.cur_store.set_flags(0x200000) store_ctx = X509StoreContext(self.cur_store, ossl_cert) try: store_ctx.verify_certificate() self.interms[self.cur_depth].append(js) self.assigned_fprints.add(fprint) self.all_certs.append(ossl_cert) self.test_cert(crypt_cert, js) except X509StoreContextError as cex: self.trace_logger.log(cex, custom_msg='Exc in verification') if isinstance(cex.message, (types.ListType, types.TupleType)): if cex.message[0] == 10: self.num_expired += 1 self.test_cert(crypt_cert, js, 'Expired') except Exception as e: self.trace_logger.log( e, custom_msg='General Exc in verification') self.report() except Exception as e: logger.error('Exception in processing certs %s' % e) self.trace_logger.log(e) self.num_errs += 1 new_certs_size = len(self.all_certs) - before_file_certs_size logger.info('File %s contributed with %s certificates' % (fname, new_certs_size))
def load_x509_der(data, backend=None): return load_der_x509_certificate(data, get_backend(backend))
args = parser.parse_args() for file_name in args.files: with open(file_name, 'rb') as hnd: crt = hnd.read() is_pem = file_name.endswith('.pem') or crt.startswith('-----BEGIN') if is_pem or args.pem: parts = re.split('-{5,}BEGIN', crt) if len(parts) == 0: continue if len(parts[0]) == 0: parts.pop(0) crt_arr = ['-----BEGIN' + x for x in parts] for key in crt_arr: try: x509 = load_pem_x509_certificate(key, get_backend()) print_mod_hex(x509, print_e=args.exponent) except Exception as e: traceback.print_exc() sys.stderr.write('Exception in parsing key: %s\n' % e) if not is_pem or args.der: try: x509 = load_der_x509_certificate(crt, get_backend()) print_mod_hex(x509, print_e=args.exponent) except Exception as e: traceback.print_exc() sys.stderr.write('Exception in parsing key: %s\n' % e)
action='store_const', const=True, help='Print also public exponent') args = parser.parse_args() for file_name in args.files: reg = re.compile(r'::\s*([0-9a-zA-Z+/=\s\t\r\n]{20,})$', re.MULTILINE | re.DOTALL) with open(file_name, 'r') as hnd: data = hnd.read() num_certs = data.count('userCertificate;') matches = re.findall(reg, data) num_certs_found = 0 for idx, match in enumerate(matches): match = re.sub('[\r\t\n\s]', '', match) try: bindata = base64.b64decode(match) x509 = load_der_x509_certificate(bindata, get_backend()) print_mod_hex(x509, print_e=args.exponent) num_certs_found += 1 except Exception as e: traceback.print_exc() sys.stderr.write('Exception in parsing key idx %d: %s\n' % (idx, e)) sys.stderr.write('Finished, #of certs: %d, # of certs found: %d\n' % (num_certs, num_certs_found))