def import_certificates(folder_with_certs_to_import: Path) -> None: repo = RootCertificatesRepository(Path("certificates")) import_errors = [] for cert_path in folder_with_certs_to_import.glob("*"): if cert_path.name.endswith(".pem") or cert_path.name.endswith(".crt"): cert_content = cert_path.read_text().encode("ascii") parsing_function = load_pem_x509_certificate elif cert_path.name.endswith(".der"): cert_content = cert_path.read_bytes() parsing_function = load_der_x509_certificate else: print(f"Skipping file {cert_path}.") continue try: parsed_cert = parsing_function(cert_content, default_backend()) except ValueError as e: import_errors.append(f"Error loading {cert_path}: {e.args[0]}") continue new_cert_path = repo.store_certificate(parsed_cert) print(f"Stored certificate at {new_cert_path}") if import_errors: print("\nWARNING: Not all certificates could be imported:") for error_mgs in import_errors: print(error_mgs)
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # Fetch all the certificates from the the AOSP repo cert_records = [] temp_dir = TemporaryDirectory() try: # TODO(AD): Stop using shell commands # Clone the AOSP repo git_command = self._GIT_CMD.format(repo_url=self._REPO_URL, local_path=temp_dir.name) with open(os.devnull, "w") as dev_null: subprocess.check_output(git_command, shell=True, stderr=dev_null) # Find the latest tag that looks like android-8XXX - we don't care about android-iot or android-wear tag_list = subprocess.check_output( self._GIT_FIND_TAG_CMD, shell=True, cwd=temp_dir.name, stderr=dev_null ).decode("ascii") last_tag = tag_list.strip().rsplit("\n", 1)[1].strip() # Switch to this tag subprocess.check_output( self._GIT_CHECKOUT_TAG_CMD.format(tag=last_tag), shell=True, cwd=temp_dir.name, stderr=dev_null ) # Inspect each certificate cert_files_path = Path(temp_dir.name) / "files" for cert_path in cert_files_path.glob("*"): with open(cert_path, mode="r") as cert_file: cert_pem = cert_file.read() # Parse each certificate and store it if needed parsed_cert = load_pem_x509_certificate(cert_pem.encode(encoding="ascii"), default_backend()) if should_update_repo: certs_repo.store_certificate(parsed_cert) cert_records.append( ScrapedRootCertificateRecord( CertificateUtils.get_canonical_subject_name(parsed_cert), parsed_cert.fingerprint(hashes.SHA256()), hashes.SHA256(), ) ) finally: # Workaround for Windows https://bugs.python.org/issue26660 if platform == "win32": for file_path in Path(temp_dir.name).glob("**/*"): os.chmod(file_path, stat.S_IWRITE) temp_dir.cleanup() # Finally generate the records trusted_cert_records = RootRecordsValidator.validate_with_repository(certs_repo, cert_records) date_fetched = datetime.utcnow().date() version = last_tag.split("android-")[1] return TrustStore(PlatformEnum.GOOGLE_AOSP, version, self._REPO_URL, date_fetched, trusted_cert_records)
def import_certificate(certificate_path: str) -> None: """Save a PEM-formatted certificate to the local repository at ./certificates. """ with open(certificate_path, mode='r') as pem_file: cert_pem = pem_file.read() # Parse the certificate to double check the fingerprint parsed_cert = load_pem_x509_certificate(cert_pem.encode(encoding='ascii'), default_backend()) repo = RootCertificatesRepository(Path('certificates')) new_cert_path = repo.store_certificate(parsed_cert) print(f'Stored certificate at {new_cert_path}')
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # There's no specific version available in the certdata file os_version = None # Then fetch and parse the page with urlopen(self._PAGE_URL) as response: page_content = response.read().decode("utf-8") entries = self._scrape_certdata(page_content) certificate_entries = [ entry for entry in entries if isinstance(entry, _CertdataCertificateEntry) ] trust_entries = [ entry for entry in entries if isinstance(entry, _CertdataTrustEntry) ] # Store the certificates we found in the local repo if needed if should_update_repo: for cert_entry in certificate_entries: certs_repo.store_certificate(cert_entry.certificate) trusted_certificates = RootRecordsValidator.validate_with_repository( certs_repo, [ ScrapedRootCertificateRecord( entry.name, entry.sha1_fingerprint, hashes.SHA1()) for entry in trust_entries if entry.trust_enum == _CerdataEntryServerAuthTrustEnum.TRUSTED ], ) blocked_certificates = RootRecordsValidator.validate_with_repository( certs_repo, [ ScrapedRootCertificateRecord( entry.name, entry.sha1_fingerprint, hashes.SHA1()) for entry in trust_entries if entry.trust_enum == _CerdataEntryServerAuthTrustEnum.NOT_TRUSTED ], ) return TrustStore( PlatformEnum.MOZILLA_NSS, os_version, self._PAGE_URL, datetime.utcnow().date(), trusted_certificates, blocked_certificates, )
def test_online(self): certs_repo = RootCertificatesRepository.get_default() store_fetcher = AppleTrustStoreFetcher() fetched_store = store_fetcher.fetch(certs_repo) assert fetched_store assert 100 < len(fetched_store.trusted_certificates) assert 6 < len(fetched_store.blocked_certificates)
def refresh_trust_stores() -> None: """Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores. """ # Also pass the local certs repo so it gets updated when fetching the trust stores certs_repo = RootCertificatesRepository.get_default() # For each supported platform, fetch the trust store store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: if platform in [PlatformEnum.ORACLE_JAVA, PlatformEnum.OPENJDK]: # TODO: Fix this print(f"Skipping {platform.name}... TODO: Fixme") continue print(f"Refreshing {platform.name}...") fetched_store = store_fetcher.fetch(platform, certs_repo) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path(ROOT_PATH) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml" try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: print(f"Detected changes for {platform.name}; updating store...") with open(store_path, mode="w") as store_file: yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False) else: print(f"No changes detected for {platform.name}")
def test_online(self): certs_repo = RootCertificatesRepository.get_default() store_fetcher = MacosTrustStoreFetcher() fetched_store = store_fetcher.fetch(certs_repo) self.assertTrue(fetched_store) self.assertGreater(len(fetched_store.trusted_certificates), 100) self.assertGreater(len(fetched_store.blocked_certificates), 6)
def validate_with_repository( certs_repo: RootCertificatesRepository, fingerprint_hash_algorithm: Union[hashes.SHA1, hashes.SHA256], parsed_root_records: List[Tuple[str, bytes]], ) -> Set[RootCertificateRecord]: validated_root_records = set() # For each (subj_name, fingerprint) try to find the corresponding certificate in the supplied cert repo for scraped_subj_name, fingerprint in parsed_root_records: try: cert = certs_repo.lookup_certificate_with_fingerprint( fingerprint, fingerprint_hash_algorithm) validated_root_records.add( RootCertificateRecord.from_certificate(cert)) except CertificateNotFoundError: # We have never seen this certificate - use whatever name we scraped from the page logging.error( f'Could not find certificate "{scraped_subj_name}" in local repository' ) record = RootCertificateRecord.from_scraped_record( scraped_subj_name, fingerprint) validated_root_records.add(record) except ValueError as e: if 'Unsupported ASN1 string type' in e.args[0]: # Could not parse the certificate: https://github.com/pyca/cryptography/issues/3542 logging.error( f'Parsing error for certificate "{scraped_subj_name}"') # Give up and just use the scraped name record = RootCertificateRecord.from_scraped_record( scraped_subj_name, fingerprint) validated_root_records.add(record) else: raise return validated_root_records
def _extract_trusted_root_records( key_store: jks.KeyStore, should_update_repo: bool, cert_repo: RootCertificatesRepository ) -> List[ScrapedRootCertificateRecord]: root_records = [] for alias, item in key_store.certs.items(): parsed_cert = load_der_x509_certificate(item.cert, default_backend()) if should_update_repo: cert_repo.store_certificate(parsed_cert) root_records.append(ScrapedRootCertificateRecord( CertificateUtils.get_canonical_subject_name(parsed_cert), parsed_cert.fingerprint(hashes.SHA256()), hashes.SHA256()) ) return root_records
def export_trusted_certificates_as_pem(self, certs_repository: RootCertificatesRepository) -> str: # Lookup each certificate in the folders we use as the repository of all root certs all_certs_as_pem = [] for cert_record in self.trusted_certificates: cert = certs_repository.lookup_certificate_with_fingerprint(cert_record.fingerprint) # Export each certificate as PEM all_certs_as_pem.append(cert.public_bytes(Encoding.PEM).decode('ascii')) return '\n'.join(all_certs_as_pem)
def refresh_trust_stores() -> None: """Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores. """ # Also pass the local certs repo so it gets updated when fetching the trust stores certs_repo = RootCertificatesRepository.get_default() # For each supported platform, fetch the trust store has_any_store_changed = False store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: if platform == PlatformEnum.ORACLE_JAVA: # TODO: Fix this print(f"Skipping {platform.name}... TODO: Fixme") continue print(f"Refreshing {platform.name}...") fetched_store = store_fetcher.fetch(platform, certs_repo) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path( ROOT_PATH ) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml" try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: has_any_store_changed = True print(f"Detected changes for {platform.name}; updating store...") with open(store_path, mode="w") as store_file: yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False) else: print(f"No changes detected for {platform.name}") # If we are running on travis if "TRAVIS" in environ: print("Running on Travis...") # Enable the deploy step if a change was detected with open("should_travis_deploy", mode="w") as travis_file: travis_flag = "1" if has_any_store_changed else "0" travis_file.write(f"export SHOULD_TRAVIS_DEPLOY={travis_flag}\n")
def export_trust_stores() -> None: """Export the content of the trust store of each supported platform to a PEM file at ./export. """ certs_repo = RootCertificatesRepository.get_default() out_pem_folder = ROOT_PATH / 'export' out_pem_folder.mkdir(exist_ok=True) # Export each trust store as a PEM file to ./export print(f'Exporting stores as PEM to {out_pem_folder}...') for platform in PlatformEnum: print(f'Exporting {platform.name}...') store = TrustStore.get_default_for_platform(platform) all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo) out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem' with open(out_pem_path, mode='w') as out_pem_file: out_pem_file.write(all_certs_pem)
def test_default_repository_integrity(self): # Given the local repo of certificates repo = RootCertificatesRepository.get_default() # Each certificate that it returns is stored at the expected location expected_repo_path = Path(os.path.abspath( os.path.dirname(__file__))) / '..' / 'certificates' for certificate in repo.get_all_certificates(): expected_file_name = hexlify(certificate.fingerprint( SHA256())).decode('ascii') expected_cert_path = expected_repo_path / f'{expected_file_name}.pem' with open(expected_cert_path) as stored_cert_file: stored_cert_pem = stored_cert_file.read() stored_cert = load_pem_x509_certificate( stored_cert_pem.encode(encoding='ascii'), default_backend()) self.assertEqual(stored_cert, certificate)
def refresh_trust_stores() -> None: """Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores. """ # Also pass the local certs repo so it gets updated when fetching the trust stores certs_repo = RootCertificatesRepository.get_default() # For each supported platform, fetch the trust store has_any_store_changed = False store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: print(f'Refreshing {platform.name}...') fetched_store = store_fetcher.fetch(platform, certs_repo) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path( ROOT_PATH ) / 'trust_stores' / f'{fetched_store.platform.name.lower()}.yaml' try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: has_any_store_changed = True print(f'Detected changes for {platform.name}; updating store...') with open(store_path, mode='w') as store_file: yaml.dump(fetched_store, store_file, encoding='utf-8', default_flow_style=False) else: print(f'No changes detected for {platform.name}') # If we are running on travis if 'TRAVIS' in environ: print('Running on Travis...') # Enable the deploy step if a change was detected with open('should_travis_deploy', mode='w') as travis_file: travis_flag = '1' if has_any_store_changed else '0' travis_file.write(f'export SHOULD_TRAVIS_DEPLOY={travis_flag}\n')
def test_default_repository_integrity(self): # Given the local repo of certificates repo = RootCertificatesRepository.get_default() # Each certificate that it returns is stored at the expected location expected_repo_path = Path(os.path.abspath( os.path.dirname(__file__))) / ".." / "certificates" all_certificates = repo.get_all_certificates() assert all_certificates for certificate in all_certificates: expected_file_name = hexlify(certificate.fingerprint( SHA256())).decode("ascii") expected_cert_path = expected_repo_path / f"{expected_file_name}.pem" with open(expected_cert_path) as stored_cert_file: stored_cert_pem = stored_cert_file.read() stored_cert = load_pem_x509_certificate( stored_cert_pem.encode(encoding="ascii"), default_backend()) assert stored_cert == certificate
import os from pathlib import Path from trust_stores_observatory.certificates_repository import RootCertificatesRepository from trust_stores_observatory.trust_store import PlatformEnum, TrustStore certs_repo = RootCertificatesRepository.get_default() root_path = Path(os.path.abspath(os.path.dirname(__file__))) out_pem_folder = root_path / 'export' out_pem_folder.mkdir(exist_ok=True) print(f'Exporting stores as PEM to {out_pem_folder}...') # Export each trust store as a PEM file to ./export for platform in PlatformEnum: print(f'Exporting {platform.name}...') store = TrustStore.get_default_for_platform(platform) all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo) out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem' with open(out_pem_path, mode='w') as out_pem_file: out_pem_file.write(all_certs_pem)