def import_certificates(folder_with_certs_to_import: Path) -> None:
repo = RootCertificatesRepository(Path("certificates"))
import_errors = []
for cert_path in folder_with_certs_to_import.glob("*"):
if cert_path.name.endswith(".pem") or cert_path.name.endswith(".crt"):
cert_content = cert_path.read_text().encode("ascii")
parsing_function = load_pem_x509_certificate
elif cert_path.name.endswith(".der"):
cert_content = cert_path.read_bytes()
parsing_function = load_der_x509_certificate
else:
print(f"Skipping file {cert_path}.")
continue
try:
parsed_cert = parsing_function(cert_content, default_backend())
except ValueError as e:
import_errors.append(f"Error loading {cert_path}: {e.args[0]}")
continue
new_cert_path = repo.store_certificate(parsed_cert)
print(f"Stored certificate at {new_cert_path}")
if import_errors:
print("\nWARNING: Not all certificates could be imported:")
for error_mgs in import_errors:
print(error_mgs)
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore:
# Fetch all the certificates from the the AOSP repo
cert_records = []
temp_dir = TemporaryDirectory()
try:
# TODO(AD): Stop using shell commands
# Clone the AOSP repo
git_command = self._GIT_CMD.format(repo_url=self._REPO_URL, local_path=temp_dir.name)
with open(os.devnull, "w") as dev_null:
subprocess.check_output(git_command, shell=True, stderr=dev_null)
# Find the latest tag that looks like android-8XXX - we don't care about android-iot or android-wear
tag_list = subprocess.check_output(
self._GIT_FIND_TAG_CMD, shell=True, cwd=temp_dir.name, stderr=dev_null
).decode("ascii")
last_tag = tag_list.strip().rsplit("\n", 1)[1].strip()
# Switch to this tag
subprocess.check_output(
self._GIT_CHECKOUT_TAG_CMD.format(tag=last_tag), shell=True, cwd=temp_dir.name, stderr=dev_null
)
# Inspect each certificate
cert_files_path = Path(temp_dir.name) / "files"
for cert_path in cert_files_path.glob("*"):
with open(cert_path, mode="r") as cert_file:
cert_pem = cert_file.read()
# Parse each certificate and store it if needed
parsed_cert = load_pem_x509_certificate(cert_pem.encode(encoding="ascii"), default_backend())
if should_update_repo:
certs_repo.store_certificate(parsed_cert)
cert_records.append(
ScrapedRootCertificateRecord(
CertificateUtils.get_canonical_subject_name(parsed_cert),
parsed_cert.fingerprint(hashes.SHA256()),
hashes.SHA256(),
)
)
finally:
# Workaround for Windows https://bugs.python.org/issue26660
if platform == "win32":
for file_path in Path(temp_dir.name).glob("**/*"):
os.chmod(file_path, stat.S_IWRITE)
temp_dir.cleanup()
# Finally generate the records
trusted_cert_records = RootRecordsValidator.validate_with_repository(certs_repo, cert_records)
date_fetched = datetime.utcnow().date()
version = last_tag.split("android-")[1]
return TrustStore(PlatformEnum.GOOGLE_AOSP, version, self._REPO_URL, date_fetched, trusted_cert_records)
def import_certificate(certificate_path: str) -> None:
"""Save a PEM-formatted certificate to the local repository at ./certificates.
"""
with open(certificate_path, mode='r') as pem_file:
cert_pem = pem_file.read()
# Parse the certificate to double check the fingerprint
parsed_cert = load_pem_x509_certificate(cert_pem.encode(encoding='ascii'),
default_backend())
repo = RootCertificatesRepository(Path('certificates'))
new_cert_path = repo.store_certificate(parsed_cert)
print(f'Stored certificate at {new_cert_path}')
def fetch(self,
certs_repo: RootCertificatesRepository,
should_update_repo: bool = True) -> TrustStore:
# There's no specific version available in the certdata file
os_version = None
# Then fetch and parse the page
with urlopen(self._PAGE_URL) as response:
page_content = response.read().decode("utf-8")
entries = self._scrape_certdata(page_content)
certificate_entries = [
entry for entry in entries
if isinstance(entry, _CertdataCertificateEntry)
]
trust_entries = [
entry for entry in entries
if isinstance(entry, _CertdataTrustEntry)
]
# Store the certificates we found in the local repo if needed
if should_update_repo:
for cert_entry in certificate_entries:
certs_repo.store_certificate(cert_entry.certificate)
trusted_certificates = RootRecordsValidator.validate_with_repository(
certs_repo,
[
ScrapedRootCertificateRecord(
entry.name, entry.sha1_fingerprint, hashes.SHA1())
for entry in trust_entries
if entry.trust_enum == _CerdataEntryServerAuthTrustEnum.TRUSTED
],
)
blocked_certificates = RootRecordsValidator.validate_with_repository(
certs_repo,
[
ScrapedRootCertificateRecord(
entry.name, entry.sha1_fingerprint, hashes.SHA1())
for entry in trust_entries if entry.trust_enum ==
_CerdataEntryServerAuthTrustEnum.NOT_TRUSTED
],
)
return TrustStore(
PlatformEnum.MOZILLA_NSS,
os_version,
self._PAGE_URL,
datetime.utcnow().date(),
trusted_certificates,
blocked_certificates,
)
def test_online(self):
certs_repo = RootCertificatesRepository.get_default()
store_fetcher = AppleTrustStoreFetcher()
fetched_store = store_fetcher.fetch(certs_repo)
assert fetched_store
assert 100 < len(fetched_store.trusted_certificates)
assert 6 < len(fetched_store.blocked_certificates)
def refresh_trust_stores() -> None:
"""Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores.
"""
# Also pass the local certs repo so it gets updated when fetching the trust stores
certs_repo = RootCertificatesRepository.get_default()
# For each supported platform, fetch the trust store
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
if platform in [PlatformEnum.ORACLE_JAVA, PlatformEnum.OPENJDK]:
# TODO: Fix this
print(f"Skipping {platform.name}... TODO: Fixme")
continue
print(f"Refreshing {platform.name}...")
fetched_store = store_fetcher.fetch(platform, certs_repo)
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(ROOT_PATH) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml"
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
print(f"Detected changes for {platform.name}; updating store...")
with open(store_path, mode="w") as store_file:
yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False)
else:
print(f"No changes detected for {platform.name}")
def test_online(self):
certs_repo = RootCertificatesRepository.get_default()
store_fetcher = MacosTrustStoreFetcher()
fetched_store = store_fetcher.fetch(certs_repo)
self.assertTrue(fetched_store)
self.assertGreater(len(fetched_store.trusted_certificates), 100)
self.assertGreater(len(fetched_store.blocked_certificates), 6)
def validate_with_repository(
certs_repo: RootCertificatesRepository,
fingerprint_hash_algorithm: Union[hashes.SHA1, hashes.SHA256],
parsed_root_records: List[Tuple[str, bytes]],
) -> Set[RootCertificateRecord]:
validated_root_records = set()
# For each (subj_name, fingerprint) try to find the corresponding certificate in the supplied cert repo
for scraped_subj_name, fingerprint in parsed_root_records:
try:
cert = certs_repo.lookup_certificate_with_fingerprint(
fingerprint, fingerprint_hash_algorithm)
validated_root_records.add(
RootCertificateRecord.from_certificate(cert))
except CertificateNotFoundError:
# We have never seen this certificate - use whatever name we scraped from the page
logging.error(
f'Could not find certificate "{scraped_subj_name}" in local repository'
)
record = RootCertificateRecord.from_scraped_record(
scraped_subj_name, fingerprint)
validated_root_records.add(record)
except ValueError as e:
if 'Unsupported ASN1 string type' in e.args[0]:
# Could not parse the certificate: https://github.com/pyca/cryptography/issues/3542
logging.error(
f'Parsing error for certificate "{scraped_subj_name}"')
# Give up and just use the scraped name
record = RootCertificateRecord.from_scraped_record(
scraped_subj_name, fingerprint)
validated_root_records.add(record)
else:
raise
return validated_root_records
def _extract_trusted_root_records(
key_store: jks.KeyStore,
should_update_repo: bool,
cert_repo: RootCertificatesRepository
) -> List[ScrapedRootCertificateRecord]:
root_records = []
for alias, item in key_store.certs.items():
parsed_cert = load_der_x509_certificate(item.cert, default_backend())
if should_update_repo:
cert_repo.store_certificate(parsed_cert)
root_records.append(ScrapedRootCertificateRecord(
CertificateUtils.get_canonical_subject_name(parsed_cert),
parsed_cert.fingerprint(hashes.SHA256()),
hashes.SHA256())
)
return root_records
def export_trusted_certificates_as_pem(self, certs_repository: RootCertificatesRepository) -> str:
# Lookup each certificate in the folders we use as the repository of all root certs
all_certs_as_pem = []
for cert_record in self.trusted_certificates:
cert = certs_repository.lookup_certificate_with_fingerprint(cert_record.fingerprint)
# Export each certificate as PEM
all_certs_as_pem.append(cert.public_bytes(Encoding.PEM).decode('ascii'))
return '\n'.join(all_certs_as_pem)
def refresh_trust_stores() -> None:
"""Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores.
"""
# Also pass the local certs repo so it gets updated when fetching the trust stores
certs_repo = RootCertificatesRepository.get_default()
# For each supported platform, fetch the trust store
has_any_store_changed = False
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
if platform == PlatformEnum.ORACLE_JAVA:
# TODO: Fix this
print(f"Skipping {platform.name}... TODO: Fixme")
continue
print(f"Refreshing {platform.name}...")
fetched_store = store_fetcher.fetch(platform, certs_repo)
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(
ROOT_PATH
) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml"
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
has_any_store_changed = True
print(f"Detected changes for {platform.name}; updating store...")
with open(store_path, mode="w") as store_file:
yaml.dump(fetched_store,
store_file,
encoding="utf-8",
default_flow_style=False)
else:
print(f"No changes detected for {platform.name}")
# If we are running on travis
if "TRAVIS" in environ:
print("Running on Travis...")
# Enable the deploy step if a change was detected
with open("should_travis_deploy", mode="w") as travis_file:
travis_flag = "1" if has_any_store_changed else "0"
travis_file.write(f"export SHOULD_TRAVIS_DEPLOY={travis_flag}\n")
def export_trust_stores() -> None:
"""Export the content of the trust store of each supported platform to a PEM file at ./export.
"""
certs_repo = RootCertificatesRepository.get_default()
out_pem_folder = ROOT_PATH / 'export'
out_pem_folder.mkdir(exist_ok=True)
# Export each trust store as a PEM file to ./export
print(f'Exporting stores as PEM to {out_pem_folder}...')
for platform in PlatformEnum:
print(f'Exporting {platform.name}...')
store = TrustStore.get_default_for_platform(platform)
all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo)
out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem'
with open(out_pem_path, mode='w') as out_pem_file:
out_pem_file.write(all_certs_pem)
def test_default_repository_integrity(self):
# Given the local repo of certificates
repo = RootCertificatesRepository.get_default()
# Each certificate that it returns is stored at the expected location
expected_repo_path = Path(os.path.abspath(
os.path.dirname(__file__))) / '..' / 'certificates'
for certificate in repo.get_all_certificates():
expected_file_name = hexlify(certificate.fingerprint(
SHA256())).decode('ascii')
expected_cert_path = expected_repo_path / f'{expected_file_name}.pem'
with open(expected_cert_path) as stored_cert_file:
stored_cert_pem = stored_cert_file.read()
stored_cert = load_pem_x509_certificate(
stored_cert_pem.encode(encoding='ascii'),
default_backend())
self.assertEqual(stored_cert, certificate)
def refresh_trust_stores() -> None:
"""Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores.
"""
# Also pass the local certs repo so it gets updated when fetching the trust stores
certs_repo = RootCertificatesRepository.get_default()
# For each supported platform, fetch the trust store
has_any_store_changed = False
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
print(f'Refreshing {platform.name}...')
fetched_store = store_fetcher.fetch(platform, certs_repo)
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(
ROOT_PATH
) / 'trust_stores' / f'{fetched_store.platform.name.lower()}.yaml'
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
has_any_store_changed = True
print(f'Detected changes for {platform.name}; updating store...')
with open(store_path, mode='w') as store_file:
yaml.dump(fetched_store,
store_file,
encoding='utf-8',
default_flow_style=False)
else:
print(f'No changes detected for {platform.name}')
# If we are running on travis
if 'TRAVIS' in environ:
print('Running on Travis...')
# Enable the deploy step if a change was detected
with open('should_travis_deploy', mode='w') as travis_file:
travis_flag = '1' if has_any_store_changed else '0'
travis_file.write(f'export SHOULD_TRAVIS_DEPLOY={travis_flag}\n')
def test_default_repository_integrity(self):
# Given the local repo of certificates
repo = RootCertificatesRepository.get_default()
# Each certificate that it returns is stored at the expected location
expected_repo_path = Path(os.path.abspath(
os.path.dirname(__file__))) / ".." / "certificates"
all_certificates = repo.get_all_certificates()
assert all_certificates
for certificate in all_certificates:
expected_file_name = hexlify(certificate.fingerprint(
SHA256())).decode("ascii")
expected_cert_path = expected_repo_path / f"{expected_file_name}.pem"
with open(expected_cert_path) as stored_cert_file:
stored_cert_pem = stored_cert_file.read()
stored_cert = load_pem_x509_certificate(
stored_cert_pem.encode(encoding="ascii"),
default_backend())
assert stored_cert == certificate
import os
from pathlib import Path
from trust_stores_observatory.certificates_repository import RootCertificatesRepository
from trust_stores_observatory.trust_store import PlatformEnum, TrustStore
certs_repo = RootCertificatesRepository.get_default()
root_path = Path(os.path.abspath(os.path.dirname(__file__)))
out_pem_folder = root_path / 'export'
out_pem_folder.mkdir(exist_ok=True)
print(f'Exporting stores as PEM to {out_pem_folder}...')
# Export each trust store as a PEM file to ./export
for platform in PlatformEnum:
print(f'Exporting {platform.name}...')
store = TrustStore.get_default_for_platform(platform)
all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo)
out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem'
with open(out_pem_path, mode='w') as out_pem_file:
out_pem_file.write(all_certs_pem)
