def fetch(self, cert_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore:
# Fetch the latest JDK package
final_url = self._get_latest_download_url()
request = Request(final_url)
response = urlopen(request)
# Parse the JDK package
jdk_temp_file = NamedTemporaryFile(delete=False)
try:
jdk_temp_file.write(response.read())
jdk_temp_file.close()
with JdkPackage(jdk_temp_file.name) as parsed_jre:
# Extract the data we need
version = parsed_jre.get_version()
blacklisted_file_content = parsed_jre.get_blacklisted_certs()
cacerts_key_store = jks.KeyStore.loads(parsed_jre.get_cacerts(), parsed_jre.get_cacerts_password())
finally:
os.remove(jdk_temp_file.name)
# Process the data extracted from the JRE
# Trusted CA certs
scraped_trusted_records = JdkPackage.extract_trusted_root_records(
cacerts_key_store, should_update_repo, cert_repo
)
trusted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_trusted_records)
# Blacklisted CA certs - will fail if a blacklisted cert is not already available in the local repo
scraped_blacklisted_records = JdkPackage.extract_blacklisted_root_records(blacklisted_file_content)
blacklisted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_blacklisted_records)
return TrustStore(
PlatformEnum.OPENJDK, version, final_url, datetime.utcnow().date(), trusted_records, blacklisted_records
)
def fetch(self,
certs_repo: RootCertificatesRepository,
should_update_repo: bool = True) -> TrustStore:
# Find the URL to the latest spreadsheet and download it
spreadsheet_url = self._find_latest_root_certificates_url()
with urlopen(spreadsheet_url) as response:
spreadsheet_content = response.read()
with TemporaryFile() as temp_file:
temp_file.write(spreadsheet_content)
workbook = load_workbook(temp_file)
# Extract the data from the spreadsheet
version, scraped_trusted_root_records, scraped_blocked_root_records = self._parse_spreadsheet(
workbook)
# Look for each parsed certificate in the supplied certs repo
trusted_root_records = RootRecordsValidator.validate_with_repository(
certs_repo, scraped_trusted_root_records)
blocked_root_records = RootRecordsValidator.validate_with_repository(
certs_repo, scraped_blocked_root_records)
date_fetched = datetime.utcnow().date()
return TrustStore(PlatformEnum.MICROSOFT_WINDOWS, version,
spreadsheet_url, date_fetched, trusted_root_records,
blocked_root_records)
def fetch(self,
certs_repo: RootCertificatesRepository,
should_update_repo: bool = True) -> TrustStore:
with urlopen(self._CSV_URL) as response:
csv_content = response.read().decode("utf-8")
# Extract the data from the CSV
scraped_trusted_root_records, scraped_blocked_root_records = self._parse_spreadsheet(
csv_content)
# Look for each parsed certificate in the supplied certs repo
trusted_root_records = RootRecordsValidator.validate_with_repository(
certs_repo, scraped_trusted_root_records)
blocked_root_records = RootRecordsValidator.validate_with_repository(
certs_repo, scraped_blocked_root_records)
date_fetched = datetime.utcnow().date()
return TrustStore(
PlatformEnum.MICROSOFT_WINDOWS,
None,
self._PAGE_URL,
date_fetched,
trusted_root_records,
blocked_root_records,
)
def refresh_trust_stores() -> None:
"""Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores.
"""
# Also pass the local certs repo so it gets updated when fetching the trust stores
certs_repo = RootCertificatesRepository.get_default()
# For each supported platform, fetch the trust store
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
if platform in [PlatformEnum.ORACLE_JAVA, PlatformEnum.OPENJDK]:
# TODO: Fix this
print(f"Skipping {platform.name}... TODO: Fixme")
continue
print(f"Refreshing {platform.name}...")
fetched_store = store_fetcher.fetch(platform, certs_repo)
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(ROOT_PATH) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml"
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
print(f"Detected changes for {platform.name}; updating store...")
with open(store_path, mode="w") as store_file:
yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False)
else:
print(f"No changes detected for {platform.name}")
def fetch(self,
certs_repo: RootCertificatesRepository,
should_update_repo: bool = True) -> TrustStore:
# First find the latest page with the list of root certificates
os_version, trust_store_url = self._find_latest_root_certificates_page(
)
# Then fetch and parse the page
with urlopen(trust_store_url) as response:
page_content = response.read()
parsed_page = BeautifulSoup(page_content, 'html.parser')
# There are two divs on the page, one with trusted certificates and one with blocked certificates
root_certificates: Dict[str, Set[RootCertificateRecord]] = {
'trusted': set(),
'blocked': set()
}
# We parse both divs
for div_id in ['trusted', 'blocked']:
scraped_root_records = self._parse_root_records_in_div(
parsed_page, div_id=div_id)
# Look for each certificate in the supplied certs repo
root_certificates[
div_id] = RootRecordsValidator.validate_with_repository(
certs_repo, scraped_root_records)
return TrustStore(self._PLATFORM, os_version, trust_store_url,
datetime.utcnow().date(),
root_certificates['trusted'],
root_certificates['blocked'])
def fetch(self,
certs_repo: RootCertificatesRepository,
should_update_repo: bool = True) -> TrustStore:
spreadsheet_url = self._find_latest_root_certificates_url()
with urlopen(spreadsheet_url) as response:
spreadsheet_content = response.read()
with TemporaryFile() as temp_file:
temp_file.write(spreadsheet_content)
workbook = load_workbook(temp_file)
worksheet = workbook.active
version = worksheet['A1'].value.split('As of')[1].strip()
# Iterate over each row in the work sheet
parsed_trusted_root_records = []
parsed_blocked_root_records = []
for row in worksheet.iter_rows(min_row=4, max_col=6, max_row=500):
subject_name = row[1].value
if subject_name is None:
# Most likely indicates the end of the data
continue
is_cert_trusted = False
status = row[4].value.strip()
if 'Active' in status:
# Some certs are disabled or have a notBefore constraint
is_cert_trusted = True
fingerprint_cell = row[3].value
if fingerprint_cell is None:
# One certificate actually does not have the fingerprint cell properly filled
logging.error(f'No fingerprint for {subject_name}')
continue
fingerprint_hex = fingerprint_cell.replace(':', '').strip()
fingerprint = bytes(bytearray.fromhex(fingerprint_hex))
if is_cert_trusted:
parsed_trusted_root_records.append((subject_name, fingerprint))
else:
parsed_blocked_root_records.append((subject_name, fingerprint))
# Look for each certificate in the supplied certs repo
trusted_root_records = RootRecordsValidator.validate_with_repository(
certs_repo,
hashes.SHA256(),
parsed_trusted_root_records,
)
blocked_root_records = RootRecordsValidator.validate_with_repository(
certs_repo, hashes.SHA256(), parsed_blocked_root_records)
# TODO: FILTER OUT NON SSL CERTS
date_fetched = datetime.utcnow().date()
return TrustStore(PlatformEnum.MICROSOFT_WINDOWS, version,
spreadsheet_url, date_fetched, trusted_root_records,
blocked_root_records)
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore:
# Fetch all the certificates from the the AOSP repo
cert_records = []
temp_dir = TemporaryDirectory()
try:
# TODO(AD): Stop using shell commands
# Clone the AOSP repo
git_command = self._GIT_CMD.format(repo_url=self._REPO_URL, local_path=temp_dir.name)
with open(os.devnull, "w") as dev_null:
subprocess.check_output(git_command, shell=True, stderr=dev_null)
# Find the latest tag that looks like android-8XXX - we don't care about android-iot or android-wear
tag_list = subprocess.check_output(
self._GIT_FIND_TAG_CMD, shell=True, cwd=temp_dir.name, stderr=dev_null
).decode("ascii")
last_tag = tag_list.strip().rsplit("\n", 1)[1].strip()
# Switch to this tag
subprocess.check_output(
self._GIT_CHECKOUT_TAG_CMD.format(tag=last_tag), shell=True, cwd=temp_dir.name, stderr=dev_null
)
# Inspect each certificate
cert_files_path = Path(temp_dir.name) / "files"
for cert_path in cert_files_path.glob("*"):
with open(cert_path, mode="r") as cert_file:
cert_pem = cert_file.read()
# Parse each certificate and store it if needed
parsed_cert = load_pem_x509_certificate(cert_pem.encode(encoding="ascii"), default_backend())
if should_update_repo:
certs_repo.store_certificate(parsed_cert)
cert_records.append(
ScrapedRootCertificateRecord(
CertificateUtils.get_canonical_subject_name(parsed_cert),
parsed_cert.fingerprint(hashes.SHA256()),
hashes.SHA256(),
)
)
finally:
# Workaround for Windows https://bugs.python.org/issue26660
if platform == "win32":
for file_path in Path(temp_dir.name).glob("**/*"):
os.chmod(file_path, stat.S_IWRITE)
temp_dir.cleanup()
# Finally generate the records
trusted_cert_records = RootRecordsValidator.validate_with_repository(certs_repo, cert_records)
date_fetched = datetime.utcnow().date()
version = last_tag.split("android-")[1]
return TrustStore(PlatformEnum.GOOGLE_AOSP, version, self._REPO_URL, date_fetched, trusted_cert_records)
def fetch(self,
certs_repo: RootCertificatesRepository,
should_update_repo: bool = True) -> TrustStore:
# There's no specific version available in the certdata file
os_version = None
# Then fetch and parse the page
with urlopen(self._PAGE_URL) as response:
page_content = response.read().decode("utf-8")
entries = self._scrape_certdata(page_content)
certificate_entries = [
entry for entry in entries
if isinstance(entry, _CertdataCertificateEntry)
]
trust_entries = [
entry for entry in entries
if isinstance(entry, _CertdataTrustEntry)
]
# Store the certificates we found in the local repo if needed
if should_update_repo:
for cert_entry in certificate_entries:
certs_repo.store_certificate(cert_entry.certificate)
trusted_certificates = RootRecordsValidator.validate_with_repository(
certs_repo,
[
ScrapedRootCertificateRecord(
entry.name, entry.sha1_fingerprint, hashes.SHA1())
for entry in trust_entries
if entry.trust_enum == _CerdataEntryServerAuthTrustEnum.TRUSTED
],
)
blocked_certificates = RootRecordsValidator.validate_with_repository(
certs_repo,
[
ScrapedRootCertificateRecord(
entry.name, entry.sha1_fingerprint, hashes.SHA1())
for entry in trust_entries if entry.trust_enum ==
_CerdataEntryServerAuthTrustEnum.NOT_TRUSTED
],
)
return TrustStore(
PlatformEnum.MOZILLA_NSS,
os_version,
self._PAGE_URL,
datetime.utcnow().date(),
trusted_certificates,
blocked_certificates,
)
def refresh_trust_stores() -> None:
"""Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores.
"""
# Also pass the local certs repo so it gets updated when fetching the trust stores
certs_repo = RootCertificatesRepository.get_default()
# For each supported platform, fetch the trust store
has_any_store_changed = False
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
if platform == PlatformEnum.ORACLE_JAVA:
# TODO: Fix this
print(f"Skipping {platform.name}... TODO: Fixme")
continue
print(f"Refreshing {platform.name}...")
fetched_store = store_fetcher.fetch(platform, certs_repo)
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(
ROOT_PATH
) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml"
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
has_any_store_changed = True
print(f"Detected changes for {platform.name}; updating store...")
with open(store_path, mode="w") as store_file:
yaml.dump(fetched_store,
store_file,
encoding="utf-8",
default_flow_style=False)
else:
print(f"No changes detected for {platform.name}")
# If we are running on travis
if "TRAVIS" in environ:
print("Running on Travis...")
# Enable the deploy step if a change was detected
with open("should_travis_deploy", mode="w") as travis_file:
travis_flag = "1" if has_any_store_changed else "0"
travis_file.write(f"export SHOULD_TRAVIS_DEPLOY={travis_flag}\n")
def export_trust_stores() -> None:
"""Export the content of the trust store of each supported platform to a PEM file at ./export.
"""
certs_repo = RootCertificatesRepository.get_default()
out_pem_folder = ROOT_PATH / 'export'
out_pem_folder.mkdir(exist_ok=True)
# Export each trust store as a PEM file to ./export
print(f'Exporting stores as PEM to {out_pem_folder}...')
for platform in PlatformEnum:
print(f'Exporting {platform.name}...')
store = TrustStore.get_default_for_platform(platform)
all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo)
out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem'
with open(out_pem_path, mode='w') as out_pem_file:
out_pem_file.write(all_certs_pem)
def refresh_trust_stores() -> None:
"""Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores.
"""
# Also pass the local certs repo so it gets updated when fetching the trust stores
certs_repo = RootCertificatesRepository.get_default()
# For each supported platform, fetch the trust store
has_any_store_changed = False
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
print(f'Refreshing {platform.name}...')
fetched_store = store_fetcher.fetch(platform, certs_repo)
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(
ROOT_PATH
) / 'trust_stores' / f'{fetched_store.platform.name.lower()}.yaml'
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
has_any_store_changed = True
print(f'Detected changes for {platform.name}; updating store...')
with open(store_path, mode='w') as store_file:
yaml.dump(fetched_store,
store_file,
encoding='utf-8',
default_flow_style=False)
else:
print(f'No changes detected for {platform.name}')
# If we are running on travis
if 'TRAVIS' in environ:
print('Running on Travis...')
# Enable the deploy step if a change was detected
with open('should_travis_deploy', mode='w') as travis_file:
travis_flag = '1' if has_any_store_changed else '0'
travis_file.write(f'export SHOULD_TRAVIS_DEPLOY={travis_flag}\n')
def fetch(
self,
cert_repo: RootCertificatesRepository,
should_update_repo: bool=True
) -> TrustStore:
# Fetch the latest JRE package
final_url = self._get_latest_download_url()
request = Request(
final_url,
# Cookie set when 'Accept License Agreement' is selected
headers={'Cookie': 'oraclelicense=accept-securebackup-cookie'}
)
response = urlopen(request)
# Parse the JRE package
jre_temp_file = NamedTemporaryFile(delete=False)
try:
jre_temp_file.write(response.read())
jre_temp_file.close()
with JrePackage(jre_temp_file.name) as parsed_jre:
# Extract the data we need
version = parsed_jre.get_version()
blacklisted_file_content = parsed_jre.get_blacklisted_certs()
cacerts_key_store = jks.KeyStore.loads(parsed_jre.get_cacerts(), parsed_jre.get_cacerts_password())
finally:
os.remove(jre_temp_file.name)
# Process the data extracted from the JRE
# Trusted CA certs
scraped_trusted_records = self._extract_trusted_root_records(cacerts_key_store, should_update_repo, cert_repo)
trusted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_trusted_records)
# Blacklisted CA certs - will fail if a blacklisted cert is not already available in the local repo
scraped_blacklisted_records = self._extract_blacklisted_root_records(blacklisted_file_content)
blacklisted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_blacklisted_records)
return TrustStore(
PlatformEnum.ORACLE_JAVA,
version,
final_url,
datetime.utcnow().date(),
trusted_records,
blacklisted_records
)
import os
from pathlib import Path
from trust_stores_observatory.certificates_repository import RootCertificatesRepository
from trust_stores_observatory.trust_store import PlatformEnum, TrustStore
certs_repo = RootCertificatesRepository.get_default()
root_path = Path(os.path.abspath(os.path.dirname(__file__)))
out_pem_folder = root_path / 'export'
out_pem_folder.mkdir(exist_ok=True)
print(f'Exporting stores as PEM to {out_pem_folder}...')
# Export each trust store as a PEM file to ./export
for platform in PlatformEnum:
print(f'Exporting {platform.name}...')
store = TrustStore.get_default_for_platform(platform)
all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo)
out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem'
with open(out_pem_path, mode='w') as out_pem_file:
out_pem_file.write(all_certs_pem)
# For each supported platform, fetch the trust store
has_any_store_changed = False
store_fetcher = TrustStoreFetcher()
for platform in PlatformEnum:
print(f'Refreshing {platform.name}...')
fetched_store = store_fetcher.fetch(platform, certs_repo)
root_path = os.path.abspath(os.path.dirname(__file__))
# Compare the existing trust store with the one we fetched
has_store_changed = False
store_path = Path(
root_path
) / 'trust_stores' / f'{fetched_store.platform.name.lower()}.yaml'
try:
existing_store = TrustStore.from_yaml(store_path)
if existing_store != fetched_store:
has_store_changed = True
except FileNotFoundError:
# The store does not exist in the repo yet
has_store_changed = True
if has_store_changed:
has_any_store_changed = True
print(f'Detected changes for {platform.name}; updating store...')
with open(store_path, mode='w') as store_file:
yaml.dump(fetched_store,
store_file,
encoding='utf-8',
default_flow_style=False)
else:
