def fetch(self, cert_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # Fetch the latest JDK package final_url = self._get_latest_download_url() request = Request(final_url) response = urlopen(request) # Parse the JDK package jdk_temp_file = NamedTemporaryFile(delete=False) try: jdk_temp_file.write(response.read()) jdk_temp_file.close() with JdkPackage(jdk_temp_file.name) as parsed_jre: # Extract the data we need version = parsed_jre.get_version() blacklisted_file_content = parsed_jre.get_blacklisted_certs() cacerts_key_store = jks.KeyStore.loads(parsed_jre.get_cacerts(), parsed_jre.get_cacerts_password()) finally: os.remove(jdk_temp_file.name) # Process the data extracted from the JRE # Trusted CA certs scraped_trusted_records = JdkPackage.extract_trusted_root_records( cacerts_key_store, should_update_repo, cert_repo ) trusted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_trusted_records) # Blacklisted CA certs - will fail if a blacklisted cert is not already available in the local repo scraped_blacklisted_records = JdkPackage.extract_blacklisted_root_records(blacklisted_file_content) blacklisted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_blacklisted_records) return TrustStore( PlatformEnum.OPENJDK, version, final_url, datetime.utcnow().date(), trusted_records, blacklisted_records )
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # Find the URL to the latest spreadsheet and download it spreadsheet_url = self._find_latest_root_certificates_url() with urlopen(spreadsheet_url) as response: spreadsheet_content = response.read() with TemporaryFile() as temp_file: temp_file.write(spreadsheet_content) workbook = load_workbook(temp_file) # Extract the data from the spreadsheet version, scraped_trusted_root_records, scraped_blocked_root_records = self._parse_spreadsheet( workbook) # Look for each parsed certificate in the supplied certs repo trusted_root_records = RootRecordsValidator.validate_with_repository( certs_repo, scraped_trusted_root_records) blocked_root_records = RootRecordsValidator.validate_with_repository( certs_repo, scraped_blocked_root_records) date_fetched = datetime.utcnow().date() return TrustStore(PlatformEnum.MICROSOFT_WINDOWS, version, spreadsheet_url, date_fetched, trusted_root_records, blocked_root_records)
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: with urlopen(self._CSV_URL) as response: csv_content = response.read().decode("utf-8") # Extract the data from the CSV scraped_trusted_root_records, scraped_blocked_root_records = self._parse_spreadsheet( csv_content) # Look for each parsed certificate in the supplied certs repo trusted_root_records = RootRecordsValidator.validate_with_repository( certs_repo, scraped_trusted_root_records) blocked_root_records = RootRecordsValidator.validate_with_repository( certs_repo, scraped_blocked_root_records) date_fetched = datetime.utcnow().date() return TrustStore( PlatformEnum.MICROSOFT_WINDOWS, None, self._PAGE_URL, date_fetched, trusted_root_records, blocked_root_records, )
def refresh_trust_stores() -> None: """Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores. """ # Also pass the local certs repo so it gets updated when fetching the trust stores certs_repo = RootCertificatesRepository.get_default() # For each supported platform, fetch the trust store store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: if platform in [PlatformEnum.ORACLE_JAVA, PlatformEnum.OPENJDK]: # TODO: Fix this print(f"Skipping {platform.name}... TODO: Fixme") continue print(f"Refreshing {platform.name}...") fetched_store = store_fetcher.fetch(platform, certs_repo) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path(ROOT_PATH) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml" try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: print(f"Detected changes for {platform.name}; updating store...") with open(store_path, mode="w") as store_file: yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False) else: print(f"No changes detected for {platform.name}")
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # First find the latest page with the list of root certificates os_version, trust_store_url = self._find_latest_root_certificates_page( ) # Then fetch and parse the page with urlopen(trust_store_url) as response: page_content = response.read() parsed_page = BeautifulSoup(page_content, 'html.parser') # There are two divs on the page, one with trusted certificates and one with blocked certificates root_certificates: Dict[str, Set[RootCertificateRecord]] = { 'trusted': set(), 'blocked': set() } # We parse both divs for div_id in ['trusted', 'blocked']: scraped_root_records = self._parse_root_records_in_div( parsed_page, div_id=div_id) # Look for each certificate in the supplied certs repo root_certificates[ div_id] = RootRecordsValidator.validate_with_repository( certs_repo, scraped_root_records) return TrustStore(self._PLATFORM, os_version, trust_store_url, datetime.utcnow().date(), root_certificates['trusted'], root_certificates['blocked'])
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: spreadsheet_url = self._find_latest_root_certificates_url() with urlopen(spreadsheet_url) as response: spreadsheet_content = response.read() with TemporaryFile() as temp_file: temp_file.write(spreadsheet_content) workbook = load_workbook(temp_file) worksheet = workbook.active version = worksheet['A1'].value.split('As of')[1].strip() # Iterate over each row in the work sheet parsed_trusted_root_records = [] parsed_blocked_root_records = [] for row in worksheet.iter_rows(min_row=4, max_col=6, max_row=500): subject_name = row[1].value if subject_name is None: # Most likely indicates the end of the data continue is_cert_trusted = False status = row[4].value.strip() if 'Active' in status: # Some certs are disabled or have a notBefore constraint is_cert_trusted = True fingerprint_cell = row[3].value if fingerprint_cell is None: # One certificate actually does not have the fingerprint cell properly filled logging.error(f'No fingerprint for {subject_name}') continue fingerprint_hex = fingerprint_cell.replace(':', '').strip() fingerprint = bytes(bytearray.fromhex(fingerprint_hex)) if is_cert_trusted: parsed_trusted_root_records.append((subject_name, fingerprint)) else: parsed_blocked_root_records.append((subject_name, fingerprint)) # Look for each certificate in the supplied certs repo trusted_root_records = RootRecordsValidator.validate_with_repository( certs_repo, hashes.SHA256(), parsed_trusted_root_records, ) blocked_root_records = RootRecordsValidator.validate_with_repository( certs_repo, hashes.SHA256(), parsed_blocked_root_records) # TODO: FILTER OUT NON SSL CERTS date_fetched = datetime.utcnow().date() return TrustStore(PlatformEnum.MICROSOFT_WINDOWS, version, spreadsheet_url, date_fetched, trusted_root_records, blocked_root_records)
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # Fetch all the certificates from the the AOSP repo cert_records = [] temp_dir = TemporaryDirectory() try: # TODO(AD): Stop using shell commands # Clone the AOSP repo git_command = self._GIT_CMD.format(repo_url=self._REPO_URL, local_path=temp_dir.name) with open(os.devnull, "w") as dev_null: subprocess.check_output(git_command, shell=True, stderr=dev_null) # Find the latest tag that looks like android-8XXX - we don't care about android-iot or android-wear tag_list = subprocess.check_output( self._GIT_FIND_TAG_CMD, shell=True, cwd=temp_dir.name, stderr=dev_null ).decode("ascii") last_tag = tag_list.strip().rsplit("\n", 1)[1].strip() # Switch to this tag subprocess.check_output( self._GIT_CHECKOUT_TAG_CMD.format(tag=last_tag), shell=True, cwd=temp_dir.name, stderr=dev_null ) # Inspect each certificate cert_files_path = Path(temp_dir.name) / "files" for cert_path in cert_files_path.glob("*"): with open(cert_path, mode="r") as cert_file: cert_pem = cert_file.read() # Parse each certificate and store it if needed parsed_cert = load_pem_x509_certificate(cert_pem.encode(encoding="ascii"), default_backend()) if should_update_repo: certs_repo.store_certificate(parsed_cert) cert_records.append( ScrapedRootCertificateRecord( CertificateUtils.get_canonical_subject_name(parsed_cert), parsed_cert.fingerprint(hashes.SHA256()), hashes.SHA256(), ) ) finally: # Workaround for Windows https://bugs.python.org/issue26660 if platform == "win32": for file_path in Path(temp_dir.name).glob("**/*"): os.chmod(file_path, stat.S_IWRITE) temp_dir.cleanup() # Finally generate the records trusted_cert_records = RootRecordsValidator.validate_with_repository(certs_repo, cert_records) date_fetched = datetime.utcnow().date() version = last_tag.split("android-")[1] return TrustStore(PlatformEnum.GOOGLE_AOSP, version, self._REPO_URL, date_fetched, trusted_cert_records)
def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore: # There's no specific version available in the certdata file os_version = None # Then fetch and parse the page with urlopen(self._PAGE_URL) as response: page_content = response.read().decode("utf-8") entries = self._scrape_certdata(page_content) certificate_entries = [ entry for entry in entries if isinstance(entry, _CertdataCertificateEntry) ] trust_entries = [ entry for entry in entries if isinstance(entry, _CertdataTrustEntry) ] # Store the certificates we found in the local repo if needed if should_update_repo: for cert_entry in certificate_entries: certs_repo.store_certificate(cert_entry.certificate) trusted_certificates = RootRecordsValidator.validate_with_repository( certs_repo, [ ScrapedRootCertificateRecord( entry.name, entry.sha1_fingerprint, hashes.SHA1()) for entry in trust_entries if entry.trust_enum == _CerdataEntryServerAuthTrustEnum.TRUSTED ], ) blocked_certificates = RootRecordsValidator.validate_with_repository( certs_repo, [ ScrapedRootCertificateRecord( entry.name, entry.sha1_fingerprint, hashes.SHA1()) for entry in trust_entries if entry.trust_enum == _CerdataEntryServerAuthTrustEnum.NOT_TRUSTED ], ) return TrustStore( PlatformEnum.MOZILLA_NSS, os_version, self._PAGE_URL, datetime.utcnow().date(), trusted_certificates, blocked_certificates, )
def refresh_trust_stores() -> None: """Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores. """ # Also pass the local certs repo so it gets updated when fetching the trust stores certs_repo = RootCertificatesRepository.get_default() # For each supported platform, fetch the trust store has_any_store_changed = False store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: if platform == PlatformEnum.ORACLE_JAVA: # TODO: Fix this print(f"Skipping {platform.name}... TODO: Fixme") continue print(f"Refreshing {platform.name}...") fetched_store = store_fetcher.fetch(platform, certs_repo) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path( ROOT_PATH ) / "trust_stores" / f"{fetched_store.platform.name.lower()}.yaml" try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: has_any_store_changed = True print(f"Detected changes for {platform.name}; updating store...") with open(store_path, mode="w") as store_file: yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False) else: print(f"No changes detected for {platform.name}") # If we are running on travis if "TRAVIS" in environ: print("Running on Travis...") # Enable the deploy step if a change was detected with open("should_travis_deploy", mode="w") as travis_file: travis_flag = "1" if has_any_store_changed else "0" travis_file.write(f"export SHOULD_TRAVIS_DEPLOY={travis_flag}\n")
def export_trust_stores() -> None: """Export the content of the trust store of each supported platform to a PEM file at ./export. """ certs_repo = RootCertificatesRepository.get_default() out_pem_folder = ROOT_PATH / 'export' out_pem_folder.mkdir(exist_ok=True) # Export each trust store as a PEM file to ./export print(f'Exporting stores as PEM to {out_pem_folder}...') for platform in PlatformEnum: print(f'Exporting {platform.name}...') store = TrustStore.get_default_for_platform(platform) all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo) out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem' with open(out_pem_path, mode='w') as out_pem_file: out_pem_file.write(all_certs_pem)
def refresh_trust_stores() -> None: """Fetch the trust store of each supported platform and update the corresponding local YAML file at ./trust_stores. """ # Also pass the local certs repo so it gets updated when fetching the trust stores certs_repo = RootCertificatesRepository.get_default() # For each supported platform, fetch the trust store has_any_store_changed = False store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: print(f'Refreshing {platform.name}...') fetched_store = store_fetcher.fetch(platform, certs_repo) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path( ROOT_PATH ) / 'trust_stores' / f'{fetched_store.platform.name.lower()}.yaml' try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: has_any_store_changed = True print(f'Detected changes for {platform.name}; updating store...') with open(store_path, mode='w') as store_file: yaml.dump(fetched_store, store_file, encoding='utf-8', default_flow_style=False) else: print(f'No changes detected for {platform.name}') # If we are running on travis if 'TRAVIS' in environ: print('Running on Travis...') # Enable the deploy step if a change was detected with open('should_travis_deploy', mode='w') as travis_file: travis_flag = '1' if has_any_store_changed else '0' travis_file.write(f'export SHOULD_TRAVIS_DEPLOY={travis_flag}\n')
def fetch( self, cert_repo: RootCertificatesRepository, should_update_repo: bool=True ) -> TrustStore: # Fetch the latest JRE package final_url = self._get_latest_download_url() request = Request( final_url, # Cookie set when 'Accept License Agreement' is selected headers={'Cookie': 'oraclelicense=accept-securebackup-cookie'} ) response = urlopen(request) # Parse the JRE package jre_temp_file = NamedTemporaryFile(delete=False) try: jre_temp_file.write(response.read()) jre_temp_file.close() with JrePackage(jre_temp_file.name) as parsed_jre: # Extract the data we need version = parsed_jre.get_version() blacklisted_file_content = parsed_jre.get_blacklisted_certs() cacerts_key_store = jks.KeyStore.loads(parsed_jre.get_cacerts(), parsed_jre.get_cacerts_password()) finally: os.remove(jre_temp_file.name) # Process the data extracted from the JRE # Trusted CA certs scraped_trusted_records = self._extract_trusted_root_records(cacerts_key_store, should_update_repo, cert_repo) trusted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_trusted_records) # Blacklisted CA certs - will fail if a blacklisted cert is not already available in the local repo scraped_blacklisted_records = self._extract_blacklisted_root_records(blacklisted_file_content) blacklisted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_blacklisted_records) return TrustStore( PlatformEnum.ORACLE_JAVA, version, final_url, datetime.utcnow().date(), trusted_records, blacklisted_records )
import os from pathlib import Path from trust_stores_observatory.certificates_repository import RootCertificatesRepository from trust_stores_observatory.trust_store import PlatformEnum, TrustStore certs_repo = RootCertificatesRepository.get_default() root_path = Path(os.path.abspath(os.path.dirname(__file__))) out_pem_folder = root_path / 'export' out_pem_folder.mkdir(exist_ok=True) print(f'Exporting stores as PEM to {out_pem_folder}...') # Export each trust store as a PEM file to ./export for platform in PlatformEnum: print(f'Exporting {platform.name}...') store = TrustStore.get_default_for_platform(platform) all_certs_pem = store.export_trusted_certificates_as_pem(certs_repo) out_pem_path = out_pem_folder / f'{platform.name.lower()}.pem' with open(out_pem_path, mode='w') as out_pem_file: out_pem_file.write(all_certs_pem)
# For each supported platform, fetch the trust store has_any_store_changed = False store_fetcher = TrustStoreFetcher() for platform in PlatformEnum: print(f'Refreshing {platform.name}...') fetched_store = store_fetcher.fetch(platform, certs_repo) root_path = os.path.abspath(os.path.dirname(__file__)) # Compare the existing trust store with the one we fetched has_store_changed = False store_path = Path( root_path ) / 'trust_stores' / f'{fetched_store.platform.name.lower()}.yaml' try: existing_store = TrustStore.from_yaml(store_path) if existing_store != fetched_store: has_store_changed = True except FileNotFoundError: # The store does not exist in the repo yet has_store_changed = True if has_store_changed: has_any_store_changed = True print(f'Detected changes for {platform.name}; updating store...') with open(store_path, mode='w') as store_file: yaml.dump(fetched_store, store_file, encoding='utf-8', default_flow_style=False) else: