def _download(self): cpe_dict_item = DownloadItem(source_url=CPE_DICT_URL, target_path=self._cpe_dict_path()) repo_mapping_item = DownloadItem(source_url=REPO_TO_CPE_URL, target_path=self._repo_mapping_path()) download_items = [cpe_dict_item, repo_mapping_item] for item in download_items: self.downloader.add(item) self.downloader.run() return { item.target_path: item.status_code for item in download_items if item.status_code not in VALID_HTTP_CODES }
def _download_metadata(self, batch): download_items = [] for repository in batch: # primary_db has higher priority, use primary.xml if not found try: repository.md_files["primary_db"] = repository.repomd.get_metadata("primary_db")["location"] except RepoMDTypeNotFound: repository.md_files["primary"] = repository.repomd.get_metadata("primary")["location"] # updateinfo.xml may be missing completely try: repository.md_files["updateinfo"] = repository.repomd.get_metadata("updateinfo")["location"] except RepoMDTypeNotFound: pass try: repository.md_files["modules"] = repository.repomd.get_metadata("modules")["location"] except RepoMDTypeNotFound: pass # queue metadata files for download for md_location in repository.md_files.values(): ca_cert, cert, key = self._get_certs_tuple(repository.cert_name) item = DownloadItem( source_url=urljoin(repository.repo_url, md_location), target_path=os.path.join(repository.tmp_directory, os.path.basename(md_location)), ca_cert=ca_cert, cert=cert, key=key ) download_items.append(item) self.downloader.add(item) self.downloader.run() # Return failed downloads return {item.target_path: item.status_code for item in download_items if item.status_code not in VALID_HTTP_CODES}
def _download_metadata(self, batch): for repository in batch: # primary_db has higher priority, use primary.xml if not found try: repository.md_files[ "primary_db"] = repository.repomd.get_metadata( "primary_db")["location"] except RepoMDTypeNotFound: repository.md_files[ "primary"] = repository.repomd.get_metadata( "primary")["location"] # updateinfo.xml may be missing completely try: repository.md_files[ "updateinfo"] = repository.repomd.get_metadata( "updateinfo")["location"] except RepoMDTypeNotFound: pass # queue metadata files for download for md_location in repository.md_files.values(): ca_cert, cert, key = self._get_certs_tuple( repository.cert_name) self.downloader.add( DownloadItem(source_url=urljoin(repository.repo_url, md_location), target_path=os.path.join( repository.tmp_directory, os.path.basename(md_location)), ca_cert=ca_cert, cert=cert, key=key)) self.downloader.run()
def _download_repomds(self): download_items = [] certs_tmp_dict = {} for repository in self.repositories: repomd_url = urljoin(repository.repo_url, REPOMD_PATH) repository.tmp_directory = tempfile.mkdtemp(prefix="repo-") ca_cert, cert, key = self._get_certs_tuple(repository.cert_name) # Check certificate expiration date if repository.cert_name: certs_tmp_dict[repository.cert_name] = cert item = DownloadItem( source_url=repomd_url, target_path=os.path.join(repository.tmp_directory, "repomd.xml"), ca_cert=ca_cert, cert=cert, key=key ) # Save for future status code check download_items.append(item) self.downloader.add(item) for cert_name, cert in certs_tmp_dict.items(): self._check_cert_expiration_date(cert_name, cert) self.downloader.run() # Return failed downloads return {item.target_path: item.status_code for item in download_items if item.status_code not in VALID_HTTP_CODES}
def _download_head(self): item = DownloadItem(source_url=URL, target_path=self._tmp_head()) download_items = [item] self.downloader.add(item) self.downloader.run(headers_only=True) return {item.target_path: item.status_code for item in download_items if item.status_code not in VALID_HTTP_CODES}
def download_mock(self, download_item: DownloadItem): """Mock downloading function with copying from testing data directory.""" src = download_item.source_url.split('/')[-1] src_path = 'test_data/repodata/integration/' + src if src.endswith('.gz'): # prepare archive file base = src[:-3] os.system(f"gzip -c test_data/repodata/{base} > /tmp/{src}") src_path = f"/tmp/{src}" shutil.copy(src_path, download_item.target_path) self.logger.info(f"File {src} mock-downloaded.") download_item.status_code = 200
def _download_meta(self): download_items = [] for repo in self.repos: repo.tmp_directory = tempfile.mkdtemp(prefix="cverepo-") item = DownloadItem( source_url=repo.meta_url(), target_path=repo.meta_tmp() ) # Save for future status code check download_items.append(item) self.downloader.add(item) self.downloader.run() # Return failed downloads return {item.target_path: item.status_code for item in download_items if item.status_code not in VALID_HTTP_CODES}
def _download_repomds(self): download_items = [] for repository in self.repositories: repomd_url = urljoin(repository.repo_url, REPOMD_PATH) repository.tmp_directory = tempfile.mkdtemp(prefix="repo-") item = DownloadItem( source_url=repomd_url, target_path=os.path.join(repository.tmp_directory, "repomd.xml") ) # Save for future status code check download_items.append(item) self.downloader.add(item) self.downloader.run() # Return failed downloads return {item.target_path: item.status_code for item in download_items if item.status_code not in VALID_HTTP_CODES}
def _download_xml(self): self.downloader.add(DownloadItem(source_url=URL, target_path=self._tmp_xml())) self.downloader.run()
def _download_json(self, batch): for repo in batch: self.downloader.add( DownloadItem(source_url=repo.json_url(), target_path=repo.json_tmpgz())) self.downloader.run()