def _download(self):
cpe_dict_item = DownloadItem(source_url=CPE_DICT_URL,
target_path=self._cpe_dict_path())
repo_mapping_item = DownloadItem(source_url=REPO_TO_CPE_URL,
target_path=self._repo_mapping_path())
download_items = [cpe_dict_item, repo_mapping_item]
for item in download_items:
self.downloader.add(item)
self.downloader.run()
return {
item.target_path: item.status_code
for item in download_items
if item.status_code not in VALID_HTTP_CODES
}
def _download_metadata(self, batch):
download_items = []
for repository in batch:
# primary_db has higher priority, use primary.xml if not found
try:
repository.md_files["primary_db"] = repository.repomd.get_metadata("primary_db")["location"]
except RepoMDTypeNotFound:
repository.md_files["primary"] = repository.repomd.get_metadata("primary")["location"]
# updateinfo.xml may be missing completely
try:
repository.md_files["updateinfo"] = repository.repomd.get_metadata("updateinfo")["location"]
except RepoMDTypeNotFound:
pass
try:
repository.md_files["modules"] = repository.repomd.get_metadata("modules")["location"]
except RepoMDTypeNotFound:
pass
# queue metadata files for download
for md_location in repository.md_files.values():
ca_cert, cert, key = self._get_certs_tuple(repository.cert_name)
item = DownloadItem(
source_url=urljoin(repository.repo_url, md_location),
target_path=os.path.join(repository.tmp_directory, os.path.basename(md_location)),
ca_cert=ca_cert,
cert=cert,
key=key
)
download_items.append(item)
self.downloader.add(item)
self.downloader.run()
# Return failed downloads
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def _download_metadata(self, batch):
for repository in batch:
# primary_db has higher priority, use primary.xml if not found
try:
repository.md_files[
"primary_db"] = repository.repomd.get_metadata(
"primary_db")["location"]
except RepoMDTypeNotFound:
repository.md_files[
"primary"] = repository.repomd.get_metadata(
"primary")["location"]
# updateinfo.xml may be missing completely
try:
repository.md_files[
"updateinfo"] = repository.repomd.get_metadata(
"updateinfo")["location"]
except RepoMDTypeNotFound:
pass
# queue metadata files for download
for md_location in repository.md_files.values():
ca_cert, cert, key = self._get_certs_tuple(
repository.cert_name)
self.downloader.add(
DownloadItem(source_url=urljoin(repository.repo_url,
md_location),
target_path=os.path.join(
repository.tmp_directory,
os.path.basename(md_location)),
ca_cert=ca_cert,
cert=cert,
key=key))
self.downloader.run()
def _download_repomds(self):
download_items = []
certs_tmp_dict = {}
for repository in self.repositories:
repomd_url = urljoin(repository.repo_url, REPOMD_PATH)
repository.tmp_directory = tempfile.mkdtemp(prefix="repo-")
ca_cert, cert, key = self._get_certs_tuple(repository.cert_name)
# Check certificate expiration date
if repository.cert_name:
certs_tmp_dict[repository.cert_name] = cert
item = DownloadItem(
source_url=repomd_url,
target_path=os.path.join(repository.tmp_directory, "repomd.xml"),
ca_cert=ca_cert,
cert=cert,
key=key
)
# Save for future status code check
download_items.append(item)
self.downloader.add(item)
for cert_name, cert in certs_tmp_dict.items():
self._check_cert_expiration_date(cert_name, cert)
self.downloader.run()
# Return failed downloads
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def _download_head(self):
item = DownloadItem(source_url=URL,
target_path=self._tmp_head())
download_items = [item]
self.downloader.add(item)
self.downloader.run(headers_only=True)
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def download_mock(self, download_item: DownloadItem):
"""Mock downloading function with copying from
testing data directory."""
src = download_item.source_url.split('/')[-1]
src_path = 'test_data/repodata/integration/' + src
if src.endswith('.gz'): # prepare archive file
base = src[:-3]
os.system(f"gzip -c test_data/repodata/{base} > /tmp/{src}")
src_path = f"/tmp/{src}"
shutil.copy(src_path, download_item.target_path)
self.logger.info(f"File {src} mock-downloaded.")
download_item.status_code = 200
def _download_meta(self):
download_items = []
for repo in self.repos:
repo.tmp_directory = tempfile.mkdtemp(prefix="cverepo-")
item = DownloadItem(
source_url=repo.meta_url(),
target_path=repo.meta_tmp()
)
# Save for future status code check
download_items.append(item)
self.downloader.add(item)
self.downloader.run()
# Return failed downloads
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def _download_repomds(self):
download_items = []
for repository in self.repositories:
repomd_url = urljoin(repository.repo_url, REPOMD_PATH)
repository.tmp_directory = tempfile.mkdtemp(prefix="repo-")
item = DownloadItem(
source_url=repomd_url,
target_path=os.path.join(repository.tmp_directory, "repomd.xml")
)
# Save for future status code check
download_items.append(item)
self.downloader.add(item)
self.downloader.run()
# Return failed downloads
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def _download_xml(self):
self.downloader.add(DownloadItem(source_url=URL,
target_path=self._tmp_xml()))
self.downloader.run()
def _download_json(self, batch):
for repo in batch:
self.downloader.add(
DownloadItem(source_url=repo.json_url(),
target_path=repo.json_tmpgz()))
self.downloader.run()
