def _download_repomds(self):
download_items = []
certs_tmp_dict = {}
for repository in sorted(self.repositories,
key=attrgetter("repo_url")):
repomd_url = urljoin(repository.repo_url, REPOMD_PATH)
repository.tmp_directory = tempfile.mkdtemp(prefix="repo-")
ca_cert, cert, key = self._get_certs_tuple(repository.cert_name)
# Check certificate expiration date
if repository.cert_name:
certs_tmp_dict[repository.cert_name] = repository.cert
item = DownloadItem(source_url=repomd_url,
target_path=os.path.join(
repository.tmp_directory, "repomd.xml"),
ca_cert=ca_cert,
cert=cert,
key=key)
# Save for future status code check
download_items.append(item)
self.downloader.add(item)
for cert_name, cert in certs_tmp_dict.items():
self._check_cert_expiration_date(cert_name, cert)
self.downloader.run()
# Return failed downloads
return {
item.target_path: item.status_code
for item in download_items
if item.status_code not in VALID_HTTP_CODES
}
def _download_head(self):
item = DownloadItem(source_url=URL,
target_path=self._tmp_head())
download_items = [item]
self.downloader.add(item)
self.downloader.run(headers_only=True)
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def download_mock(self, download_item: DownloadItem):
"""Mock downloading function with copying from
testing data directory."""
src = download_item.source_url.split('/')[-1]
src_path = 'test_data/repodata/integration/' + src
if src.endswith('.gz'): # prepare archive file
base = src[:-3]
os.system(f"gzip -c test_data/repodata/{base} > /tmp/{src}")
src_path = f"/tmp/{src}"
shutil.copy(src_path, download_item.target_path)
self.logger.info(f"File {src} mock-downloaded.")
download_item.status_code = 200
def _download_feed(self):
item = DownloadItem(
source_url=f"{OVAL_FEED_BASE_URL}feed.json",
target_path=self.feed_path
)
# Save for future status code check
download_items = [item]
self.downloader.add(item)
self.downloader.run()
# Return failed downloads
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def _download_definitions(self, batch):
download_items = []
for oval_file in batch:
os.makedirs(os.path.dirname(oval_file.local_path), exist_ok=True) # Make sure subdirs exist
item = DownloadItem(
source_url=oval_file.url,
target_path=oval_file.local_path
)
# Save for future status code check
download_items.append(item)
self.downloader.add(item)
self.downloader.run()
# Return failed downloads
return {item.target_path: item.status_code for item in download_items
if item.status_code not in VALID_HTTP_CODES}
def _download_metadata(self, batch):
download_items = []
for repository in batch:
# primary_db has higher priority, use primary.xml if not found
try:
repository.md_files[
"primary_db"] = repository.repomd.get_metadata(
"primary_db")["location"]
except RepoMDTypeNotFound:
repository.md_files[
"primary"] = repository.repomd.get_metadata(
"primary")["location"]
# updateinfo.xml may be missing completely
try:
repository.md_files[
"updateinfo"] = repository.repomd.get_metadata(
"updateinfo")["location"]
except RepoMDTypeNotFound:
pass
try:
repository.md_files[
"modules"] = repository.repomd.get_metadata(
"modules")["location"]
except RepoMDTypeNotFound:
pass
# queue metadata files for download
for md_location in repository.md_files.values():
ca_cert, cert, key = self._get_certs_tuple(
repository.cert_name)
item = DownloadItem(source_url=urljoin(repository.repo_url,
md_location),
target_path=os.path.join(
repository.tmp_directory,
os.path.basename(md_location)),
ca_cert=ca_cert,
cert=cert,
key=key)
download_items.append(item)
self.downloader.add(item)
self.downloader.run()
# Return failed downloads
return {
item.target_path: item.status_code
for item in download_items
if item.status_code not in VALID_HTTP_CODES
}
def _download_xml(self):
self.downloader.add(DownloadItem(source_url=URL,
target_path=self._tmp_xml()))
self.downloader.run()