def test_read_RHEL_treeinfo(self): for i in os.listdir(self.treeinfo_path): if "RHEL" not in i: continue if "optional" in i: continue path = os.path.join(self.treeinfo_path, i) ti = TreeInfo() ti.load(path) self.assertEqual(ti.release.short, "RHEL") self.assertEqual(ti.release.name, "Red Hat Enterprise Linux") self.assertEqual(ti.release.version, str(ti.release.version)) # variants self.assertEqual(len(ti.variants), 2) # variant: BaseOS var = ti.variants["BaseOS"] self.assertEqual(var.id, "BaseOS") self.assertEqual(var.uid, "BaseOS") self.assertEqual(var.name, "BaseOS") self.assertEqual(var.type, "variant") self.assertEqual(var.paths.packages, "Packages") self.assertEqual(var.paths.repository, ".") # variant: AppStream var = ti.variants["AppStream"] self.assertEqual(var.id, "AppStream") self.assertEqual(var.uid, "AppStream") self.assertEqual(var.name, "AppStream") self.assertEqual(var.type, "variant") self.assertEqual(var.paths.packages, "Packages")
def test_f21_server(self): ti = TreeInfo() ti.load(os.path.join(DIR, "treeinfo/fedora-21-Alpha-Server.x86_64")) # product self.assertEqual(ti.release.name, "Fedora") self.assertEqual(ti.release.short, "Fedora") self.assertEqual(ti.release.version, "21") # tree self.assertEqual(ti.tree.arch, "x86_64") # XXX: converts float to int self.assertEqual(ti.tree.build_timestamp, 1410862874) # variants self.assertEqual(len(ti.variants), 1) # variant: Server var = ti.variants["Server"] self.assertEqual(var.id, "Server") self.assertEqual(var.uid, "Server") self.assertEqual(var.name, "Server") self.assertEqual(var.type, "variant") self.assertEqual(var.paths.packages, "Packages") self.assertEqual(var.paths.repository, ".") # stage2 self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img") self.assertEqual(ti.stage2.instimage, None) # images expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", "boot.iso": "images/boot.iso", } self.assertEqual(ti.images.images["x86_64"], expected_images) expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", } self.assertEqual(ti.images.images["xen"], expected_images) # checksums expected_checksums = { "images/boot.iso": ("sha256", "e9c8b207d25c4af1a36c43a4cc3bf27dd2aefa9757eac63a8147e12e415f45f3"), "images/efiboot.img": ("sha256", "0655a92312272a2ee929f579c7ac5086c0ab75b7453a0293a38502d69a8335d3"), "images/macboot.img": ("sha256", "c25721c2e69ea82eedd78107baf69fc0d2900c5a23d0b689140edc2b372393f9"), "images/pxeboot/initrd.img": ("sha256", "b6cffe60149c0396b3413bf3f73c3a1aa3090048302daeeeb8830911d903aa2f"), "images/pxeboot/upgrade.img": ("sha256", "260ab3a63fb4ef1f0427a6b46d494c638760e0bb4e7e9f58295d0cf4e77886fc"), "images/pxeboot/vmlinuz": ("sha256", "974cbed883608046bacfc7a4f232320750150c2821edcd4650fbbe709a6c1d3d"), "repodata/repomd.xml": ("sha256", "1799b6652f287c73e1f1107ef5efe73a0714aaf34e8e9b81a08d3f741f5dc178"), } self.assertEqual(ti.checksums.checksums, expected_checksums) self._test_identity(ti)
def test_f20(self): ti = TreeInfo() ti.load(os.path.join(DIR, "treeinfo/fedora-20-Fedora.x86_64")) # product self.assertEqual(ti.release.name, "Fedora") self.assertEqual(ti.release.short, "Fedora") self.assertEqual(ti.release.version, "20") # tree self.assertEqual(ti.tree.arch, "x86_64") # XXX: converts float to int self.assertEqual(ti.tree.build_timestamp, 1386857206) # variants self.assertEqual(len(ti.variants), 1) # variant: Fedora var = ti.variants["Fedora"] self.assertEqual(var.id, "Fedora") self.assertEqual(var.uid, "Fedora") self.assertEqual(var.name, "Fedora") self.assertEqual(var.type, "variant") self.assertEqual(var.paths.packages, "Packages") self.assertEqual(var.paths.repository, ".") # stage2 self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img") self.assertEqual(ti.stage2.instimage, None) # images expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", "boot.iso": "images/boot.iso", } self.assertEqual(ti.images.images["x86_64"], expected_images) expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", } self.assertEqual(ti.images.images["xen"], expected_images) # checksums expected_checksums = { "images/boot.iso": ("sha256", "376be7d4855ad6281cb139430606a782fd6189dcb01d7b61448e915802cc350f"), "images/efiboot.img": ("sha256", "3bcba2a9b45366ab3f6d82dbe512421ddcb693f3bcbd9e30cc57aa0fa13c7835"), "images/macboot.img": ("sha256", "698b3492534399e31fa1033f51f25b85d463e743f6a26322a9b117f9aac4fdf3"), "images/pxeboot/initrd.img": ("sha256", "d0a81824e3425b6871ec4896a66e891aed35e291c50dfa30b08f6fc6ab04ca8b"), "images/pxeboot/upgrade.img": ("sha256", "a274b8756290447950f1e48cff9d9d6fd1144579ad98ed9f91929ff9ac1bbfa9"), "images/pxeboot/vmlinuz": ("sha256", "d3a2fbfcf08ac76dfb8135771380ec97ae3129b4e623891adb21bb1cd8ba59f6"), "repodata/repomd.xml": ("sha256", "108b4102829c0839c7712832577fe7da24f0a9491f4dc25d4145efe6aced2ebf"), } self.assertEqual(ti.checksums.checksums, expected_checksums) self._test_identity(ti)
def test_f21_server(self): ti = TreeInfo() ti.load(os.path.join(DIR, "treeinfo/fedora-21-Alpha-Server.x86_64")) # product self.assertEqual(ti.release.name, "Fedora") self.assertEqual(ti.release.short, "Fedora") self.assertEqual(ti.release.version, "21") # tree self.assertEqual(ti.tree.arch, "x86_64") # XXX: converts float to int self.assertEqual(ti.tree.build_timestamp, 1410862874) # variants self.assertEqual(len(ti.variants), 1) # variant: Server var = ti.variants["Server"] self.assertEqual(var.id, "Server") self.assertEqual(var.uid, "Server") self.assertEqual(var.name, "Server") self.assertEqual(var.type, "variant") self.assertEqual(var.paths.packages, "Packages") self.assertEqual(var.paths.repository, ".") # stage2 self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img") self.assertEqual(ti.stage2.instimage, None) # images expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", "boot.iso": "images/boot.iso", } self.assertEqual(ti.images.images["x86_64"], expected_images) expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", } self.assertEqual(ti.images.images["xen"], expected_images) # checksums expected_checksums = { "images/boot.iso": ("sha256", "e9c8b207d25c4af1a36c43a4cc3bf27dd2aefa9757eac63a8147e12e415f45f3"), "images/efiboot.img": ("sha256", "0655a92312272a2ee929f579c7ac5086c0ab75b7453a0293a38502d69a8335d3"), "images/macboot.img": ("sha256", "c25721c2e69ea82eedd78107baf69fc0d2900c5a23d0b689140edc2b372393f9"), "images/pxeboot/initrd.img": ("sha256", "b6cffe60149c0396b3413bf3f73c3a1aa3090048302daeeeb8830911d903aa2f"), "images/pxeboot/upgrade.img": ("sha256", "260ab3a63fb4ef1f0427a6b46d494c638760e0bb4e7e9f58295d0cf4e77886fc"), "images/pxeboot/vmlinuz": ("sha256", "974cbed883608046bacfc7a4f232320750150c2821edcd4650fbbe709a6c1d3d"), "repodata/repomd.xml": ("sha256", "1799b6652f287c73e1f1107ef5efe73a0714aaf34e8e9b81a08d3f741f5dc178"), } self.assertEqual(ti.checksums.checksums, expected_checksums) self._test_identity(ti)
def test_f20(self): ti = TreeInfo() ti.load(os.path.join(DIR, "treeinfo/fedora-20-Fedora.x86_64")) # product self.assertEqual(ti.release.name, "Fedora") self.assertEqual(ti.release.short, "Fedora") self.assertEqual(ti.release.version, "20") # tree self.assertEqual(ti.tree.arch, "x86_64") # XXX: converts float to int self.assertEqual(ti.tree.build_timestamp, 1386857206) # variants self.assertEqual(len(ti.variants), 1) # variant: Fedora var = ti.variants["Fedora"] self.assertEqual(var.id, "Fedora") self.assertEqual(var.uid, "Fedora") self.assertEqual(var.name, "Fedora") self.assertEqual(var.type, "variant") self.assertEqual(var.paths.packages, "Packages") self.assertEqual(var.paths.repository, ".") # stage2 self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img") self.assertEqual(ti.stage2.instimage, None) # images expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", "boot.iso": "images/boot.iso", } self.assertEqual(ti.images.images["x86_64"], expected_images) expected_images = { "kernel": "images/pxeboot/vmlinuz", "initrd": "images/pxeboot/initrd.img", "upgrade": "images/pxeboot/upgrade.img", } self.assertEqual(ti.images.images["xen"], expected_images) # checksums expected_checksums = { "images/boot.iso": ("sha256", "376be7d4855ad6281cb139430606a782fd6189dcb01d7b61448e915802cc350f"), "images/efiboot.img": ("sha256", "3bcba2a9b45366ab3f6d82dbe512421ddcb693f3bcbd9e30cc57aa0fa13c7835"), "images/macboot.img": ("sha256", "698b3492534399e31fa1033f51f25b85d463e743f6a26322a9b117f9aac4fdf3"), "images/pxeboot/initrd.img": ("sha256", "d0a81824e3425b6871ec4896a66e891aed35e291c50dfa30b08f6fc6ab04ca8b"), "images/pxeboot/upgrade.img": ("sha256", "a274b8756290447950f1e48cff9d9d6fd1144579ad98ed9f91929ff9ac1bbfa9"), "images/pxeboot/vmlinuz": ("sha256", "d3a2fbfcf08ac76dfb8135771380ec97ae3129b4e623891adb21bb1cd8ba59f6"), "repodata/repomd.xml": ("sha256", "108b4102829c0839c7712832577fe7da24f0a9491f4dc25d4145efe6aced2ebf"), } self.assertEqual(ti.checksums.checksums, expected_checksums) self._test_identity(ti)
def get_kickstart_data(remote): """ Get Kickstart data from remote. """ kickstart = {} namespaces = [".treeinfo", "treeinfo"] for namespace in namespaces: downloader = remote.get_downloader(url=urljoin(remote.url, namespace)) try: result = downloader.fetch() except ClientResponseError as exc: if 404 == exc.status: continue raise kickstart = TreeInfo() kickstart.load(f=result.path) parser = SortedConfigParser() kickstart.serialize(parser) kickstart_parsed = parser._sections sha256 = result.artifact_attributes["sha256"] kickstart = KickstartData(kickstart_parsed).to_dict(hash=sha256) break return kickstart
def test_centos8_baseos(self): """Publish CentOS 8 BaseOS.""" publication_href = self.rpm_publish(url=CENTOS8_BASEOS_URL) # Test that the .treeinfo file is available and AppStream sub-repo is published correctly body = {"publication": publication_href, "name": "centos8", "base_path": "centos8"} response = self.client.using_handler(api.json_handler).post(RPM_DISTRIBUTION_PATH, body) distribution_task = self.client.get(response["task"]) distribution_href = distribution_task["created_resources"][0] self.addCleanup(self.client.delete, distribution_href) distribution = self.client.get(distribution_href) treeinfo_file = utils.http_get(urljoin(distribution["base_url"], ".treeinfo")) treeinfo = TreeInfo() with NamedTemporaryFile("wb") as temp_file: temp_file.write(treeinfo_file) temp_file.flush() treeinfo.load(f=temp_file.name) for variant_name, variant in treeinfo.variants.variants.items(): if variant_name == "BaseOS": self.assertEqual(variant.paths.repository, ".") self.assertEqual(variant.paths.packages, "Packages") elif variant_name == "AppStream": self.assertEqual(variant.paths.repository, "AppStream") self.assertEqual(variant.paths.packages, "AppStream/Packages") # Find the first package in the 'AppStream/Packages/a/' directory and download it parser = PackagesHtmlParser() a_packages_href = urljoin( distribution["base_url"], "{}/a/".format(variant.paths.packages) ) a_packages_listing = utils.http_get(a_packages_href) parser.feed(a_packages_listing.__str__()) full_package_path = urljoin(a_packages_href, parser.package_href) utils.http_get(full_package_path)
def test_read_treeinfo(self): for i in os.listdir(self.treeinfo_path): if i in ("fedora-8-Everything.x86_64", "fedora-8-Everything.ppc", "fedora-8-Everything.i386"): # version == "development" -> can't read correctly continue path = os.path.join(self.treeinfo_path, i) ti = TreeInfo() ti.load(path) self.assertEqual(ti.release.short, "Fedora") self.assertEqual(ti.release.name, "Fedora") self.assertEqual(ti.release.version, str(int(ti.release.version)))
def test_read_treeinfo(self): for i in os.listdir(self.treeinfo_path): if i in ("fedora-8-Everything.x86_64", "fedora-8-Everything.ppc", "fedora-8-Everything.i386"): # version == "development" -> can't read correctly continue path = os.path.join(self.treeinfo_path, i) ti = TreeInfo() ti.load(path) self.assertEqual(ti.release.short, "Fedora") self.assertEqual(ti.release.name, "Fedora") self.assertEqual(ti.release.version, str(int(ti.release.version)))
def test_read_RHEL_Server_optional_treeinfo(self): path = os.path.join(self.treeinfo_path, "RHEL-7-Server-optional.x86_64") ti = TreeInfo() ti.load(path) var = ti.variants["Server-optional"] self.assertEqual(var.id, "optional") self.assertEqual(var.uid, "Server-optional") self.assertEqual(var.name, "optional") self.assertEqual(var.type, "optional") self.assertEqual(var.paths.packages, "Packages") self.assertEqual(var.paths.repository, ".")
def _load_tree_info(self, root_url, file_path=None, file_content=None): """Load the treeinfo metadata. :param root_url: a URL of the installation root :param file_path: a path to a treeinfo file or None :param file_content: a content of a treeinfo file or None :raise InvalidTreeInfoError: if the metadata is invalid """ try: # Load and validate the metadata. tree_info = TreeInfo() if file_content: tree_info.loads(file_content) else: tree_info.load(file_path) tree_info.validate() log.debug("Loaded treeinfo metadata:\n%s", tree_info.dumps()) # Load the release version. release_version = tree_info.release.version.lower() # Create repositories for variants and optional variants. # Child variants (like addons) will be ignored. repo_list = [] for name in tree_info.variants: log.debug("Processing the '%s' variant.", name) # Get the variant metadata. data = tree_info.variants[name] # Create the repo metadata. repo_md = TreeInfoRepoMetadata( repo_name=name, tree_info=data, root_url=root_url, ) repo_list.append(repo_md) except configparser.Error as e: log.debug("Failed to load treeinfo metadata: %s", e) raise InvalidTreeInfoError("Invalid metadata: {}".format(str(e))) from None # Update this treeinfo representation. self._root_url = root_url self._repositories = repo_list self._release_version = release_version log.debug("The treeinfo metadata is loaded.")
def _test_identity(self, ti): first = os.path.join(self.tmp_dir, "first") second = os.path.join(self.tmp_dir, "second") # write original file ti.dump(first) # read file and write it back ti = TreeInfo() ti.load(first) ti.dump(second) # check if first and second files are identical self.assertSameFiles(first, second)
def _test_identity(self, ti): first = os.path.join(self.tmp_dir, "first") second = os.path.join(self.tmp_dir, "second") # write original file ti.dump(first) # read file and write it back ti = TreeInfo() ti.load(first) ti.dump(second) # check if first and second files are identical self.assertSameFiles(first, second)
def _load_tree_info(self, root_path, file_path=None, file_content=None): """Load the treeinfo metadata. :param root_path: a path to the installation root :param file_path: a path to a treeinfo file or None :param file_content: a content of a treeinfo file or None :raise InvalidTreeInfoError: if the metadata is invalid """ try: # Load and validate the metadata. tree_info = TreeInfo() if file_content: tree_info.loads(file_content) else: tree_info.load(file_path) tree_info.validate() # Load the release version. release_version = tree_info.release.version.lower() # Load the repositories. repo_list = [] for name in tree_info.variants: # Get the variant metadata. data = tree_info.variants[name] # Create the repo metadata. repo_md = TreeInfoRepoMetadata( repo_name=name, tree_info=data, root_path=root_path, ) repo_list.append(repo_md) except configparser.Error as e: log.debug("Failed to load treeinfo metadata: %s", e) raise InvalidTreeInfoError("Invalid metadata: {}".format( str(e))) from None # Update this treeinfo representation. self._root_path = root_path self._repositories = repo_list self._release_version = release_version log.debug("The treeinfo metadata is loaded.")
def test_parse_opensuse_treeinfo(self): ti = TreeInfo() ti.load(os.path.join(self.treeinfo_path, "opensuse")) self.assertEqual(ti.release.name, "openSUSE Leap") self.assertEqual(ti.release.short, "openSUSE Leap") self.assertEqual(ti.release.version, "15.1") self.assertEqual(ti.variants.variants, {}) self.assertEqual(ti.tree.arch, "x86_64") self.assertEqual(ti.tree.platforms, set(["x86_64", "xen"])) self.assertEqual( ti.images.images, { "x86_64": { "initrd": "boot/x86_64/loader/initrd", "kernel": "boot/x86_64/loader/linux", }, "xen": { "initrd": "boot/x86_64/loader/initrd", "kernel": "boot/x86_64/loader/linux", }, }, )
class InstallTreeMetadata(object): def __init__(self): self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def load_file(self, root_path): """Loads installation tree metadata from root path. :param root_path: Path to the installation root. :type root_path: str :returns: True if the metadata were loaded, False otherwise. """ self._clear() self._path = root_path if os.access(os.path.join(root_path, ".treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, ".treeinfo")) elif os.access(os.path.join(root_path, "treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, "treeinfo")) else: return False return True def load_url(self, url, proxies, sslverify, sslcert, headers): """Load URL link. This can be also to local file. Parameters here are passed to requests object so make them compatible with requests. :param url: URL poiting to the installation tree. :param proxies: Proxy used for the request. :param sslverify: sslverify object which will be used in request. :param headers: Additional headers of the request. :returns: True if the install tree repo metadata was successfully loaded. False otherwise. :raise: IOError is thrown in case of immediate failure. """ # Retry treeinfo downloads with a progressively longer pause, # so NetworkManager have a chance setup a network and we have # full connectivity before trying to download things. (#1292613) self._clear() xdelay = util.xprogressive_delay() response = None ret_code = [None, None] session = util.requests_session() for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1): if retry_count > 0: time.sleep(next(xdelay)) # Downloading .treeinfo log.info("Trying to download '.treeinfo'") (response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo", headers, proxies, sslverify, sslcert) if response: break # Downloading treeinfo log.info("Trying to download 'treeinfo'") (response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo", headers, proxies, sslverify, sslcert) if response: break # The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo # is on the server. # # Server returned HTTP 404 code -> no need to try again if (ret_code[0] is not None and ret_code[0] == 404 and ret_code[1] is not None and ret_code[1] == 404): response = None log.error("Got HTTP 404 Error when downloading [.]treeinfo files") break if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES: # retry log.info("Retrying repo info download for %s, retrying (%d/%d)", url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES) else: # run out of retries err_msg = ("Repo info download for %s failed after %d retries" % (url, retry_count)) log.error(err_msg) raise IOError("Can't get .treeinfo file from the url {}".format(url)) if response: # get the treeinfo contents text = response.text response.close() self._tree_info.loads(text) self._path = url return True return False @staticmethod def _download_treeinfo_file(session, url, file_name, headers, proxies, verify, cert): try: result = session.get("%s/%s" % (url, file_name), headers=headers, proxies=proxies, verify=verify, cert=cert, timeout=constants.NETWORK_CONNECTION_TIMEOUT) # Server returned HTTP 4XX or 5XX codes if 400 <= result.status_code < 600: log.info("Server returned %i code", result.status_code) return None, result.status_code log.debug("Retrieved '%s' from %s", file_name, url) except requests.exceptions.RequestException as e: log.info("Error downloading '%s': %s", file_name, e) return (None, None) return result, result.status_code def _clear(self): """Clear metadata repositories.""" self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def get_release_version(self): """Get release version from the repository. :returns: Version as lowercase string. :raises: ValueError if version is not present. """ version = self._tree_info.release.version if not version: raise ValueError("Can't read release version from the .treeinfo file! " "Is the .treeinfo file corrupted?") return version.lower() def get_metadata_repos(self): """Get all repository metadata objects.""" if not self._meta_repos: self._read_variants() return self._meta_repos def get_repo_metadata_by_name(self, name): """Get repository metadata object with given name. :param name: Name of the variant to return. :rtype name: VariantRepo object or None. """ for variant in self.get_metadata_repos(): if name == variant.name: return variant return None def get_base_repo_metadata(self, additional_names=None): """Get repo metadata about base repository.""" repos = constants.DEFAULT_REPOS if additional_names: repos.extend(additional_names) for repo_md in self.get_metadata_repos(): if repo_md.name in repos: return repo_md return None def _read_variants(self): for variant_name in self._tree_info.variants: variant_object = self._tree_info.variants[variant_name] self._meta_repos.append(RepoMetadata(variant_name, variant_object, self._path))
class InstallTreeMetadata(object): def __init__(self): self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def load_file(self, root_path): """Loads installation tree metadata from root path. :param root_path: Path to the installation root. :type root_path: str :returns: True if the metadata were loaded, False otherwise. """ self._clear() self._path = root_path if os.access(os.path.join(root_path, ".treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, ".treeinfo")) elif os.access(os.path.join(root_path, "treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, "treeinfo")) else: return False return True def load_url(self, url, proxies, sslverify, headers): """Load URL link. This can be also to local file. Parameters here are passed to requests object so make them compatible with requests. :param url: URL poiting to the installation tree. :param proxies: Proxy used for the request. :param sslverify: sslverify object which will be used in request. :param headers: Additional headers of the request. :returns: True if the install tree repo metadata was successfully loaded. False otherwise. :raise: IOError is thrown in case of immediate failure. """ # Retry treeinfo downloads with a progressively longer pause, # so NetworkManager have a chance setup a network and we have # full connectivity before trying to download things. (#1292613) self._clear() xdelay = util.xprogressive_delay() response = None ret_code = [None, None] session = util.requests_session() for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1): if retry_count > 0: time.sleep(next(xdelay)) # Downloading .treeinfo log.info("Trying to download '.treeinfo'") (response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo", headers, proxies, sslverify) if response: break # Downloading treeinfo log.info("Trying to download 'treeinfo'") (response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo", headers, proxies, sslverify) if response: break # The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo # is on the server. # # Server returned HTTP 404 code -> no need to try again if (ret_code[0] is not None and ret_code[0] == 404 and ret_code[1] is not None and ret_code[1] == 404): response = None log.error("Got HTTP 404 Error when downloading [.]treeinfo files") break if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES: # retry log.info("Retrying repo info download for %s, retrying (%d/%d)", url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES) else: # run out of retries err_msg = ("Repo info download for %s failed after %d retries" % (url, retry_count)) log.error(err_msg) raise IOError("Can't get .treeinfo file from the url {}".format(url)) if response: # get the treeinfo contents text = response.text response.close() self._tree_info.loads(text) self._path = url return True return False @staticmethod def _download_treeinfo_file(session, url, file_name, headers, proxies, verify): try: result = session.get("%s/%s" % (url, file_name), headers=headers, proxies=proxies, verify=verify) # Server returned HTTP 4XX or 5XX codes if 400 <= result.status_code < 600: log.info("Server returned %i code", result.status_code) return None, result.status_code log.debug("Retrieved '%s' from %s", file_name, url) except requests.exceptions.RequestException as e: log.info("Error downloading '%s': %s", file_name, e) return (None, None) return result, result.status_code def _clear(self): """Clear metadata repositories.""" self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def get_release_version(self): """Get release version from the repository. :returns: Version as lowercase string. :raises: ValueError if version is not present. """ version = self._tree_info.release.version if not version: raise ValueError("Can't read release version from the .treeinfo file! " "Is the .treeinfo file corrupted?") return version.lower() def get_metadata_repos(self): """Get all repository metadata objects.""" if not self._meta_repos: self._read_variants() return self._meta_repos def get_repo_metadata_by_name(self, name): """Get repository metadata object with given name. :param name: Name of the variant to return. :rtype name: VariantRepo object or None. """ for variant in self.get_metadata_repos(): if name == variant.name: return variant return None def get_base_repo_metadata(self, additional_names=None): """Get repo metadata about base repository.""" repos = constants.DEFAULT_REPOS if additional_names: repos.extend(additional_names) for repo_md in self.get_metadata_repos(): if repo_md.name in repos: return repo_md return None def _read_variants(self): for variant_name in self._tree_info.variants: variant_object = self._tree_info.variants[variant_name] self._meta_repos.append(RepoMetadata(variant_name, variant_object, self._path))