def _load_tree_info(self, root_url, file_path=None, file_content=None): """Load the treeinfo metadata. :param root_url: a URL of the installation root :param file_path: a path to a treeinfo file or None :param file_content: a content of a treeinfo file or None :raise InvalidTreeInfoError: if the metadata is invalid """ try: # Load and validate the metadata. tree_info = TreeInfo() if file_content: tree_info.loads(file_content) else: tree_info.load(file_path) tree_info.validate() log.debug("Loaded treeinfo metadata:\n%s", tree_info.dumps()) # Load the release version. release_version = tree_info.release.version.lower() # Create repositories for variants and optional variants. # Child variants (like addons) will be ignored. repo_list = [] for name in tree_info.variants: log.debug("Processing the '%s' variant.", name) # Get the variant metadata. data = tree_info.variants[name] # Create the repo metadata. repo_md = TreeInfoRepoMetadata( repo_name=name, tree_info=data, root_url=root_url, ) repo_list.append(repo_md) except configparser.Error as e: log.debug("Failed to load treeinfo metadata: %s", e) raise InvalidTreeInfoError("Invalid metadata: {}".format(str(e))) from None # Update this treeinfo representation. self._root_url = root_url self._repositories = repo_list self._release_version = release_version log.debug("The treeinfo metadata is loaded.")
def _load_tree_info(self, root_path, file_path=None, file_content=None): """Load the treeinfo metadata. :param root_path: a path to the installation root :param file_path: a path to a treeinfo file or None :param file_content: a content of a treeinfo file or None :raise InvalidTreeInfoError: if the metadata is invalid """ try: # Load and validate the metadata. tree_info = TreeInfo() if file_content: tree_info.loads(file_content) else: tree_info.load(file_path) tree_info.validate() # Load the release version. release_version = tree_info.release.version.lower() # Load the repositories. repo_list = [] for name in tree_info.variants: # Get the variant metadata. data = tree_info.variants[name] # Create the repo metadata. repo_md = TreeInfoRepoMetadata( repo_name=name, tree_info=data, root_path=root_path, ) repo_list.append(repo_md) except configparser.Error as e: log.debug("Failed to load treeinfo metadata: %s", e) raise InvalidTreeInfoError("Invalid metadata: {}".format( str(e))) from None # Update this treeinfo representation. self._root_path = root_path self._repositories = repo_list self._release_version = release_version log.debug("The treeinfo metadata is loaded.")
def append_custom_repo_to_treeinfo(treeinfo_content, path): ti = TreeInfo() ti.loads(treeinfo_content) variant = Variant(ti) variant.id = CUSTOM_REPO_NAME variant.uid = CUSTOM_REPO_NAME variant.name = CUSTOM_REPO_NAME variant.paths.repository = os.path.join(".", CUSTOM_REPO_NAME) variant.paths.packages = os.path.join(".", CUSTOM_REPO_NAME, PACKAGES_DIR) variant.type = "variant" ti.variants.add(variant) ti.dump(path)
class InstallTreeMetadata(object): def __init__(self): self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def load_file(self, root_path): """Loads installation tree metadata from root path. :param root_path: Path to the installation root. :type root_path: str :returns: True if the metadata were loaded, False otherwise. """ self._clear() self._path = root_path if os.access(os.path.join(root_path, ".treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, ".treeinfo")) elif os.access(os.path.join(root_path, "treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, "treeinfo")) else: return False return True def load_url(self, url, proxies, sslverify, sslcert, headers): """Load URL link. This can be also to local file. Parameters here are passed to requests object so make them compatible with requests. :param url: URL poiting to the installation tree. :param proxies: Proxy used for the request. :param sslverify: sslverify object which will be used in request. :param headers: Additional headers of the request. :returns: True if the install tree repo metadata was successfully loaded. False otherwise. :raise: IOError is thrown in case of immediate failure. """ # Retry treeinfo downloads with a progressively longer pause, # so NetworkManager have a chance setup a network and we have # full connectivity before trying to download things. (#1292613) self._clear() xdelay = util.xprogressive_delay() response = None ret_code = [None, None] session = util.requests_session() for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1): if retry_count > 0: time.sleep(next(xdelay)) # Downloading .treeinfo log.info("Trying to download '.treeinfo'") (response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo", headers, proxies, sslverify, sslcert) if response: break # Downloading treeinfo log.info("Trying to download 'treeinfo'") (response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo", headers, proxies, sslverify, sslcert) if response: break # The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo # is on the server. # # Server returned HTTP 404 code -> no need to try again if (ret_code[0] is not None and ret_code[0] == 404 and ret_code[1] is not None and ret_code[1] == 404): response = None log.error("Got HTTP 404 Error when downloading [.]treeinfo files") break if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES: # retry log.info("Retrying repo info download for %s, retrying (%d/%d)", url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES) else: # run out of retries err_msg = ("Repo info download for %s failed after %d retries" % (url, retry_count)) log.error(err_msg) raise IOError("Can't get .treeinfo file from the url {}".format(url)) if response: # get the treeinfo contents text = response.text response.close() self._tree_info.loads(text) self._path = url return True return False @staticmethod def _download_treeinfo_file(session, url, file_name, headers, proxies, verify, cert): try: result = session.get("%s/%s" % (url, file_name), headers=headers, proxies=proxies, verify=verify, cert=cert, timeout=constants.NETWORK_CONNECTION_TIMEOUT) # Server returned HTTP 4XX or 5XX codes if 400 <= result.status_code < 600: log.info("Server returned %i code", result.status_code) return None, result.status_code log.debug("Retrieved '%s' from %s", file_name, url) except requests.exceptions.RequestException as e: log.info("Error downloading '%s': %s", file_name, e) return (None, None) return result, result.status_code def _clear(self): """Clear metadata repositories.""" self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def get_release_version(self): """Get release version from the repository. :returns: Version as lowercase string. :raises: ValueError if version is not present. """ version = self._tree_info.release.version if not version: raise ValueError("Can't read release version from the .treeinfo file! " "Is the .treeinfo file corrupted?") return version.lower() def get_metadata_repos(self): """Get all repository metadata objects.""" if not self._meta_repos: self._read_variants() return self._meta_repos def get_repo_metadata_by_name(self, name): """Get repository metadata object with given name. :param name: Name of the variant to return. :rtype name: VariantRepo object or None. """ for variant in self.get_metadata_repos(): if name == variant.name: return variant return None def get_base_repo_metadata(self, additional_names=None): """Get repo metadata about base repository.""" repos = constants.DEFAULT_REPOS if additional_names: repos.extend(additional_names) for repo_md in self.get_metadata_repos(): if repo_md.name in repos: return repo_md return None def _read_variants(self): for variant_name in self._tree_info.variants: variant_object = self._tree_info.variants[variant_name] self._meta_repos.append(RepoMetadata(variant_name, variant_object, self._path))
class InstallTreeMetadata(object): def __init__(self): self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def load_file(self, root_path): """Loads installation tree metadata from root path. :param root_path: Path to the installation root. :type root_path: str :returns: True if the metadata were loaded, False otherwise. """ self._clear() self._path = root_path if os.access(os.path.join(root_path, ".treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, ".treeinfo")) elif os.access(os.path.join(root_path, "treeinfo"), os.R_OK): self._tree_info.load(os.path.join(root_path, "treeinfo")) else: return False return True def load_url(self, url, proxies, sslverify, headers): """Load URL link. This can be also to local file. Parameters here are passed to requests object so make them compatible with requests. :param url: URL poiting to the installation tree. :param proxies: Proxy used for the request. :param sslverify: sslverify object which will be used in request. :param headers: Additional headers of the request. :returns: True if the install tree repo metadata was successfully loaded. False otherwise. :raise: IOError is thrown in case of immediate failure. """ # Retry treeinfo downloads with a progressively longer pause, # so NetworkManager have a chance setup a network and we have # full connectivity before trying to download things. (#1292613) self._clear() xdelay = util.xprogressive_delay() response = None ret_code = [None, None] session = util.requests_session() for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1): if retry_count > 0: time.sleep(next(xdelay)) # Downloading .treeinfo log.info("Trying to download '.treeinfo'") (response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo", headers, proxies, sslverify) if response: break # Downloading treeinfo log.info("Trying to download 'treeinfo'") (response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo", headers, proxies, sslverify) if response: break # The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo # is on the server. # # Server returned HTTP 404 code -> no need to try again if (ret_code[0] is not None and ret_code[0] == 404 and ret_code[1] is not None and ret_code[1] == 404): response = None log.error("Got HTTP 404 Error when downloading [.]treeinfo files") break if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES: # retry log.info("Retrying repo info download for %s, retrying (%d/%d)", url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES) else: # run out of retries err_msg = ("Repo info download for %s failed after %d retries" % (url, retry_count)) log.error(err_msg) raise IOError("Can't get .treeinfo file from the url {}".format(url)) if response: # get the treeinfo contents text = response.text response.close() self._tree_info.loads(text) self._path = url return True return False @staticmethod def _download_treeinfo_file(session, url, file_name, headers, proxies, verify): try: result = session.get("%s/%s" % (url, file_name), headers=headers, proxies=proxies, verify=verify) # Server returned HTTP 4XX or 5XX codes if 400 <= result.status_code < 600: log.info("Server returned %i code", result.status_code) return None, result.status_code log.debug("Retrieved '%s' from %s", file_name, url) except requests.exceptions.RequestException as e: log.info("Error downloading '%s': %s", file_name, e) return (None, None) return result, result.status_code def _clear(self): """Clear metadata repositories.""" self._tree_info = TreeInfo() self._meta_repos = [] self._path = "" def get_release_version(self): """Get release version from the repository. :returns: Version as lowercase string. :raises: ValueError if version is not present. """ version = self._tree_info.release.version if not version: raise ValueError("Can't read release version from the .treeinfo file! " "Is the .treeinfo file corrupted?") return version.lower() def get_metadata_repos(self): """Get all repository metadata objects.""" if not self._meta_repos: self._read_variants() return self._meta_repos def get_repo_metadata_by_name(self, name): """Get repository metadata object with given name. :param name: Name of the variant to return. :rtype name: VariantRepo object or None. """ for variant in self.get_metadata_repos(): if name == variant.name: return variant return None def get_base_repo_metadata(self, additional_names=None): """Get repo metadata about base repository.""" repos = constants.DEFAULT_REPOS if additional_names: repos.extend(additional_names) for repo_md in self.get_metadata_repos(): if repo_md.name in repos: return repo_md return None def _read_variants(self): for variant_name in self._tree_info.variants: variant_object = self._tree_info.variants[variant_name] self._meta_repos.append(RepoMetadata(variant_name, variant_object, self._path))