def test_f20(self):
ti = TreeInfo()
ti.load(os.path.join(DIR, "treeinfo/fedora-20-Fedora.x86_64"))
# product
self.assertEqual(ti.release.name, "Fedora")
self.assertEqual(ti.release.short, "Fedora")
self.assertEqual(ti.release.version, "20")
# tree
self.assertEqual(ti.tree.arch, "x86_64")
# XXX: converts float to int
self.assertEqual(ti.tree.build_timestamp, 1386857206)
# variants
self.assertEqual(len(ti.variants), 1)
# variant: Fedora
var = ti.variants["Fedora"]
self.assertEqual(var.id, "Fedora")
self.assertEqual(var.uid, "Fedora")
self.assertEqual(var.name, "Fedora")
self.assertEqual(var.type, "variant")
self.assertEqual(var.paths.packages, "Packages")
self.assertEqual(var.paths.repository, ".")
# stage2
self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img")
self.assertEqual(ti.stage2.instimage, None)
# images
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
"boot.iso": "images/boot.iso",
}
self.assertEqual(ti.images.images["x86_64"], expected_images)
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
}
self.assertEqual(ti.images.images["xen"], expected_images)
# checksums
expected_checksums = {
"images/boot.iso": ("sha256", "376be7d4855ad6281cb139430606a782fd6189dcb01d7b61448e915802cc350f"),
"images/efiboot.img": ("sha256", "3bcba2a9b45366ab3f6d82dbe512421ddcb693f3bcbd9e30cc57aa0fa13c7835"),
"images/macboot.img": ("sha256", "698b3492534399e31fa1033f51f25b85d463e743f6a26322a9b117f9aac4fdf3"),
"images/pxeboot/initrd.img": ("sha256", "d0a81824e3425b6871ec4896a66e891aed35e291c50dfa30b08f6fc6ab04ca8b"),
"images/pxeboot/upgrade.img": ("sha256", "a274b8756290447950f1e48cff9d9d6fd1144579ad98ed9f91929ff9ac1bbfa9"),
"images/pxeboot/vmlinuz": ("sha256", "d3a2fbfcf08ac76dfb8135771380ec97ae3129b4e623891adb21bb1cd8ba59f6"),
"repodata/repomd.xml": ("sha256", "108b4102829c0839c7712832577fe7da24f0a9491f4dc25d4145efe6aced2ebf"),
}
self.assertEqual(ti.checksums.checksums, expected_checksums)
self._test_identity(ti)
def test_f21_server(self):
ti = TreeInfo()
ti.load(os.path.join(DIR, "treeinfo/fedora-21-Alpha-Server.x86_64"))
# product
self.assertEqual(ti.release.name, "Fedora")
self.assertEqual(ti.release.short, "Fedora")
self.assertEqual(ti.release.version, "21")
# tree
self.assertEqual(ti.tree.arch, "x86_64")
# XXX: converts float to int
self.assertEqual(ti.tree.build_timestamp, 1410862874)
# variants
self.assertEqual(len(ti.variants), 1)
# variant: Server
var = ti.variants["Server"]
self.assertEqual(var.id, "Server")
self.assertEqual(var.uid, "Server")
self.assertEqual(var.name, "Server")
self.assertEqual(var.type, "variant")
self.assertEqual(var.paths.packages, "Packages")
self.assertEqual(var.paths.repository, ".")
# stage2
self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img")
self.assertEqual(ti.stage2.instimage, None)
# images
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
"boot.iso": "images/boot.iso",
}
self.assertEqual(ti.images.images["x86_64"], expected_images)
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
}
self.assertEqual(ti.images.images["xen"], expected_images)
# checksums
expected_checksums = {
"images/boot.iso": ("sha256", "e9c8b207d25c4af1a36c43a4cc3bf27dd2aefa9757eac63a8147e12e415f45f3"),
"images/efiboot.img": ("sha256", "0655a92312272a2ee929f579c7ac5086c0ab75b7453a0293a38502d69a8335d3"),
"images/macboot.img": ("sha256", "c25721c2e69ea82eedd78107baf69fc0d2900c5a23d0b689140edc2b372393f9"),
"images/pxeboot/initrd.img": ("sha256", "b6cffe60149c0396b3413bf3f73c3a1aa3090048302daeeeb8830911d903aa2f"),
"images/pxeboot/upgrade.img": ("sha256", "260ab3a63fb4ef1f0427a6b46d494c638760e0bb4e7e9f58295d0cf4e77886fc"),
"images/pxeboot/vmlinuz": ("sha256", "974cbed883608046bacfc7a4f232320750150c2821edcd4650fbbe709a6c1d3d"),
"repodata/repomd.xml": ("sha256", "1799b6652f287c73e1f1107ef5efe73a0714aaf34e8e9b81a08d3f741f5dc178"),
}
self.assertEqual(ti.checksums.checksums, expected_checksums)
self._test_identity(ti)
def test_create(self):
ti = TreeInfo()
ti.release.name = "Fedora"
ti.release.short = "F"
ti.release.version = "20"
ti.tree.arch = "x86_64"
ti.tree.build_timestamp = 123456
variant = Variant(ti)
variant.id = "Fedora"
variant.uid = "Fedora"
variant.name = "Fedora"
variant.type = "variant"
variant.paths.repository = "repo"
variant.paths.packages = "pkgs"
variant.paths.source_repository = "src repo"
variant.paths.source_packages = "src pkgs"
variant.paths.debug_repository = "debug repo"
variant.paths.debug_packages = "debug pkgs"
variant.paths.identity = "cert.pem"
ti.variants.add(variant)
ti.dump(self.ini_path)
self._test_identity(ti)
def test_read_treeinfo(self):
for i in os.listdir(self.treeinfo_path):
if i in ("fedora-8-Everything.x86_64", "fedora-8-Everything.ppc", "fedora-8-Everything.i386"):
# version == "development" -> can't read correctly
continue
path = os.path.join(self.treeinfo_path, i)
ti = TreeInfo()
ti.load(path)
self.assertEqual(ti.release.short, "Fedora")
self.assertEqual(ti.release.name, "Fedora")
self.assertEqual(ti.release.version, str(int(ti.release.version)))
def _test_identity(self, ti):
first = os.path.join(self.tmp_dir, "first")
second = os.path.join(self.tmp_dir, "second")
# write original file
ti.dump(first)
# read file and write it back
ti = TreeInfo()
ti.load(first)
ti.dump(second)
# check if first and second files are identical
self.assertSameFiles(first, second)
class InstallTreeMetadata(object):
def __init__(self):
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
def load_file(self, root_path):
"""Loads installation tree metadata from root path.
:param root_path: Path to the installation root.
:type root_path: str
:returns: True if the metadata were loaded, False otherwise.
"""
self._clear()
self._path = root_path
if os.access(os.path.join(root_path, ".treeinfo"), os.R_OK):
self._tree_info.load(os.path.join(root_path, ".treeinfo"))
elif os.access(os.path.join(root_path, "treeinfo"), os.R_OK):
self._tree_info.load(os.path.join(root_path, "treeinfo"))
else:
return False
return True
def load_url(self, url, proxies, sslverify, sslcert, headers):
"""Load URL link.
This can be also to local file.
Parameters here are passed to requests object so make them compatible with requests.
:param url: URL poiting to the installation tree.
:param proxies: Proxy used for the request.
:param sslverify: sslverify object which will be used in request.
:param headers: Additional headers of the request.
:returns: True if the install tree repo metadata was successfully loaded. False otherwise.
:raise: IOError is thrown in case of immediate failure.
"""
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
self._clear()
xdelay = util.xprogressive_delay()
response = None
ret_code = [None, None]
session = util.requests_session()
for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1):
if retry_count > 0:
time.sleep(next(xdelay))
# Downloading .treeinfo
log.info("Trying to download '.treeinfo'")
(response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo",
headers, proxies,
sslverify, sslcert)
if response:
break
# Downloading treeinfo
log.info("Trying to download 'treeinfo'")
(response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo",
headers, proxies,
sslverify, sslcert)
if response:
break
# The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo
# is on the server.
#
# Server returned HTTP 404 code -> no need to try again
if (ret_code[0] is not None and ret_code[0] == 404
and ret_code[1] is not None and ret_code[1] == 404):
response = None
log.error("Got HTTP 404 Error when downloading [.]treeinfo files")
break
if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES:
# retry
log.info("Retrying repo info download for %s, retrying (%d/%d)",
url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES)
else:
# run out of retries
err_msg = ("Repo info download for %s failed after %d retries" %
(url, retry_count))
log.error(err_msg)
raise IOError("Can't get .treeinfo file from the url {}".format(url))
if response:
# get the treeinfo contents
text = response.text
response.close()
self._tree_info.loads(text)
self._path = url
return True
return False
@staticmethod
def _download_treeinfo_file(session, url, file_name, headers, proxies, verify, cert):
try:
result = session.get("%s/%s" % (url, file_name), headers=headers,
proxies=proxies, verify=verify, cert=cert,
timeout=constants.NETWORK_CONNECTION_TIMEOUT)
# Server returned HTTP 4XX or 5XX codes
if 400 <= result.status_code < 600:
log.info("Server returned %i code", result.status_code)
return None, result.status_code
log.debug("Retrieved '%s' from %s", file_name, url)
except requests.exceptions.RequestException as e:
log.info("Error downloading '%s': %s", file_name, e)
return (None, None)
return result, result.status_code
def _clear(self):
"""Clear metadata repositories."""
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
def get_release_version(self):
"""Get release version from the repository.
:returns: Version as lowercase string.
:raises: ValueError if version is not present.
"""
version = self._tree_info.release.version
if not version:
raise ValueError("Can't read release version from the .treeinfo file! "
"Is the .treeinfo file corrupted?")
return version.lower()
def get_metadata_repos(self):
"""Get all repository metadata objects."""
if not self._meta_repos:
self._read_variants()
return self._meta_repos
def get_repo_metadata_by_name(self, name):
"""Get repository metadata object with given name.
:param name: Name of the variant to return.
:rtype name: VariantRepo object or None.
"""
for variant in self.get_metadata_repos():
if name == variant.name:
return variant
return None
def get_base_repo_metadata(self, additional_names=None):
"""Get repo metadata about base repository."""
repos = constants.DEFAULT_REPOS
if additional_names:
repos.extend(additional_names)
for repo_md in self.get_metadata_repos():
if repo_md.name in repos:
return repo_md
return None
def _read_variants(self):
for variant_name in self._tree_info.variants:
variant_object = self._tree_info.variants[variant_name]
self._meta_repos.append(RepoMetadata(variant_name, variant_object, self._path))
def _clear(self):
"""Clear metadata repositories."""
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
def __init__(self):
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
def test_f20(self):
ti = TreeInfo()
ti.load(os.path.join(DIR, "treeinfo/fedora-20-Fedora.x86_64"))
# product
self.assertEqual(ti.release.name, "Fedora")
self.assertEqual(ti.release.short, "Fedora")
self.assertEqual(ti.release.version, "20")
# tree
self.assertEqual(ti.tree.arch, "x86_64")
# XXX: converts float to int
self.assertEqual(ti.tree.build_timestamp, 1386857206)
# variants
self.assertEqual(len(ti.variants), 1)
# variant: Fedora
var = ti.variants["Fedora"]
self.assertEqual(var.id, "Fedora")
self.assertEqual(var.uid, "Fedora")
self.assertEqual(var.name, "Fedora")
self.assertEqual(var.type, "variant")
self.assertEqual(var.paths.packages, "Packages")
self.assertEqual(var.paths.repository, ".")
# stage2
self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img")
self.assertEqual(ti.stage2.instimage, None)
# images
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
"boot.iso": "images/boot.iso",
}
self.assertEqual(ti.images.images["x86_64"], expected_images)
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
}
self.assertEqual(ti.images.images["xen"], expected_images)
# checksums
expected_checksums = {
"images/boot.iso":
("sha256",
"376be7d4855ad6281cb139430606a782fd6189dcb01d7b61448e915802cc350f"
),
"images/efiboot.img":
("sha256",
"3bcba2a9b45366ab3f6d82dbe512421ddcb693f3bcbd9e30cc57aa0fa13c7835"
),
"images/macboot.img":
("sha256",
"698b3492534399e31fa1033f51f25b85d463e743f6a26322a9b117f9aac4fdf3"
),
"images/pxeboot/initrd.img":
("sha256",
"d0a81824e3425b6871ec4896a66e891aed35e291c50dfa30b08f6fc6ab04ca8b"
),
"images/pxeboot/upgrade.img":
("sha256",
"a274b8756290447950f1e48cff9d9d6fd1144579ad98ed9f91929ff9ac1bbfa9"
),
"images/pxeboot/vmlinuz":
("sha256",
"d3a2fbfcf08ac76dfb8135771380ec97ae3129b4e623891adb21bb1cd8ba59f6"
),
"repodata/repomd.xml":
("sha256",
"108b4102829c0839c7712832577fe7da24f0a9491f4dc25d4145efe6aced2ebf"
),
}
self.assertEqual(ti.checksums.checksums, expected_checksums)
self._test_identity(ti)
def test_f21_server(self):
ti = TreeInfo()
ti.load(os.path.join(DIR, "treeinfo/fedora-21-Alpha-Server.x86_64"))
# product
self.assertEqual(ti.release.name, "Fedora")
self.assertEqual(ti.release.short, "Fedora")
self.assertEqual(ti.release.version, "21")
# tree
self.assertEqual(ti.tree.arch, "x86_64")
# XXX: converts float to int
self.assertEqual(ti.tree.build_timestamp, 1410862874)
# variants
self.assertEqual(len(ti.variants), 1)
# variant: Server
var = ti.variants["Server"]
self.assertEqual(var.id, "Server")
self.assertEqual(var.uid, "Server")
self.assertEqual(var.name, "Server")
self.assertEqual(var.type, "variant")
self.assertEqual(var.paths.packages, "Packages")
self.assertEqual(var.paths.repository, ".")
# stage2
self.assertEqual(ti.stage2.mainimage, "LiveOS/squashfs.img")
self.assertEqual(ti.stage2.instimage, None)
# images
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
"boot.iso": "images/boot.iso",
}
self.assertEqual(ti.images.images["x86_64"], expected_images)
expected_images = {
"kernel": "images/pxeboot/vmlinuz",
"initrd": "images/pxeboot/initrd.img",
"upgrade": "images/pxeboot/upgrade.img",
}
self.assertEqual(ti.images.images["xen"], expected_images)
# checksums
expected_checksums = {
"images/boot.iso":
("sha256",
"e9c8b207d25c4af1a36c43a4cc3bf27dd2aefa9757eac63a8147e12e415f45f3"
),
"images/efiboot.img":
("sha256",
"0655a92312272a2ee929f579c7ac5086c0ab75b7453a0293a38502d69a8335d3"
),
"images/macboot.img":
("sha256",
"c25721c2e69ea82eedd78107baf69fc0d2900c5a23d0b689140edc2b372393f9"
),
"images/pxeboot/initrd.img":
("sha256",
"b6cffe60149c0396b3413bf3f73c3a1aa3090048302daeeeb8830911d903aa2f"
),
"images/pxeboot/upgrade.img":
("sha256",
"260ab3a63fb4ef1f0427a6b46d494c638760e0bb4e7e9f58295d0cf4e77886fc"
),
"images/pxeboot/vmlinuz":
("sha256",
"974cbed883608046bacfc7a4f232320750150c2821edcd4650fbbe709a6c1d3d"
),
"repodata/repomd.xml":
("sha256",
"1799b6652f287c73e1f1107ef5efe73a0714aaf34e8e9b81a08d3f741f5dc178"
),
}
self.assertEqual(ti.checksums.checksums, expected_checksums)
self._test_identity(ti)
def __init__(self):
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
class InstallTreeMetadata(object):
def __init__(self):
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
def load_file(self, root_path):
"""Loads installation tree metadata from root path.
:param root_path: Path to the installation root.
:type root_path: str
:returns: True if the metadata were loaded, False otherwise.
"""
self._clear()
self._path = root_path
if os.access(os.path.join(root_path, ".treeinfo"), os.R_OK):
self._tree_info.load(os.path.join(root_path, ".treeinfo"))
elif os.access(os.path.join(root_path, "treeinfo"), os.R_OK):
self._tree_info.load(os.path.join(root_path, "treeinfo"))
else:
return False
return True
def load_url(self, url, proxies, sslverify, headers):
"""Load URL link.
This can be also to local file.
Parameters here are passed to requests object so make them compatible with requests.
:param url: URL poiting to the installation tree.
:param proxies: Proxy used for the request.
:param sslverify: sslverify object which will be used in request.
:param headers: Additional headers of the request.
:returns: True if the install tree repo metadata was successfully loaded. False otherwise.
:raise: IOError is thrown in case of immediate failure.
"""
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
self._clear()
xdelay = util.xprogressive_delay()
response = None
ret_code = [None, None]
session = util.requests_session()
for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1):
if retry_count > 0:
time.sleep(next(xdelay))
# Downloading .treeinfo
log.info("Trying to download '.treeinfo'")
(response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo",
headers, proxies, sslverify)
if response:
break
# Downloading treeinfo
log.info("Trying to download 'treeinfo'")
(response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo",
headers, proxies, sslverify)
if response:
break
# The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo
# is on the server.
#
# Server returned HTTP 404 code -> no need to try again
if (ret_code[0] is not None and ret_code[0] == 404
and ret_code[1] is not None and ret_code[1] == 404):
response = None
log.error("Got HTTP 404 Error when downloading [.]treeinfo files")
break
if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES:
# retry
log.info("Retrying repo info download for %s, retrying (%d/%d)",
url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES)
else:
# run out of retries
err_msg = ("Repo info download for %s failed after %d retries" %
(url, retry_count))
log.error(err_msg)
raise IOError("Can't get .treeinfo file from the url {}".format(url))
if response:
# get the treeinfo contents
text = response.text
response.close()
self._tree_info.loads(text)
self._path = url
return True
return False
@staticmethod
def _download_treeinfo_file(session, url, file_name, headers, proxies, verify):
try:
result = session.get("%s/%s" % (url, file_name), headers=headers,
proxies=proxies, verify=verify)
# Server returned HTTP 4XX or 5XX codes
if 400 <= result.status_code < 600:
log.info("Server returned %i code", result.status_code)
return None, result.status_code
log.debug("Retrieved '%s' from %s", file_name, url)
except requests.exceptions.RequestException as e:
log.info("Error downloading '%s': %s", file_name, e)
return (None, None)
return result, result.status_code
def _clear(self):
"""Clear metadata repositories."""
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
def get_release_version(self):
"""Get release version from the repository.
:returns: Version as lowercase string.
:raises: ValueError if version is not present.
"""
version = self._tree_info.release.version
if not version:
raise ValueError("Can't read release version from the .treeinfo file! "
"Is the .treeinfo file corrupted?")
return version.lower()
def get_metadata_repos(self):
"""Get all repository metadata objects."""
if not self._meta_repos:
self._read_variants()
return self._meta_repos
def get_repo_metadata_by_name(self, name):
"""Get repository metadata object with given name.
:param name: Name of the variant to return.
:rtype name: VariantRepo object or None.
"""
for variant in self.get_metadata_repos():
if name == variant.name:
return variant
return None
def get_base_repo_metadata(self, additional_names=None):
"""Get repo metadata about base repository."""
repos = constants.DEFAULT_REPOS
if additional_names:
repos.extend(additional_names)
for repo_md in self.get_metadata_repos():
if repo_md.name in repos:
return repo_md
return None
def _read_variants(self):
for variant_name in self._tree_info.variants:
variant_object = self._tree_info.variants[variant_name]
self._meta_repos.append(RepoMetadata(variant_name, variant_object, self._path))
def _clear(self):
"""Clear metadata repositories."""
self._tree_info = TreeInfo()
self._meta_repos = []
self._path = ""
