def _download_metadata(self, downloader, url):
"""Download metadata from the given URL."""
# How many times should we try the download?
retry_max = self.MAX_TREEINFO_DOWNLOAD_RETRIES
# Don't retry to download files that returned HTTP 404 code.
not_found = set()
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
xdelay = xprogressive_delay()
for retry_count in range(0, retry_max):
# Delay if we are retrying the download.
if retry_count > 0:
log.info("Retrying download (%d/%d)", retry_count,
retry_max - 1)
time.sleep(next(xdelay))
# Download the metadata file.
for name in self.TREE_INFO_NAMES:
file_url = "{}/{}".format(url, name)
try:
with downloader(file_url) as r:
if r.status_code == 404:
not_found.add(name)
r.raise_for_status()
return r.text
except RequestException as e:
log.debug("Failed to download '%s': %s", name, e)
continue
if not_found == set(self.TREE_INFO_NAMES):
raise NoTreeInfoError("No treeinfo metadata found (404).")
raise NoTreeInfoError("Couldn't download treeinfo metadata.")
def load_url(self, url, proxies, sslverify, sslcert, headers):
"""Load URL link.
This can be also to local file.
Parameters here are passed to requests object so make them compatible with requests.
:param url: URL poiting to the installation tree.
:param proxies: Proxy used for the request.
:param sslverify: sslverify object which will be used in request.
:param headers: Additional headers of the request.
:returns: True if the install tree repo metadata was successfully loaded. False otherwise.
:raise: IOError is thrown in case of immediate failure.
"""
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
self._clear()
xdelay = util.xprogressive_delay()
response = None
ret_code = [None, None]
session = util.requests_session()
for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1):
if retry_count > 0:
time.sleep(next(xdelay))
# Downloading .treeinfo
log.info("Trying to download '.treeinfo'")
(response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo",
headers, proxies,
sslverify, sslcert)
if response:
break
# Downloading treeinfo
log.info("Trying to download 'treeinfo'")
(response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo",
headers, proxies,
sslverify, sslcert)
if response:
break
# The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo
# is on the server.
#
# Server returned HTTP 404 code -> no need to try again
if (ret_code[0] is not None and ret_code[0] == 404
and ret_code[1] is not None and ret_code[1] == 404):
response = None
log.error("Got HTTP 404 Error when downloading [.]treeinfo files")
break
if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES:
# retry
log.info("Retrying repo info download for %s, retrying (%d/%d)",
url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES)
else:
# run out of retries
err_msg = ("Repo info download for %s failed after %d retries" %
(url, retry_count))
log.error(err_msg)
raise IOError("Can't get .treeinfo file from the url {}".format(url))
if response:
# get the treeinfo contents
text = response.text
response.close()
self._tree_info.loads(text)
self._path = url
return True
return False
def load_url(self, url, proxies, sslverify, headers):
"""Load URL link.
This can be also to local file.
Parameters here are passed to requests object so make them compatible with requests.
:param url: URL poiting to the installation tree.
:param proxies: Proxy used for the request.
:param sslverify: sslverify object which will be used in request.
:param headers: Additional headers of the request.
:returns: True if the install tree repo metadata was successfully loaded. False otherwise.
:raise: IOError is thrown in case of immediate failure.
"""
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
self._clear()
xdelay = util.xprogressive_delay()
response = None
ret_code = [None, None]
session = util.requests_session()
for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1):
if retry_count > 0:
time.sleep(next(xdelay))
# Downloading .treeinfo
log.info("Trying to download '.treeinfo'")
(response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo",
headers, proxies, sslverify)
if response:
break
# Downloading treeinfo
log.info("Trying to download 'treeinfo'")
(response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo",
headers, proxies, sslverify)
if response:
break
# The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo
# is on the server.
#
# Server returned HTTP 404 code -> no need to try again
if (ret_code[0] is not None and ret_code[0] == 404
and ret_code[1] is not None and ret_code[1] == 404):
response = None
log.error("Got HTTP 404 Error when downloading [.]treeinfo files")
break
if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES:
# retry
log.info("Retrying repo info download for %s, retrying (%d/%d)",
url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES)
else:
# run out of retries
err_msg = ("Repo info download for %s failed after %d retries" %
(url, retry_count))
log.error(err_msg)
raise IOError("Can't get .treeinfo file from the url {}".format(url))
if response:
# get the treeinfo contents
text = response.text
response.close()
self._tree_info.loads(text)
self._path = url
return True
return False