def load_data(self, data: RepoConfigurationData):
"""Loads installation tree metadata from the given data.
param data: the repo configuration data
:raise: NoTreeInfoError if there is no .treeinfo file
"""
self._reset()
if data.type != URL_TYPE_BASEURL:
raise NoTreeInfoError("Unsupported type of URL ({}).".format(data.type))
if not data.url:
raise NoTreeInfoError("No URL specified.")
# Download the metadata.
log.debug("Load treeinfo metadata for '%s'.", data.url)
with requests_session() as session:
downloader = self._get_downloader(session, data)
content = self._download_metadata(downloader, data.url)
# Process the metadata.
self._load_tree_info(
root_url=data.url,
file_content=content
)
def run(self):
"""Run the task.
If the image is local, we return its local location. Otherwise,
the image is downloaded at the specified download location and
we return that one.
:return: a path to the image
"""
log.info("Downloading the image...")
if self._url.startswith("file://"):
log.info("Nothing to download.")
return self._url.removeprefix("file://")
with requests_session() as session:
try:
# Send a GET request to the image URL.
response = self._send_request(session)
# Download the image to a file.
self._download_image(response)
except requests.exceptions.RequestException as e:
raise PayloadInstallationError(
"Error while downloading the image: {}".format(e)
) from e
return self._download_path
def download_escrow_certificate(url):
"""Download the escrow certificate.
:param url: an URL of the certificate
:return: a content of the certificate
"""
# Do we need a network connection?
if not url.startswith("/") and not url.startswith("file:"):
network_proxy = NETWORK.get_proxy()
if not network_proxy.Connected:
raise KickstartError(_("Escrow certificate %s requires the network.") % url)
# Download the certificate.
log.info("Downloading an escrow certificate from: %s", url)
try:
request = util.requests_session().get(url, verify=True)
except requests.exceptions.SSLError as e:
raise KickstartError(_("SSL error while downloading the escrow certificate:\n\n%s") % e)
except requests.exceptions.RequestException as e:
raise KickstartError(_("The following error was encountered while downloading the "
"escrow certificate:\n\n%s") % e)
try:
certificate = request.content
finally:
request.close()
return certificate
def _reverse_geocode_nominatim(self, coordinates):
"""Reverse geocoding using the Nominatim API.
Reverse geocoding tries to convert geographic coordinates
to an accurate address.
:param coordinates: input coordinates
:type coordinates: Coordinates
:return: an address or None if no address was found
:rtype: GeocodingResult or None
"""
url = "%s&addressdetails=1&lat=%f&lon=%f" % (
self.NOMINATIM_API_URL,
coordinates.latitude,
coordinates.longitude)
try:
reply = requests_session().get(url, timeout=constants.NETWORK_CONNECTION_TIMEOUT, verify=True)
if reply.status_code == requests.codes.ok:
reply_dict = reply.json()
territory_code = reply_dict['address']['country_code'].upper()
return GeocodingResult(coordinates=coordinates,
territory_code=territory_code)
else:
log.error("Geoloc: Nominatim reverse geocoding failed with status code: %s", reply.status_code)
return None
except requests.exceptions.RequestException as e:
log.debug("Geoloc: RequestException during Nominatim reverse geocoding:\n%s", e)
except ValueError as e:
log.debug("Geoloc: Unable to decode Nominatim reverse geocoding JSON:\n%s", e)
def _reverse_geocode_nominatim(self, coordinates):
"""Reverse geocoding using the Nominatim API.
Reverse geocoding tries to convert geographic coordinates
to an accurate address.
:param coordinates: input coordinates
:type coordinates: Coordinates
:return: an address or None if no address was found
:rtype: GeocodingResult or None
"""
url = "%s&addressdetails=1&lat=%f&lon=%f" % (
self.NOMINATIM_API_URL,
coordinates.latitude,
coordinates.longitude)
try:
reply = requests_session().get(url, timeout=constants.NETWORK_CONNECTION_TIMEOUT, verify=True)
if reply.status_code == requests.codes.ok:
reply_dict = reply.json()
territory_code = reply_dict['address']['country_code'].upper()
return GeocodingResult(coordinates=coordinates,
territory_code=territory_code)
else:
log.error("Geoloc: Nominatim reverse geocoding failed with status code: %s", reply.status_code)
return None
except requests.exceptions.RequestException as e:
log.debug("Geoloc: RequestException during Nominatim reverse geocoding:\n%s", e)
except ValueError as e:
log.debug("Geoloc: Unable to decode Nominatim reverse geocoding JSON:\n%s", e)
def download_escrow_certificate(url):
"""Download the escrow certificate.
:param url: an URL of the certificate
:return: a content of the certificate
"""
# Do we need a network connection?
if not url.startswith("/") and not url.startswith("file:"):
network_proxy = NETWORK.get_proxy()
if not network_proxy.Connected:
raise KickstartError(_("Escrow certificate %s requires the network.") % url)
# Download the certificate.
log.info("Downloading an escrow certificate from: %s", url)
try:
request = util.requests_session().get(url, verify=True)
except requests.exceptions.SSLError as e:
raise KickstartError(_("SSL error while downloading the escrow certificate:\n\n%s") % e)
except requests.exceptions.RequestException as e:
raise KickstartError(_("The following error was encountered while downloading the "
"escrow certificate:\n\n%s") % e)
try:
certificate = request.content
finally:
request.close()
return certificate
def __init__(self, data):
"""Initialize Payload class
:param data: This param is a kickstart.AnacondaKSHandler class.
"""
self.data = data
# A list of verbose error strings from the subclass
self.verbose_errors = []
self._session = util.requests_session()
# A DBus proxy of the payload.
self._payload_proxy = None
def __init__(self, data):
"""Initialize Payload class
:param data: This param is a kickstart.AnacondaKSHandler class.
"""
self.data = data
# A list of verbose error strings from the subclass
self.verbose_errors = []
self._session = util.requests_session()
# Additional packages required by installer based on used features
self.requirements = PayloadRequirements()
def __init__(self, data):
"""Initialize Payload class
:param data: This param is a kickstart.AnacondaKSHandler class.
"""
self.data = data
self.storage = None
self.tx_id = None
self._install_tree_metadata = None
self._first_payload_reset = True
# A list of verbose error strings from the subclass
self.verbose_errors = []
self._session = util.requests_session()
# Additional packages required by installer based on used features
self.requirements = PayloadRequirements()
def run(self):
"""Run installation source image check.
:return: a tuple with the required space
:rtype: an instance of SetupImageResult
:raise: SourceSetupError on failure
"""
with requests_session() as session:
try:
# Send a HEAD request to the image URL.
response = self._send_request(session)
# Calculate the required space for the image.
size = self._get_required_space(response)
except RequestException as e:
msg = "Error while handling a request: {}".format(e)
raise SourceSetupError(msg) from e
return SetupImageResult(required_space=size)
def _download_repoMD(self):
proxies = {}
repomd = ""
headers = {"user-agent": USER_AGENT}
if self._proxy_url is not None:
try:
proxy = ProxyString(self._proxy_url)
proxies = {"http": proxy.url, "https": proxy.url}
except ProxyStringError as e:
log.info(
"Failed to parse proxy for test if repo available %s: %s",
self._proxy_url, e)
session = util.requests_session()
# Test all urls for this repo. If any of these is working it is enough.
for url in self._urls:
try:
result = session.get(
"%s/repodata/repomd.xml" % url,
headers=headers,
proxies=proxies,
verify=self._ssl_verify,
timeout=constants.NETWORK_CONNECTION_TIMEOUT)
if result.ok:
repomd = result.text
result.close()
break
else:
log.debug(
"Server returned %i code when downloading repomd",
result.status_code)
result.close()
continue
except RequestException as e:
log.debug(
"Can't download new repomd.xml from %s with proxy: %s. Error: %s",
url, proxies, e)
return repomd
def __init__(self, data, storage):
"""Initialize Payload class
:param data: This param is a kickstart.AnacondaKSHandler class.
:param storage: an instance of Blivet's storage model
"""
self.data = data
self.storage = storage
self.tx_id = None
self._install_tree_metadata = None
self._first_payload_reset = True
# A list of verbose error strings from the subclass
self.verbose_errors = []
self._session = util.requests_session()
# Additional packages required by installer based on used features
self.requirements = PayloadRequirements()
def _download_repoMD(self, method):
proxies = {}
repomd = ""
headers = {"user-agent": USER_AGENT}
sslverify = not flags.noverifyssl
if hasattr(method, "proxy"):
proxy_url = method.proxy
try:
proxy = ProxyString(proxy_url)
proxies = {"http": proxy.url, "https": proxy.url}
except ProxyStringError as e:
log.info(
"Failed to parse proxy for test if repo available %s: %s",
proxy_url, e)
session = util.requests_session()
# Test all urls for this repo. If any of these is working it is enough.
for url in self._urls:
try:
result = session.get("%s/repodata/repomd.xml" % url,
headers=headers,
proxies=proxies,
verify=sslverify)
if result.ok:
repomd = result.text
break
else:
log.debug(
"Server returned %i code when downloading repomd",
result.status_code)
continue
except RequestException as e:
log.debug(
"Can't download new repomd.xml from %s with proxy: %s. Error: %s",
url, proxies, e)
return repomd
def __init__(self):
self._result = LocationResult()
self._result_lock = threading.Lock()
self._session = requests_session()
self._refresh_condition = threading.Condition()
self._refresh_in_progress = False
def load_url(self, url, proxies, sslverify, headers):
"""Load URL link.
This can be also to local file.
Parameters here are passed to requests object so make them compatible with requests.
:param url: URL poiting to the installation tree.
:param proxies: Proxy used for the request.
:param sslverify: sslverify object which will be used in request.
:param headers: Additional headers of the request.
:returns: True if the install tree repo metadata was successfully loaded. False otherwise.
:raise: IOError is thrown in case of immediate failure.
"""
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
self._clear()
xdelay = util.xprogressive_delay()
response = None
ret_code = [None, None]
session = util.requests_session()
for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1):
if retry_count > 0:
time.sleep(next(xdelay))
# Downloading .treeinfo
log.info("Trying to download '.treeinfo'")
(response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo",
headers, proxies, sslverify)
if response:
break
# Downloading treeinfo
log.info("Trying to download 'treeinfo'")
(response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo",
headers, proxies, sslverify)
if response:
break
# The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo
# is on the server.
#
# Server returned HTTP 404 code -> no need to try again
if (ret_code[0] is not None and ret_code[0] == 404
and ret_code[1] is not None and ret_code[1] == 404):
response = None
log.error("Got HTTP 404 Error when downloading [.]treeinfo files")
break
if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES:
# retry
log.info("Retrying repo info download for %s, retrying (%d/%d)",
url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES)
else:
# run out of retries
err_msg = ("Repo info download for %s failed after %d retries" %
(url, retry_count))
log.error(err_msg)
raise IOError("Can't get .treeinfo file from the url {}".format(url))
if response:
# get the treeinfo contents
text = response.text
response.close()
self._tree_info.loads(text)
self._path = url
return True
return False
def __init__(self):
self._result = LocationResult()
self._result_lock = threading.Lock()
self._session = requests_session()
self._refresh_condition = threading.Condition()
self._refresh_in_progress = False
def load_url(self, url, proxies, sslverify, sslcert, headers):
"""Load URL link.
This can be also to local file.
Parameters here are passed to requests object so make them compatible with requests.
:param url: URL poiting to the installation tree.
:param proxies: Proxy used for the request.
:param sslverify: sslverify object which will be used in request.
:param headers: Additional headers of the request.
:returns: True if the install tree repo metadata was successfully loaded. False otherwise.
:raise: IOError is thrown in case of immediate failure.
"""
# Retry treeinfo downloads with a progressively longer pause,
# so NetworkManager have a chance setup a network and we have
# full connectivity before trying to download things. (#1292613)
self._clear()
xdelay = util.xprogressive_delay()
response = None
ret_code = [None, None]
session = util.requests_session()
for retry_count in range(0, MAX_TREEINFO_DOWNLOAD_RETRIES + 1):
if retry_count > 0:
time.sleep(next(xdelay))
# Downloading .treeinfo
log.info("Trying to download '.treeinfo'")
(response, ret_code[0]) = self._download_treeinfo_file(session, url, ".treeinfo",
headers, proxies,
sslverify, sslcert)
if response:
break
# Downloading treeinfo
log.info("Trying to download 'treeinfo'")
(response, ret_code[1]) = self._download_treeinfo_file(session, url, "treeinfo",
headers, proxies,
sslverify, sslcert)
if response:
break
# The [.]treeinfo wasn't downloaded. Try it again if [.]treeinfo
# is on the server.
#
# Server returned HTTP 404 code -> no need to try again
if (ret_code[0] is not None and ret_code[0] == 404
and ret_code[1] is not None and ret_code[1] == 404):
response = None
log.error("Got HTTP 404 Error when downloading [.]treeinfo files")
break
if retry_count < MAX_TREEINFO_DOWNLOAD_RETRIES:
# retry
log.info("Retrying repo info download for %s, retrying (%d/%d)",
url, retry_count + 1, MAX_TREEINFO_DOWNLOAD_RETRIES)
else:
# run out of retries
err_msg = ("Repo info download for %s failed after %d retries" %
(url, retry_count))
log.error(err_msg)
raise IOError("Can't get .treeinfo file from the url {}".format(url))
if response:
# get the treeinfo contents
text = response.text
response.close()
self._tree_info.loads(text)
self._path = url
return True
return False
def requests_session(self):
"""Get requests session."""
# FIXME: share in Payload module?
if not self._requests_session:
self._requests_session = requests_session()
return self._requests_session
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._min_size = 0
self._proxies = {}
self._session = util.requests_session()
self.image_path = conf.target.system_root + "/disk.img"