def _download_multipass(dl_dir: str, echoer) -> str:
"""Creates temporary Downloads installer to temp directory."""
dl_url = _fetch_installer_url()
dl_basename = os.path.basename(dl_url)
dl_path = os.path.join(dl_dir, dl_basename)
echoer.info("Downloading Multipass installer...\n{} -> {}".format(
dl_url, dl_path))
try:
request = requests.get(dl_url, stream=True, allow_redirects=True)
request.raise_for_status()
download_requests_stream(request, dl_path)
except requests.RequestException as e:
raise ProviderMultipassDownloadFailed(_requests_exception_hint(e))
digest = calculate_sha3_384(dl_path)
if digest != _MULTIPASS_DL_SHA3_384:
raise ProviderMultipassDownloadFailed(
"download failed verification (expected={} but found={})".format(
_MULTIPASS_DL_SHA3_384, digest))
echoer.info("Verified installer successfully...")
return dl_path
def download(self):
request = requests.get(self.source, stream=True, allow_redirects=True)
request.raise_for_status()
self.file = os.path.join(
self.source_dir, os.path.basename(self.source))
download_requests_stream(request, self.file)
def download(self):
# First check if we already have the source file cached.
file_cache = FileCache()
if self.source_checksum:
algorithm, hash = split_checksum(self.source_checksum)
cache_file = file_cache.get(algorithm=algorithm, hash=hash)
if cache_file:
self.file = os.path.join(self.source_dir, os.path.basename(cache_file))
# We make this copy as the provisioning logic can delete
# this file and we don't want that.
shutil.copy2(cache_file, self.file)
return self.file
# If not we download and store
self.file = os.path.join(self.source_dir, os.path.basename(self.source))
if snapcraft.internal.common.get_url_scheme(self.source) == "ftp":
download_urllib_source(self.source, self.file)
else:
request = requests.get(self.source, stream=True, allow_redirects=True)
request.raise_for_status()
download_requests_stream(request, self.file)
# We verify the file if source_checksum is defined
# and we cache the file for future reuse.
if self.source_checksum:
algorithm, digest = verify_checksum(self.source_checksum, self.file)
file_cache.cache(filename=self.file, algorithm=algorithm, hash=hash)
return self.file
def download(self, filepath: str = None) -> str:
if filepath is None:
self.file = os.path.join(self.source_dir, os.path.basename(self.source))
else:
self.file = filepath
# First check if we already have the source file cached.
file_cache = FileCache()
if self.source_checksum:
algorithm, hash = split_checksum(self.source_checksum)
cache_file = file_cache.get(algorithm=algorithm, hash=hash)
if cache_file:
# We make this copy as the provisioning logic can delete
# this file and we don't want that.
shutil.copy2(cache_file, self.file)
return self.file
# If not we download and store
if snapcraft.internal.common.get_url_scheme(self.source) == "ftp":
download_urllib_source(self.source, self.file)
else:
request = requests.get(self.source, stream=True, allow_redirects=True)
request.raise_for_status()
download_requests_stream(request, self.file)
# We verify the file if source_checksum is defined
# and we cache the file for future reuse.
if self.source_checksum:
algorithm, digest = verify_checksum(self.source_checksum, self.file)
file_cache.cache(filename=self.file, algorithm=algorithm, hash=hash)
return self.file
def download(self):
request = requests.get(self.source, stream=True, allow_redirects=True)
request.raise_for_status()
self.file = os.path.join(self.source_dir,
os.path.basename(self.source))
download_requests_stream(request, self.file)
def _download_snap(self, name, channel, arch, download_path,
download_url, expected_sha512):
if self._is_downloaded(download_path, expected_sha512):
logger.info('Already downloaded {} at {}'.format(
name, download_path))
return
logger.info('Downloading {}'.format(name, download_path))
# HttpAdapter cannot help here as this is a stream.
# LP: #1617765
not_downloaded = True
retry_count = 5
while not_downloaded and retry_count:
request = self.cpi.get(download_url, stream=True)
request.raise_for_status()
try:
download_requests_stream(request, download_path)
not_downloaded = False
except requests.exceptions.ChunkedEncodingError as e:
retry_count -= 1
if not retry_count:
raise e
sleep(1)
if self._is_downloaded(download_path, expected_sha512):
logger.info('Successfully downloaded {} at {}'.format(
name, download_path))
else:
raise errors.SHAMismatchError(download_path, expected_sha512)
def download(self):
self.file = os.path.join(
self.source_dir, os.path.basename(self.source))
if common.get_url_scheme(self.source) == 'ftp':
download_urllib_source(self.source, self.file)
else:
request = requests.get(
self.source, stream=True, allow_redirects=True)
request.raise_for_status()
download_requests_stream(request, self.file)
def download(self):
self.file = os.path.join(
self.source_dir, os.path.basename(self.source))
if snapcraft.internal.common.get_url_scheme(self.source) == 'ftp':
download_urllib_source(self.source, self.file)
else:
request = requests.get(
self.source, stream=True, allow_redirects=True)
request.raise_for_status()
download_requests_stream(request, self.file)
def _download_snap(self, name, channel, arch, download_path, download_url,
expected_sha512):
if self._is_downloaded(download_path, expected_sha512):
logger.info('Already downloaded {} at {}'.format(
name, download_path))
return
logger.info('Downloading {}'.format(name, download_path))
# we only resume when redirected to our CDN since we use internap's
# special sauce.
resume_possible = False
total_read = 0
probe_url = requests.head(download_url)
if (probe_url.is_redirect
and 'internap' in probe_url.headers['Location']):
download_url = probe_url.headers['Location']
resume_possible = True
# HttpAdapter cannot help here as this is a stream.
# LP: #1617765
not_downloaded = True
retry_count = 5
while not_downloaded and retry_count:
headers = {}
if resume_possible and os.path.exists(download_path):
total_read = os.path.getsize(download_path)
headers['Range'] = 'bytes={}-'.format(total_read)
request = self.cpi.get(download_url, headers=headers, stream=True)
request.raise_for_status()
redirections = [h.headers['Location'] for h in request.history]
if redirections:
logger.debug('Redirections for {!r}: {}'.format(
download_url, ', '.join(redirections)))
try:
download_requests_stream(request,
download_path,
total_read=total_read)
not_downloaded = False
except requests.exceptions.ChunkedEncodingError as e:
logger.debug('Error while downloading: {!r}. '
'Retries left to download: {!r}.'.format(
e, retry_count))
retry_count -= 1
if not retry_count:
raise e
sleep(1)
if self._is_downloaded(download_path, expected_sha512):
logger.info('Successfully downloaded {} at {}'.format(
name, download_path))
else:
raise errors.SHAMismatchError(download_path, expected_sha512)
def _download_snap(
self, name, channel, arch, download_path, download_url, expected_sha512
):
if self._is_downloaded(download_path, expected_sha512):
logger.info("Already downloaded {} at {}".format(name, download_path))
return
logger.info("Downloading {}".format(name))
# we only resume when redirected to our CDN since we use internap's
# special sauce.
resume_possible = False
total_read = 0
probe_url = requests.head(download_url)
if probe_url.is_redirect and "internap" in probe_url.headers["Location"]:
download_url = probe_url.headers["Location"]
resume_possible = True
# HttpAdapter cannot help here as this is a stream.
# LP: #1617765
not_downloaded = True
retry_count = 5
while not_downloaded and retry_count:
headers = {}
if resume_possible and os.path.exists(download_path):
total_read = os.path.getsize(download_path)
headers["Range"] = "bytes={}-".format(total_read)
request = self.cpi.get(download_url, headers=headers, stream=True)
request.raise_for_status()
redirections = [h.headers["Location"] for h in request.history]
if redirections:
logger.debug(
"Redirections for {!r}: {}".format(
download_url, ", ".join(redirections)
)
)
try:
download_requests_stream(request, download_path, total_read=total_read)
not_downloaded = False
except requests.exceptions.ChunkedEncodingError as e:
logger.debug(
"Error while downloading: {!r}. "
"Retries left to download: {!r}.".format(e, retry_count)
)
retry_count -= 1
if not retry_count:
raise e
sleep(1)
if self._is_downloaded(download_path, expected_sha512):
logger.info("Successfully downloaded {} at {}".format(name, download_path))
else:
raise errors.SHAMismatchError(download_path, expected_sha512)
def execute(self):
headers = self._load_headers()
self._request = requests.get(self._parts_uri, stream=True,
headers=headers)
if self._request.status_code == 304:
logger.info('The parts cache is already up to date.')
return
self._request.raise_for_status()
download_requests_stream(self._request, self.parts_yaml,
'Downloading parts list')
self._save_headers()
def execute(self):
headers = self._load_headers()
self._request = requests.get(self._parts_uri, stream=True,
headers=headers)
if self._request.status_code == 304:
logger.info('The parts cache is already up to date.')
return
self._request.raise_for_status()
download_requests_stream(self._request, self.parts_yaml,
'Downloading parts list')
self._save_headers()
def _download_snap(self, name, channel, arch, download_path,
download_url, expected_sha512):
if self._is_downloaded(download_path, expected_sha512):
logger.info('Already downloaded {} at {}'.format(
name, download_path))
return
logger.info('Downloading {}'.format(name, download_path))
request = self.cpi.get(download_url, stream=True)
request.raise_for_status()
download_requests_stream(request, download_path)
if self._is_downloaded(download_path, expected_sha512):
logger.info('Successfully downloaded {} at {}'.format(
name, download_path))
else:
raise errors.SHAMismatchError(download_path, expected_sha512)
def execute(self):
headers = self._load_headers()
try:
self._request = requests.get(self._parts_uri, stream=True, headers=headers)
except requests.exceptions.RequestException as e:
raise errors.RemotePartsUpdateConnectionError(e) from e
if self._request.status_code == 304:
logger.info("The parts cache is already up to date.")
return
self._request.raise_for_status()
download_requests_stream(
self._request, self.parts_yaml, "Downloading parts list"
)
self._save_headers()
def _download_snap(self, download_details, download_path):
# we only resume when redirected to our CDN since we use internap's
# special sauce.
total_read = 0
probe_url = requests.head(download_details.url)
if probe_url.is_redirect and "internap" in probe_url.headers[
"Location"]:
download_url = probe_url.headers["Location"]
resume_possible = True
else:
download_url = download_details.url
resume_possible = False
# HttpAdapter cannot help here as this is a stream.
# LP: #1617765
not_downloaded = True
retry_count = 5
while not_downloaded and retry_count:
headers = {}
if resume_possible and os.path.exists(download_path):
total_read = os.path.getsize(download_path)
headers["Range"] = "bytes={}-".format(total_read)
request = self.client.request("GET",
download_url,
headers=headers,
stream=True)
request.raise_for_status()
redirections = [h.headers["Location"] for h in request.history]
if redirections:
logger.debug("Redirections for {!r}: {}".format(
download_url, ", ".join(redirections)))
try:
download_requests_stream(request,
download_path,
total_read=total_read)
not_downloaded = False
except requests.exceptions.ChunkedEncodingError as e:
logger.debug("Error while downloading: {!r}. "
"Retries left to download: {!r}.".format(
e, retry_count))
retry_count -= 1
if not retry_count:
raise e
sleep(1)
def _download_and_cache(self) -> str:
request = requests.get(self.url, stream=True, allow_redirects=True)
if not request.ok:
raise errors.BuildImageRequestError(
base=self.base, status_code=request.status_code
)
# First create the prefix as tempfile.TemporaryDirectory does not do that for you
os.makedirs(self._image_cache.file_cache, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix=self._image_cache.file_cache
) as tmp_dir:
download_file = os.path.join(tmp_dir, "{}-vm".format(self.base))
download_requests_stream(request, download_file)
calculated_digest = calculate_hash(download_file, algorithm=self.algorithm)
if self.checksum != calculated_digest:
raise errors.BuildImageChecksumError(
expected=self.checksum,
calculated=calculated_digest,
algorithm=self.algorithm,
)
return self._image_cache.cache(
filename=download_file, algorithm=self.algorithm, hash=self.checksum
)
def test_download_request_stream(self):
request = requests.get(self.source, stream=True, allow_redirects=True)
indicators.download_requests_stream(request, self.dest_file)
self.assertTrue(os.path.exists(self.dest_file))
def test_download_request_stream(self):
request = requests.get(self.source, stream=True, allow_redirects=True)
indicators.download_requests_stream(request, self.dest_file)
self.assertTrue(os.path.exists(self.dest_file))
