def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url,
stream=True,
verify=self.verify,
auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" %
(url, exception_message_safe(exc)))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
try:
data = self._download_data(response, file_path)
duration = time.time() - t1
log_download(url, duration)
return data
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def download(self, url, file_path=None, auth=None, retry=1, retry_wait=0):
if file_path and os.path.exists(file_path):
# Should not happen, better to raise, probably we had to remove the dest folder before
raise ConanException(
"Error, the file to download already exists: '%s'" % file_path)
t1 = time.time()
ret = bytearray()
response = call_with_retry(self.output, retry, retry_wait,
self._download_file, url, auth)
if not response.ok: # Do not retry if not found or whatever controlled error
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
try:
total_length = response.headers.get('content-length')
if total_length is None: # no content length header
if not file_path:
ret += response.content
else:
total_length = len(response.content)
progress = human_readable_progress(total_length,
total_length)
print_progress(self.output, 50, progress)
save(file_path, response.content, append=True)
else:
dl = 0
total_length = int(total_length)
last_progress = None
chunk_size = 1024 if not file_path else 1024 * 100
for data in response.iter_content(chunk_size=chunk_size):
dl += len(data)
if not file_path:
ret.extend(data)
else:
save(file_path, data, append=True)
units = progress_units(dl, total_length)
progress = human_readable_progress(dl, total_length)
if last_progress != units: # Avoid screen refresh if nothing has change
if self.output:
print_progress(self.output, units, progress)
last_progress = units
duration = time.time() - t1
log_download(url, duration)
if not file_path:
return bytes(ret)
else:
return
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def download(self,
url,
file_path=None,
auth=None,
retry=1,
retry_wait=0,
overwrite=False,
headers=None):
if file_path and not os.path.isabs(file_path):
file_path = os.path.abspath(file_path)
if file_path and os.path.exists(file_path):
if overwrite:
if self.output:
self.output.warn("file '%s' already exists, overwriting" %
file_path)
else:
# Should not happen, better to raise, probably we had to remove
# the dest folder before
raise ConanException(
"Error, the file to download already exists: '%s'" %
file_path)
t1 = time.time()
ret = bytearray()
response = call_with_retry(self.output, retry, retry_wait,
self._download_file, url, auth, headers)
if not response.ok: # Do not retry if not found or whatever controlled error
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
try:
total_length = response.headers.get('content-length')
if total_length is None: # no content length header
if not file_path:
ret += response.content
else:
total_length = len(response.content)
progress = human_readable_progress(total_length,
total_length)
print_progress(self.output, 50, progress)
save(file_path, response.content, append=True)
else:
total_length = int(total_length)
encoding = response.headers.get('content-encoding')
gzip = (encoding == "gzip")
# chunked can be a problem: https://www.greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
# It will not send content-length or should be ignored
def download_chunks(file_handler=None, ret_buffer=None):
"""Write to a buffer or to a file handler"""
chunk_size = 1024 if not file_path else 1024 * 100
download_size = 0
last_progress = None
for data in response.iter_content(chunk_size=chunk_size):
download_size += len(data)
if ret_buffer is not None:
ret_buffer.extend(data)
if file_handler is not None:
file_handler.write(to_file_bytes(data))
units = progress_units(download_size, total_length)
progress = human_readable_progress(
download_size, total_length)
if last_progress != units: # Avoid screen refresh if nothing has change
if self.output:
print_progress(self.output, units, progress)
last_progress = units
return download_size
if file_path:
mkdir(os.path.dirname(file_path))
with open(file_path, 'wb') as handle:
dl_size = download_chunks(file_handler=handle)
else:
dl_size = download_chunks(ret_buffer=ret)
if dl_size != total_length and not gzip:
raise ConanException("Transfer interrupted before "
"complete: %s < %s" %
(dl_size, total_length))
duration = time.time() - t1
log_download(url, duration)
if not file_path:
return bytes(ret)
else:
return
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def _download_file(self, url, auth, headers, file_path, try_resume=False):
t1 = time.time()
if try_resume and file_path and os.path.exists(file_path):
range_start = os.path.getsize(file_path)
headers = headers.copy() if headers else {}
headers["range"] = "bytes={}-".format(range_start)
else:
range_start = 0
try:
response = self._requester.get(url,
stream=True,
verify=self._verify_ssl,
auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" %
(url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth is None or (hasattr(auth, "token")
and auth.token is None):
# TODO: This is a bit weird, why this conversion? Need to investigate
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
def read_response(size):
for chunk in response.iter_content(size):
yield chunk
def write_chunks(chunks, path):
ret = None
downloaded_size = range_start
if path:
mkdir(os.path.dirname(path))
mode = "ab" if range_start else "wb"
with open(path, mode) as file_handler:
for chunk in chunks:
assert ((six.PY3 and isinstance(chunk, bytes))
or (six.PY2 and isinstance(chunk, str)))
file_handler.write(chunk)
downloaded_size += len(chunk)
else:
ret_data = bytearray()
for chunk in chunks:
ret_data.extend(chunk)
downloaded_size += len(chunk)
ret = bytes(ret_data)
return ret, downloaded_size
def get_total_length():
if range_start:
content_range = response.headers.get("Content-Range", "")
match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range)
if not match or range_start != int(match.group(1)):
raise ConanException("Error in resumed download from %s\n"
"Incorrect Content-Range header %s" %
(url, content_range))
return int(match.group(3))
else:
total_size = response.headers.get('Content-Length') or len(
response.content)
return int(total_size)
try:
logger.debug("DOWNLOAD: %s" % url)
total_length = get_total_length()
action = "Downloading" if range_start == 0 else "Continuing download of"
description = "{} {}".format(
action, os.path.basename(file_path)) if file_path else None
progress = progress_bar.Progress(total_length, self._output,
description)
progress.initial_value(range_start)
chunk_size = 1024 if not file_path else 1024 * 100
written_chunks, total_downloaded_size = write_chunks(
progress.update(read_response(chunk_size)), file_path)
gzip = (response.headers.get("content-encoding") == "gzip")
response.close()
# it seems that if gzip we don't know the size, cannot resume and shouldn't raise
if total_downloaded_size != total_length and not gzip:
if (file_path
and total_length > total_downloaded_size > range_start
and response.headers.get("Accept-Ranges") == "bytes"):
written_chunks = self._download_file(url,
auth,
headers,
file_path,
try_resume=True)
else:
raise ConanException(
"Transfer interrupted before complete: %s < %s" %
(total_downloaded_size, total_length))
duration = time.time() - t1
log_download(url, duration)
return written_chunks
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url, stream=True, verify=self.verify, auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" % (url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth is None or (hasattr(auth, "token") and auth.token is None):
# TODO: This is a bit weird, why this conversion? Need to investigate
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" % (response.status_code, url))
def read_response(size):
for chunk in response.iter_content(size):
yield chunk
def write_chunks(chunks, path):
ret = None
downloaded_size = 0
if path:
mkdir(os.path.dirname(path))
with open(path, 'wb') as file_handler:
for chunk in chunks:
assert ((six.PY3 and isinstance(chunk, bytes)) or
(six.PY2 and isinstance(chunk, str)))
file_handler.write(chunk)
downloaded_size += len(chunk)
else:
ret_data = bytearray()
for chunk in chunks:
ret_data.extend(chunk)
downloaded_size += len(chunk)
ret = bytes(ret_data)
return ret, downloaded_size
try:
logger.debug("DOWNLOAD: %s" % url)
total_length = response.headers.get('content-length') or len(response.content)
total_length = int(total_length)
description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None
progress = progress_bar.Progress(total_length, self.output, description, print_dot=False)
chunk_size = 1024 if not file_path else 1024 * 100
encoding = response.headers.get('content-encoding')
gzip = (encoding == "gzip")
written_chunks, total_downloaded_size = write_chunks(
progress.update(read_response(chunk_size), chunk_size),
file_path
)
response.close()
if total_downloaded_size != total_length and not gzip:
raise ConanException("Transfer interrupted before "
"complete: %s < %s" % (total_downloaded_size, total_length))
duration = time.time() - t1
log_download(url, duration)
return written_chunks
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError("Download failed, check server, possibly try again\n%s"
% str(e))
