def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exception_message_safe(exc))) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: data = self._download_data(response, file_path) duration = time.time() - t1 log_download(url, duration) return data except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def download(self, url, file_path=None, auth=None, retry=1, retry_wait=0): if file_path and os.path.exists(file_path): # Should not happen, better to raise, probably we had to remove the dest folder before raise ConanException( "Error, the file to download already exists: '%s'" % file_path) t1 = time.time() ret = bytearray() response = call_with_retry(self.output, retry, retry_wait, self._download_file, url, auth) if not response.ok: # Do not retry if not found or whatever controlled error raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: total_length = response.headers.get('content-length') if total_length is None: # no content length header if not file_path: ret += response.content else: total_length = len(response.content) progress = human_readable_progress(total_length, total_length) print_progress(self.output, 50, progress) save(file_path, response.content, append=True) else: dl = 0 total_length = int(total_length) last_progress = None chunk_size = 1024 if not file_path else 1024 * 100 for data in response.iter_content(chunk_size=chunk_size): dl += len(data) if not file_path: ret.extend(data) else: save(file_path, data, append=True) units = progress_units(dl, total_length) progress = human_readable_progress(dl, total_length) if last_progress != units: # Avoid screen refresh if nothing has change if self.output: print_progress(self.output, units, progress) last_progress = units duration = time.time() - t1 log_download(url, duration) if not file_path: return bytes(ret) else: return except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def download(self, url, file_path=None, auth=None, retry=1, retry_wait=0, overwrite=False, headers=None): if file_path and not os.path.isabs(file_path): file_path = os.path.abspath(file_path) if file_path and os.path.exists(file_path): if overwrite: if self.output: self.output.warn("file '%s' already exists, overwriting" % file_path) else: # Should not happen, better to raise, probably we had to remove # the dest folder before raise ConanException( "Error, the file to download already exists: '%s'" % file_path) t1 = time.time() ret = bytearray() response = call_with_retry(self.output, retry, retry_wait, self._download_file, url, auth, headers) if not response.ok: # Do not retry if not found or whatever controlled error raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: total_length = response.headers.get('content-length') if total_length is None: # no content length header if not file_path: ret += response.content else: total_length = len(response.content) progress = human_readable_progress(total_length, total_length) print_progress(self.output, 50, progress) save(file_path, response.content, append=True) else: total_length = int(total_length) encoding = response.headers.get('content-encoding') gzip = (encoding == "gzip") # chunked can be a problem: https://www.greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4 # It will not send content-length or should be ignored def download_chunks(file_handler=None, ret_buffer=None): """Write to a buffer or to a file handler""" chunk_size = 1024 if not file_path else 1024 * 100 download_size = 0 last_progress = None for data in response.iter_content(chunk_size=chunk_size): download_size += len(data) if ret_buffer is not None: ret_buffer.extend(data) if file_handler is not None: file_handler.write(to_file_bytes(data)) units = progress_units(download_size, total_length) progress = human_readable_progress( download_size, total_length) if last_progress != units: # Avoid screen refresh if nothing has change if self.output: print_progress(self.output, units, progress) last_progress = units return download_size if file_path: mkdir(os.path.dirname(file_path)) with open(file_path, 'wb') as handle: dl_size = download_chunks(file_handler=handle) else: dl_size = download_chunks(ret_buffer=ret) if dl_size != total_length and not gzip: raise ConanException("Transfer interrupted before " "complete: %s < %s" % (dl_size, total_length)) duration = time.time() - t1 log_download(url, duration) if not file_path: return bytes(ret) else: return except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _download_file(self, url, auth, headers, file_path, try_resume=False): t1 = time.time() if try_resume and file_path and os.path.exists(file_path): range_start = os.path.getsize(file_path) headers = headers.copy() if headers else {} headers["range"] = "bytes={}-".format(range_start) else: range_start = 0 try: response = self._requester.get(url, stream=True, verify=self._verify_ssl, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = range_start if path: mkdir(os.path.dirname(path)) mode = "ab" if range_start else "wb" with open(path, mode) as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size def get_total_length(): if range_start: content_range = response.headers.get("Content-Range", "") match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range) if not match or range_start != int(match.group(1)): raise ConanException("Error in resumed download from %s\n" "Incorrect Content-Range header %s" % (url, content_range)) return int(match.group(3)) else: total_size = response.headers.get('Content-Length') or len( response.content) return int(total_size) try: logger.debug("DOWNLOAD: %s" % url) total_length = get_total_length() action = "Downloading" if range_start == 0 else "Continuing download of" description = "{} {}".format( action, os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self._output, description) progress.initial_value(range_start) chunk_size = 1024 if not file_path else 1024 * 100 written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size)), file_path) gzip = (response.headers.get("content-encoding") == "gzip") response.close() # it seems that if gzip we don't know the size, cannot resume and shouldn't raise if total_downloaded_size != total_length and not gzip: if (file_path and total_length > total_downloaded_size > range_start and response.headers.get("Accept-Ranges") == "bytes"): written_chunks = self._download_file(url, auth, headers, file_path, try_resume=True) else: raise ConanException( "Transfer interrupted before complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = 0 if path: mkdir(os.path.dirname(path)) with open(path, 'wb') as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size try: logger.debug("DOWNLOAD: %s" % url) total_length = response.headers.get('content-length') or len(response.content) total_length = int(total_length) description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self.output, description, print_dot=False) chunk_size = 1024 if not file_path else 1024 * 100 encoding = response.headers.get('content-encoding') gzip = (encoding == "gzip") written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size), chunk_size), file_path ) response.close() if total_downloaded_size != total_length and not gzip: raise ConanException("Transfer interrupted before " "complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))