def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exception_message_safe(exc))) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: data = self._download_data(response, file_path) duration = time.time() - t1 log_download(url, duration) return data except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def download(self, url, file_path=None, auth=None, retry=1, retry_wait=0): if file_path and os.path.exists(file_path): # Should not happen, better to raise, probably we had to remove the dest folder before raise ConanException( "Error, the file to download already exists: '%s'" % file_path) t1 = time.time() ret = bytearray() response = call_with_retry(self.output, retry, retry_wait, self._download_file, url, auth) if not response.ok: # Do not retry if not found or whatever controlled error raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: total_length = response.headers.get('content-length') if total_length is None: # no content length header if not file_path: ret += response.content else: total_length = len(response.content) progress = human_readable_progress(total_length, total_length) print_progress(self.output, 50, progress) save(file_path, response.content, append=True) else: dl = 0 total_length = int(total_length) last_progress = None chunk_size = 1024 if not file_path else 1024 * 100 for data in response.iter_content(chunk_size=chunk_size): dl += len(data) if not file_path: ret.extend(data) else: save(file_path, data, append=True) units = progress_units(dl, total_length) progress = human_readable_progress(dl, total_length) if last_progress != units: # Avoid screen refresh if nothing has change if self.output: print_progress(self.output, units, progress) last_progress = units duration = time.time() - t1 log_download(url, duration) if not file_path: return bytes(ret) else: return except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _retrieve_recipe(self, conan_reference, output): """ returns the requested conanfile object, retrieving it from remotes if necessary. Can raise NotFoundException """ def _retrieve_from_remote(remote): output.info("Trying with '%s'..." % remote.name) export_path = self._client_cache.export(conan_reference) result = self._remote_manager.get_recipe(conan_reference, export_path, remote) self._registry.set_ref(conan_reference, remote) return result if self._remote_name: output.info("Not found, retrieving from server '%s' " % self._remote_name) remote = self._registry.remote(self._remote_name) return _retrieve_from_remote(remote) else: output.info("Not found, looking in remotes...") remotes = self._registry.remotes for remote in remotes: logger.debug("Trying with remote %s" % remote.name) try: return _retrieve_from_remote(remote) # If exception continue with the next except (ConanOutdatedClient, ConanConnectionError) as exc: output.warn(str(exc)) if remote == remotes[-1]: # Last element not found raise ConanConnectionError("All remotes failed") except NotFoundException as exc: if remote == remotes[-1]: # Last element not found logger.debug("Not found in any remote, raising...%s" % exc) raise NotFoundException("Unable to find '%s' in remotes" % str(conan_reference)) raise ConanException("No remote defined")
def download(self, url): ret = bytearray() response = self.requester.get(url, stream=True, verify=self.verify) if not response.ok: raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: total_length = response.headers.get('content-length') if total_length is None: # no content length header ret += response.content else: dl = 0 total_length = int(total_length) last_progress = None for data in response.iter_content(chunk_size=1024): dl += len(data) ret.extend(data) units = progress_units(dl, total_length) if last_progress != units: # Avoid screen refresh if nothing has change if self.output: print_progress(self.output, units) last_progress = units return bytes(ret) except Exception as e: # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))
def _call_with_remote_selection(self, remote, method, *argc, **argv): """ Iterates in remotes until a response is found """ if remote: return self._call_without_remote_selection(remote, method, *argc, **argv) for remote in self.remote_names: logger.debug("Trying with remote %s" % self.remote_url(remote)) self._remote_client.remote_url = self.remote_url(remote) try: result = self._call_without_remote_selection( remote, method, *argc, **argv) self._output.success("Found in remote '%s'" % remote) return result # If exception continue with the next except (ConanOutdatedClient, ConanConnectionError) as exc: self._output.warn(str(exc)) if remote == self._remotes[-1][0]: # Last element not found raise ConanConnectionError("All remotes failed") except NotFoundException as exc: if remote == self._remotes[-1][0]: # Last element not found logger.debug("Not found in any remote, raising...%s" % exc) raise raise ConanException("No remote defined")
def _call_remote(self, remote, method, *argc, **argv): self._remote_client.remote = remote try: return getattr(self._remote_client, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("Unable to connect to %s=%s" % (remote.name, remote.url)) except ConanException: raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc)
def _call_remote(self, remote, method, *argc, **argv): self._auth_manager.remote = remote try: return getattr(self._auth_manager, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("%s\n\nUnable to connect to %s=%s" % (str(exc), remote.name, remote.url)) except ConanException as exc: raise exc.__class__("%s. [Remote: %s]" % (exception_message_safe(exc), remote.name)) except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc)
def _call_remote(self, remote, method, *argc, **argv): assert (isinstance(remote, Remote)) self._auth_manager.remote = remote try: return getattr(self._auth_manager, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("%s\n\nUnable to connect to %s=%s" % (str(exc), remote.name, remote.url)) except ConanException as exc: exc.remote = remote raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc, remote=remote)
def _call_remote(self, remote, method, *args, **kwargs): assert(isinstance(remote, Remote)) try: return self._auth_manager.call_rest_api_method(remote, method, *args, **kwargs) except ConnectionError as exc: raise ConanConnectionError(("%s\n\nUnable to connect to %s=%s\n" + "1. Make sure the remote is reachable or,\n" + "2. Disable it by using conan remote disable,\n" + "Then try again." ) % (str(exc), remote.name, remote.url)) except ConanException as exc: exc.remote = remote raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc, remote=remote)
def download(self, url, file_path=None, auth=None): ret = bytearray() response = self.requester.get(url, stream=True, verify=self.verify, auth=auth) if not response.ok: raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: total_length = response.headers.get('content-length') if total_length is None: # no content length header if not file_path: ret += response.content else: save(file_path, response.content, append=True) else: dl = 0 total_length = int(total_length) last_progress = None chunk_size = 1024 if not file_path else 1024 * 100 for data in response.iter_content(chunk_size=chunk_size): dl += len(data) if not file_path: ret.extend(data) else: save(file_path, data, append=True) units = progress_units(dl, total_length) if last_progress != units: # Avoid screen refresh if nothing has change if self.output: print_progress(self.output, units) last_progress = units if not file_path: return bytes(ret) else: return except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _call_without_remote_selection(self, remote, method, *argc, **argv): if not remote: remote = self.default_remote self._remote_client.remote_url = self.remote_url(remote) try: return getattr(self._remote_client, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError( "Unable to connect to %s=%s" % (remote, self._remote_client.remote_url)) except ConanException: raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc)
def download(self, url, file_path=None, auth=None, retry=1, retry_wait=0, overwrite=False, headers=None): if file_path and not os.path.isabs(file_path): file_path = os.path.abspath(file_path) if file_path and os.path.exists(file_path): if overwrite: if self.output: self.output.warn("file '%s' already exists, overwriting" % file_path) else: # Should not happen, better to raise, probably we had to remove # the dest folder before raise ConanException( "Error, the file to download already exists: '%s'" % file_path) t1 = time.time() ret = bytearray() response = call_with_retry(self.output, retry, retry_wait, self._download_file, url, auth, headers) if not response.ok: # Do not retry if not found or whatever controlled error raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: total_length = response.headers.get('content-length') if total_length is None: # no content length header if not file_path: ret += response.content else: total_length = len(response.content) progress = human_readable_progress(total_length, total_length) print_progress(self.output, 50, progress) save(file_path, response.content, append=True) else: total_length = int(total_length) encoding = response.headers.get('content-encoding') gzip = (encoding == "gzip") # chunked can be a problem: https://www.greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4 # It will not send content-length or should be ignored def download_chunks(file_handler=None, ret_buffer=None): """Write to a buffer or to a file handler""" chunk_size = 1024 if not file_path else 1024 * 100 download_size = 0 last_progress = None for data in response.iter_content(chunk_size=chunk_size): download_size += len(data) if ret_buffer is not None: ret_buffer.extend(data) if file_handler is not None: file_handler.write(to_file_bytes(data)) units = progress_units(download_size, total_length) progress = human_readable_progress( download_size, total_length) if last_progress != units: # Avoid screen refresh if nothing has change if self.output: print_progress(self.output, units, progress) last_progress = units return download_size if file_path: mkdir(os.path.dirname(file_path)) with open(file_path, 'wb') as handle: dl_size = download_chunks(file_handler=handle) else: dl_size = download_chunks(ret_buffer=ret) if dl_size != total_length and not gzip: raise ConanException("Transfer interrupted before " "complete: %s < %s" % (dl_size, total_length)) duration = time.time() - t1 log_download(url, duration) if not file_path: return bytes(ret) else: return except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _download_file(self, url, auth, headers, file_path, try_resume=False): t1 = time.time() if try_resume and file_path and os.path.exists(file_path): range_start = os.path.getsize(file_path) headers = headers.copy() if headers else {} headers["range"] = "bytes={}-".format(range_start) else: range_start = 0 try: response = self._requester.get(url, stream=True, verify=self._verify_ssl, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = range_start if path: mkdir(os.path.dirname(path)) mode = "ab" if range_start else "wb" with open(path, mode) as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size def get_total_length(): if range_start: content_range = response.headers.get("Content-Range", "") match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range) if not match or range_start != int(match.group(1)): raise ConanException("Error in resumed download from %s\n" "Incorrect Content-Range header %s" % (url, content_range)) return int(match.group(3)) else: total_size = response.headers.get('Content-Length') or len( response.content) return int(total_size) try: logger.debug("DOWNLOAD: %s" % url) total_length = get_total_length() action = "Downloading" if range_start == 0 else "Continuing download of" description = "{} {}".format( action, os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self._output, description) progress.initial_value(range_start) chunk_size = 1024 if not file_path else 1024 * 100 written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size)), file_path) gzip = (response.headers.get("content-encoding") == "gzip") response.close() # it seems that if gzip we don't know the size, cannot resume and shouldn't raise if total_downloaded_size != total_length and not gzip: if (file_path and total_length > total_downloaded_size > range_start and response.headers.get("Accept-Ranges") == "bytes"): written_chunks = self._download_file(url, auth, headers, file_path, try_resume=True) else: raise ConanException( "Transfer interrupted before complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = 0 if path: mkdir(os.path.dirname(path)) with open(path, 'wb') as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size try: logger.debug("DOWNLOAD: %s" % url) total_length = response.headers.get('content-length') or len(response.content) total_length = int(total_length) description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self.output, description, print_dot=False) chunk_size = 1024 if not file_path else 1024 * 100 encoding = response.headers.get('content-encoding') gzip = (encoding == "gzip") written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size), chunk_size), file_path ) response.close() if total_downloaded_size != total_length and not gzip: raise ConanException("Transfer interrupted before " "complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))