def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: logger.debug("DOWNLOAD: %s" % url) data = self._download_data(response, file_path) duration = time.time() - t1 log_download(url, duration) return data except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))
def publish_build_info(build_info_file, url, user, password, apikey): with open(build_info_file) as json_data: parsed_uri = urlparse(url) request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format( uri=parsed_uri) if user and password: response = requests.put( request_url, headers={"Content-Type": "application/json"}, data=json_data, auth=(user, password)) elif apikey: response = requests.put(request_url, headers={ "Content-Type": "application/json", "X-JFrog-Art-Api": apikey }, data=json_data) else: response = requests.put(request_url) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) elif response.status_code != 204: raise RequestErrorException(response_to_str(response))
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None): retry = retry if retry is not None else self.requester.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = copy(headers) or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) if response.status_code == 201: # Artifactory returns 201 if the file is there return response ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, abs_path=abs_path, headers=headers, auth=auth) return ret
def _upload_file(self, url, data, headers, auth): try: response = self.requester.put(url, data=data, verify=self.verify, headers=headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) response.raise_for_status( ) # Raise HTTPError for bad http response status except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None): retry = retry if retry is not None else self.requester.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = headers or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) if response.status_code == 201: # Artifactory returns 201 if the file is there return response if not self.output.is_terminal: self.output.info("") # Actual transfer of the real content it = load_in_chunks(abs_path, self.chunk_size) # Now it is a chunked read file file_size = os.stat(abs_path).st_size file_name = os.path.basename(abs_path) it = upload_with_progress(file_size, it, self.chunk_size, self.output, file_name) # Now it will print progress in each iteration iterable_to_file = IterableToFileAdapter(it, file_size) # Now it is prepared to work with request ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, data=iterable_to_file, headers=headers, auth=auth) return ret
def _handle_400_response(response, auth): if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response))
def _upload_file(self, url, abs_path, headers, auth): file_size = os.stat(abs_path).st_size file_name = os.path.basename(abs_path) description = "Uploading {}".format(file_name) def load_in_chunks(_file, size): """Lazy function (generator) to read a file piece by piece. Default chunk size: 1k.""" while True: chunk = _file.read(size) if not chunk: break yield chunk with open(abs_path, mode='rb') as file_handler: progress = progress_bar.Progress(file_size, self.output, description, print_dot=True) chunk_size = 1024 data = progress.update(load_in_chunks(file_handler, chunk_size), chunk_size) iterable_to_file = IterableToFileAdapter(data, file_size) try: response = self.requester.put(url, data=iterable_to_file, verify=self.verify, headers=headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException( response_to_str(response)) raise ForbiddenException(response_to_str(response)) response.raise_for_status( ) # Raise HTTPError for bad http response status except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def get_json(self, url, data=None): headers = self.custom_headers if data: # POST request headers.update({'Content-type': 'application/json', 'Accept': 'text/plain', 'Accept': 'application/json'}) logger.debug("REST: post: %s" % url) response = self.requester.post(url, auth=self.auth, headers=headers, verify=self.verify_ssl, stream=True, data=json.dumps(data)) else: logger.debug("REST: get: %s" % url) response = self.requester.get(url, auth=self.auth, headers=headers, verify=self.verify_ssl, stream=True) if response.status_code != 200: # Error message is text response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes) raise get_exception_from_error(response.status_code)(response_to_str(response)) content = decode_text(response.content) content_type = response.headers.get("Content-Type") if content_type != 'application/json': raise ConanException("%s\n\nResponse from remote is not json, but '%s'" % (content, content_type)) try: # This can fail, if some proxy returns 200 and an html message result = json.loads(content) except Exception: raise ConanException("Remote responded with broken json: %s" % content) if not isinstance(result, dict): raise ConanException("Unexpected server response %s" % result) return result
def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}", package_id=None): ret = {} need_sources = False if package_id: data = metadata.packages[package_id].checksums else: data = metadata.recipe.checksums need_sources = not ("conan_sources.tgz" in data) for name, value in data.items(): name_or_id = name_format.format(name) ret[value["sha1"]] = {"md5": value["md5"], "name": name_or_id if not use_id else None, "id": name_or_id if use_id else None} if need_sources: remote_name = metadata.recipe.remote remotes = self._conan_cache.registry.load_remotes() remote_url = remotes[remote_name].url parsed_uri = urlparse(remote_url) base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format( uri=parsed_uri) request_url = urljoin(base_url, "{}/conan_sources.tgz".format(request_path)) if self._user and self._password: response = requests.get(request_url, auth=(self._user, self._password)) elif self._apikey: response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey}) else: response = requests.get(request_url) if response.status_code == 200: data = response.json() ret[data["checksums"]["sha1"]] = {"md5": data["checksums"], "name": "conan_sources.tgz", "id": None} elif response.status_code == 401: raise AuthenticationException(response_to_str(response)) else: raise RequestErrorException(response_to_str(response)) return set([Artifact(k, **v) for k, v in ret.items()])
def server_capabilities(self): """Get information about the server: status, version, type and capabilities""" url = self.router.ping() logger.debug("REST: ping: %s" % url) ret = self.requester.get(url, auth=self.auth, headers=self.custom_headers, verify=self.verify_ssl) server_capabilities = ret.headers.get('X-Conan-Server-Capabilities', "") if not server_capabilities and not ret.ok: # Old Artifactory might return 401/403 without capabilities, we don't want # to cache them #5687, so raise the exception and force authentication raise get_exception_from_error(ret.status_code)(response_to_str(ret)) return [cap.strip() for cap in server_capabilities.split(",") if cap]
def server_capabilities(self, user=None, password=None): """Get information about the server: status, version, type and capabilities""" url = self.router.ping() logger.debug("REST: ping: %s" % url) if user and password: # This can happen in "conan user" cmd. Instead of empty token, use HttpBasic auth = HTTPBasicAuth(user, password) else: auth = self.auth ret = self.requester.get(url, auth=auth, headers=self.custom_headers, verify=self.verify_ssl) server_capabilities = ret.headers.get('X-Conan-Server-Capabilities', "") if not server_capabilities and not ret.ok: # Old Artifactory might return 401/403 without capabilities, we don't want # to cache them #5687, so raise the exception and force authentication raise get_exception_from_error(ret.status_code)(response_to_str(ret)) return [cap.strip() for cap in server_capabilities.split(",") if cap]
def _dedup(self, url, headers, auth): """ send the headers to see if it is possible to skip uploading the file, because it is already in the server. Artifactory support file deduplication """ dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self._requester.put(url, data="", verify=self._verify_ssl, headers=dedup_headers, auth=auth) if response.status_code == 500: raise InternalErrorException(response_to_str(response)) self._handle_400_response(response, auth) if response.status_code == 201: # Artifactory returns 201 if the file is there return response
def _download_file(self, url, auth, headers, file_path, try_resume=False): t1 = time.time() if try_resume and file_path and os.path.exists(file_path): range_start = os.path.getsize(file_path) headers = headers.copy() if headers else {} headers["range"] = "bytes={}-".format(range_start) else: range_start = 0 try: response = self._requester.get(url, stream=True, verify=self._verify_ssl, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = range_start if path: mkdir(os.path.dirname(path)) mode = "ab" if range_start else "wb" with open(path, mode) as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size def get_total_length(): if range_start: content_range = response.headers.get("Content-Range", "") match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range) if not match or range_start != int(match.group(1)): raise ConanException("Error in resumed download from %s\n" "Incorrect Content-Range header %s" % (url, content_range)) return int(match.group(3)) else: total_size = response.headers.get('Content-Length') or len( response.content) return int(total_size) try: logger.debug("DOWNLOAD: %s" % url) total_length = get_total_length() action = "Downloading" if range_start == 0 else "Continuing download of" description = "{} {}".format( action, os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self._output, description) progress.initial_value(range_start) chunk_size = 1024 if not file_path else 1024 * 100 written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size)), file_path) gzip = (response.headers.get("content-encoding") == "gzip") response.close() # it seems that if gzip we don't know the size, cannot resume and shouldn't raise if total_downloaded_size != total_length and not gzip: if (file_path and total_length > total_downloaded_size > range_start and response.headers.get("Accept-Ranges") == "bytes"): written_chunks = self._download_file(url, auth, headers, file_path, try_resume=True) else: raise ConanException( "Transfer interrupted before complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = 0 if path: mkdir(os.path.dirname(path)) with open(path, 'wb') as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size try: logger.debug("DOWNLOAD: %s" % url) total_length = response.headers.get('content-length') or len(response.content) total_length = int(total_length) description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self.output, description, print_dot=False) chunk_size = 1024 if not file_path else 1024 * 100 encoding = response.headers.get('content-encoding') gzip = (encoding == "gzip") written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size), chunk_size), file_path ) response.close() if total_downloaded_size != total_length and not gzip: raise ConanException("Transfer interrupted before " "complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))
def test_json_error(self): response = self._get_json_response() result = rest.response_to_str(response) self.assertEqual("Could not find the artifact", result)
def test_html_error(self): response = self._get_html_response() result = rest.response_to_str(response) self.assertEqual("404: Not Found", result)