def publish_build_info(build_info_file, url, user, password, apikey): with open(build_info_file) as json_data: parsed_uri = urlparse(url) request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format( uri=parsed_uri) if user and password: response = requests.put( request_url, headers={"Content-Type": "application/json"}, data=json_data, auth=(user, password)) elif apikey: response = requests.put(request_url, headers={ "Content-Type": "application/json", "X-JFrog-Art-Api": apikey }, data=json_data) else: response = requests.put(request_url) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) elif response.status_code != 204: raise RequestErrorException(response_to_str(response))
def put_file(self, file_saver, abs_filepath, token, upload_size): """ file_saver is an object with the save() method without parameters """ try: encoded_path, filesize, user = self.updown_auth_manager.get_resource_info( token) # Check size if upload_size != filesize: logger.debug("Invalid size file!!: %s: %s" % (user, abs_filepath)) raise RequestErrorException("Bad file size") abs_encoded_path = os.path.abspath( os.path.join(self.base_store_folder, encoded_path)) if not self._valid_path(abs_filepath, abs_encoded_path): raise NotFoundException("File not found") logger.debug("Put file: %s: %s" % (user, abs_filepath)) mkdir(os.path.dirname(abs_filepath)) if os.path.exists(abs_filepath): os.remove(abs_filepath) file_saver.save(os.path.dirname(abs_filepath)) except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError): raise NotFoundException("File not found")
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None): retry = retry if retry is not None else self.requester.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = copy(headers) or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) if response.status_code == 201: # Artifactory returns 201 if the file is there return response ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, abs_path=abs_path, headers=headers, auth=auth) return ret
def _upload_file(self, url, data, headers, auth): try: response = self.requester.put(url, data=data, verify=self.verify, headers=headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) response.raise_for_status( ) # Raise HTTPError for bad http response status except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None): retry = retry if retry is not None else self.requester.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = headers or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) if response.status_code == 201: # Artifactory returns 201 if the file is there return response if not self.output.is_terminal: self.output.info("") # Actual transfer of the real content it = load_in_chunks(abs_path, self.chunk_size) # Now it is a chunked read file file_size = os.stat(abs_path).st_size file_name = os.path.basename(abs_path) it = upload_with_progress(file_size, it, self.chunk_size, self.output, file_name) # Now it will print progress in each iteration iterable_to_file = IterableToFileAdapter(it, file_size) # Now it is prepared to work with request ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, data=iterable_to_file, headers=headers, auth=auth) return ret
def _handle_400_response(response, auth): if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response))
def _upload_file(self, url, abs_path, headers, auth): file_size = os.stat(abs_path).st_size file_name = os.path.basename(abs_path) description = "Uploading {}".format(file_name) def load_in_chunks(_file, size): """Lazy function (generator) to read a file piece by piece. Default chunk size: 1k.""" while True: chunk = _file.read(size) if not chunk: break yield chunk with open(abs_path, mode='rb') as file_handler: progress = progress_bar.Progress(file_size, self.output, description, print_dot=True) chunk_size = 1024 data = progress.update(load_in_chunks(file_handler, chunk_size), chunk_size) iterable_to_file = IterableToFileAdapter(data, file_size) try: response = self.requester.put(url, data=iterable_to_file, verify=self.verify, headers=headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException( response_to_str(response)) raise ForbiddenException(response_to_str(response)) response.raise_for_status( ) # Raise HTTPError for bad http response status except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def _validate_conan_reg_filenames(files): message = "Invalid conans request" # Could be partial uploads, so we can't expect for all files to be present # # conanfile and digest in files # if CONANFILE not in files: # # Log something # raise RequestErrorException("Missing %s" % CONANFILE) # if CONAN_MANIFEST not in files: # # Log something # raise RequestErrorException("Missing %s" % CONAN_MANIFEST) # All contents in same directory (from conan_id) for filename in files: if ".." in filename: # Log something raise RequestErrorException(message)
def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}", package_id=None): ret = {} need_sources = False if package_id: data = metadata.packages[package_id].checksums else: data = metadata.recipe.checksums need_sources = not ("conan_sources.tgz" in data) for name, value in data.items(): name_or_id = name_format.format(name) ret[value["sha1"]] = {"md5": value["md5"], "name": name_or_id if not use_id else None, "id": name_or_id if use_id else None} if need_sources: remote_name = metadata.recipe.remote remotes = self._conan_cache.registry.load_remotes() remote_url = remotes[remote_name].url parsed_uri = urlparse(remote_url) base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format( uri=parsed_uri) request_url = urljoin(base_url, "{}/conan_sources.tgz".format(request_path)) if self._user and self._password: response = requests.get(request_url, auth=(self._user, self._password)) elif self._apikey: response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey}) else: response = requests.get(request_url) if response.status_code == 200: data = response.json() ret[data["checksums"]["sha1"]] = {"md5": data["checksums"], "name": "conan_sources.tgz", "id": None} elif response.status_code == 401: raise AuthenticationException(response_to_str(response)) else: raise RequestErrorException(response_to_str(response)) return set([Artifact(k, **v) for k, v in ret.items()])