def upload(self, url, abs_path, auth=None, dedup=False, retry=1, retry_wait=0, headers=None): # Send always the header with the Sha1 headers = headers or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 401: raise AuthenticationException(response.content) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response.content) raise ForbiddenException(response.content) if response.status_code == 201: # Artifactory returns 201 if the file is there return response self.output.info("") # Actual transfer of the real content it = load_in_chunks(abs_path, self.chunk_size) # Now it is a chunked read file file_size = os.stat(abs_path).st_size it = upload_with_progress(file_size, it, self.chunk_size, self.output) # Now it will print progress in each iteration iterable_to_file = IterableToFileAdapter(it, file_size) # Now it is prepared to work with request ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, data=iterable_to_file, headers=headers, auth=auth) return ret
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None): retry = retry if retry is not None else self.requester.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = copy(headers) or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) if response.status_code == 201: # Artifactory returns 201 if the file is there return response ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, abs_path=abs_path, headers=headers, auth=auth) return ret
def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) try: logger.debug("DOWNLOAD: %s" % url) data = self._download_data(response, file_path) duration = time.time() - t1 log_download(url, duration) return data except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))
def _upload_file(self, url, data, headers, auth): try: response = self.requester.put(url, data=data, verify=self.verify, headers=headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) response.raise_for_status( ) # Raise HTTPError for bad http response status except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def _handle_400_response(response, auth): if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response))
def _upload_file(self, url, abs_path, headers, auth): file_size = os.stat(abs_path).st_size file_name = os.path.basename(abs_path) description = "Uploading {}".format(file_name) def load_in_chunks(_file, size): """Lazy function (generator) to read a file piece by piece. Default chunk size: 1k.""" while True: chunk = _file.read(size) if not chunk: break yield chunk with open(abs_path, mode='rb') as file_handler: progress = progress_bar.Progress(file_size, self.output, description, print_dot=True) chunk_size = 1024 data = progress.update(load_in_chunks(file_handler, chunk_size), chunk_size) iterable_to_file = IterableToFileAdapter(data, file_size) try: response = self.requester.put(url, data=iterable_to_file, verify=self.verify, headers=headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth.token is None: raise AuthenticationException( response_to_str(response)) raise ForbiddenException(response_to_str(response)) response.raise_for_status( ) # Raise HTTPError for bad http response status except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def publish_build_info(build_info_file, url, user, password, apikey): with open(build_info_file) as json_data: parsed_uri = urlparse(url) request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format( uri=parsed_uri) if user and password: response = requests.put( request_url, headers={"Content-Type": "application/json"}, data=json_data, auth=(user, password)) elif apikey: response = requests.put(request_url, headers={ "Content-Type": "application/json", "X-JFrog-Art-Api": apikey }, data=json_data) else: response = requests.put(request_url) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) elif response.status_code != 204: raise RequestErrorException(response_to_str(response))
def check_credentials(auth_user): """Just check if valid token. It not exception is raised from Bottle plugin""" if not auth_user: raise AuthenticationException("Logged user needed!") response.content_type = 'text/plain' return auth_user
def _check_error_response(ret): if ret.status_code == 401: raise AuthenticationException("Wrong user or password") # Cannot check content-type=text/html, conan server is doing it wrong if not ret.ok or "html>" in str(ret.content): raise ConanException("%s\n\nInvalid server response, check remote URL and " "try again" % str(ret.content))
def retry_with_new_token(self, *args, **kwargs): """Try LOGIN_RETRIES to obtain a password from user input for which we can get a valid token from api_client. If a token is returned, credentials are stored in localdb and rest method is called""" for _ in range(LOGIN_RETRIES): user, password = self._user_io.request_login( self._remote.name, self.user) token = None try: token = self.authenticate(user, password) except AuthenticationException: if self.user is None: self._user_io.out.error('Wrong user or password') else: self._user_io.out.error('Wrong password for user "%s"' % self.user) self._user_io.out.info( 'You can change username with "conan user <username>"') if token: logger.debug("Got token: %s" % str(token)) self._rest_client.token = token self.user = user self._store_login((user, token)) # Set custom headers of mac_digest and username self.set_custom_headers(user) return wrapper(self, *args, **kwargs) raise AuthenticationException("Too many failed login attempts, bye!")
def get(url, **kwargs): resp_basic_auth = Response() resp_basic_auth.status_code = 200 if "authenticate" in url: if kwargs["auth"].password != "PASSWORD!": raise Exception("Bad password") resp_basic_auth._content = b"TOKEN" resp_basic_auth.headers = {"Content-Type": "text/plain"} elif "ping" in url: token = getattr(kwargs["auth"], "token", None) password = getattr(kwargs["auth"], "password", None) if token and token != "TOKEN": raise Exception("Bad JWT Token") if not token and not password: raise AuthenticationException( "I'm an Artifactory without anonymous access that " "requires authentication for the ping endpoint and " "I don't return the capabilities") elif "search" in url: if kwargs["auth"].token != "TOKEN": raise Exception("Bad JWT Token") resp_basic_auth._content = b'{"results": []}' resp_basic_auth.headers = { "Content-Type": "application/json" } else: raise Exception("Shouldn't be more remote calls") return resp_basic_auth
def authenticate(self, user, password): if user is None: # The user is already in DB, just need the password prev_user = self._localdb.get_username(self._remote.url) if prev_user is None: raise ConanException("User for remote '%s' is not defined" % self._remote.name) else: user = prev_user # Create a connection to the org organization_url, _ = self._get_organization_url_and_feed(self._remote) credentials = BasicAuthentication(user, password) connection = Connection(base_url=organization_url, creds=credentials) # Authenticate the connection try: connection.authenticate() except Exception as ex: raise AuthenticationException(ex) # Generate token token_raw_bytes = password.encode() token_bytes = base64.b64encode(token_raw_bytes) # Store result in DB remote_name, prev_user, user = update_localdb( self._local_db, user, token_bytes.decode(), self._sxor(user, password), self._remote) return remote_name, prev_user, user
def _check_rule_ok(self, username, rule, conan_reference): """Checks if a rule specified in config file applies to current conans reference and current user""" try: rule_ref = ConanFileReference.loads(rule[0]) except Exception: # TODO: Log error raise InternalErrorException("Invalid server configuration. " "Contact the administrator.") authorized_users = [_.strip() for _ in rule[1].split(",")] if len(authorized_users) < 1: raise InternalErrorException("Invalid server configuration. " "Contact the administrator.") # Check if rule apply conan_reference if self._check_ref_apply_for_rule(rule_ref, conan_reference): if authorized_users[0] == "*" or username in authorized_users: return True # Ok, applies and match username else: if username: if authorized_users[0] == "?": return True #Ok, applies and match any authenticated username else: raise ForbiddenException("Permission denied") else: raise AuthenticationException() return False
def _upload_file(self, url, data, headers, auth): try: response = self.requester.put(url, data=data, verify=self.verify, headers=headers, auth=auth) if response.status_code == 401: raise AuthenticationException(response.content) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response.content) raise ForbiddenException(response.content) except ConanException: raise except Exception as exc: raise ConanException(exc) return response
def authenticate(self, username, password): valid = self.authenticator.valid_user(username, password) # If user is valid returns a token if valid: token = self.credentials_manager.get_token_for(username) return token else: raise AuthenticationException("Wrong user or password")
def check_credentials(self): organization_url, _ = self._get_organization_url_and_feed(self._remote) user, token, refresh_token = self._local_db.get_login(self._remote.url) if not user or not refresh_token: raise AuthenticationException('Username or password not valid.') # Decrypt password password = self._sxor(user, refresh_token) # Create a connection to the org credentials = BasicAuthentication(user, password) self._connection = Connection(base_url=organization_url, creds=credentials) # Authenticate the connection try: self._connection.authenticate() except Exception as ex: raise AuthenticationException(ex)
def _check_any_rule_ok(self, username, rules, *args, **kwargs): for rule in rules: # raises if don't ret = self._check_rule_ok(username, rule, *args, **kwargs) if ret: # A rule is applied ok, if not apply keep looking return True if username: raise ForbiddenException("Permission denied") else: raise AuthenticationException()
def authenticate(http_basic_credentials): if not http_basic_credentials: raise AuthenticationException("Wrong user or password") user_service = UserService(app.authenticator, app.credentials_manager) token = user_service.authenticate(http_basic_credentials.user, http_basic_credentials.password) return token
def authenticate(self, user, password): """Sends user + password to get a token""" auth = HTTPBasicAuth(user, password) url = self.router.common_authenticate() logger.debug("REST: Authenticate: %s" % url) ret = self.requester.get(url, auth=auth, headers=self.custom_headers, verify=self.verify_ssl) if ret.status_code == 401: raise AuthenticationException("Wrong user or password") # Cannot check content-type=text/html, conan server is doing it wrong if not ret.ok or "html>" in str(ret.content): raise ConanException("%s\n\nInvalid server response, check remote URL and " "try again" % str(ret.content)) return ret
def get(self, url, *args, **kwargs): if "authenticate" in url: resp_basic_auth = Response() resp_basic_auth._content = b"TOKEN" resp_basic_auth.status_code = 200 resp_basic_auth.headers = {"Content-Type": "text/plain"} return resp_basic_auth if "ping" in url and not kwargs["auth"].token: raise AuthenticationException( "I'm an Artifactory without anonymous access that " "requires authentication for the ping endpoint and " "I don't return the capabilities") raise Exception("Shouldn't be more remote calls")
def authenticate(self, user, password): """Sends user + password to get a token""" auth = HTTPBasicAuth(user, password) url = "%s/users/authenticate" % self._remote_api_url t1 = time.time() ret = self.requester.get(url, auth=auth, headers=self.custom_headers, verify=self.verify_ssl) if ret.status_code == 401: raise AuthenticationException("Wrong user or password") # Cannot check content-type=text/html, conan server is doing it wrong if not ret.ok or "html>" in str(ret.content): raise ConanException("%s\n\nInvalid server response, check remote URL and " "try again" % str(ret.content)) duration = time.time() - t1 log_client_rest_api_call(url, "GET", duration, self.custom_headers) return ret
def _retry_with_new_token(self, user, remote, method_name, *args, **kwargs): """Try LOGIN_RETRIES to obtain a password from user input for which we can get a valid token from api_client. If a token is returned, credentials are stored in localdb and rest method is called""" for _ in range(LOGIN_RETRIES): input_user, input_password = self._user_io.request_login(remote.name, user) try: self._authenticate(remote, input_user, input_password) except AuthenticationException: if user is None: self._user_io.out.error('Wrong user or password') else: self._user_io.out.error('Wrong password for user "%s"' % user) self._user_io.out.info('You can change username with "conan user <username>"') else: return self.call_rest_api_method(remote, method_name, *args, **kwargs) raise AuthenticationException("Too many failed login attempts, bye!")
def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}", package_id=None): ret = {} need_sources = False if package_id: data = metadata.packages[package_id].checksums else: data = metadata.recipe.checksums need_sources = not ("conan_sources.tgz" in data) for name, value in data.items(): name_or_id = name_format.format(name) ret[value["sha1"]] = {"md5": value["md5"], "name": name_or_id if not use_id else None, "id": name_or_id if use_id else None} if need_sources: remote_name = metadata.recipe.remote remotes = self._conan_cache.registry.load_remotes() remote_url = remotes[remote_name].url parsed_uri = urlparse(remote_url) base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format( uri=parsed_uri) request_url = urljoin(base_url, "{}/conan_sources.tgz".format(request_path)) if self._user and self._password: response = requests.get(request_url, auth=(self._user, self._password)) elif self._apikey: response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey}) else: response = requests.get(request_url) if response.status_code == 200: data = response.json() ret[data["checksums"]["sha1"]] = {"md5": data["checksums"], "name": "conan_sources.tgz", "id": None} elif response.status_code == 401: raise AuthenticationException(response_to_str(response)) else: raise RequestErrorException(response_to_str(response)) return set([Artifact(k, **v) for k, v in ret.items()])
def _download_file(self, url, auth, headers, file_path, try_resume=False): t1 = time.time() if try_resume and file_path and os.path.exists(file_path): range_start = os.path.getsize(file_path) headers = headers.copy() if headers else {} headers["range"] = "bytes={}-".format(range_start) else: range_start = 0 try: response = self._requester.get(url, stream=True, verify=self._verify_ssl, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = range_start if path: mkdir(os.path.dirname(path)) mode = "ab" if range_start else "wb" with open(path, mode) as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size def get_total_length(): if range_start: content_range = response.headers.get("Content-Range", "") match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range) if not match or range_start != int(match.group(1)): raise ConanException("Error in resumed download from %s\n" "Incorrect Content-Range header %s" % (url, content_range)) return int(match.group(3)) else: total_size = response.headers.get('Content-Length') or len( response.content) return int(total_size) try: logger.debug("DOWNLOAD: %s" % url) total_length = get_total_length() action = "Downloading" if range_start == 0 else "Continuing download of" description = "{} {}".format( action, os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self._output, description) progress.initial_value(range_start) chunk_size = 1024 if not file_path else 1024 * 100 written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size)), file_path) gzip = (response.headers.get("content-encoding") == "gzip") response.close() # it seems that if gzip we don't know the size, cannot resume and shouldn't raise if total_downloaded_size != total_length and not gzip: if (file_path and total_length > total_downloaded_size > range_start and response.headers.get("Accept-Ranges") == "bytes"): written_chunks = self._download_file(url, auth, headers, file_path, try_resume=True) else: raise ConanException( "Transfer interrupted before complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError( "Download failed, check server, possibly try again\n%s" % str(e))
def _download_file(self, url, auth, headers, file_path): t1 = time.time() try: response = self.requester.get(url, stream=True, verify=self.verify, auth=auth, headers=headers) except Exception as exc: raise ConanException("Error downloading file %s: '%s'" % (url, exc)) if not response.ok: if response.status_code == 404: raise NotFoundException("Not found: %s" % url) elif response.status_code == 403: if auth is None or (hasattr(auth, "token") and auth.token is None): # TODO: This is a bit weird, why this conversion? Need to investigate raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) elif response.status_code == 401: raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) def read_response(size): for chunk in response.iter_content(size): yield chunk def write_chunks(chunks, path): ret = None downloaded_size = 0 if path: mkdir(os.path.dirname(path)) with open(path, 'wb') as file_handler: for chunk in chunks: assert ((six.PY3 and isinstance(chunk, bytes)) or (six.PY2 and isinstance(chunk, str))) file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() for chunk in chunks: ret_data.extend(chunk) downloaded_size += len(chunk) ret = bytes(ret_data) return ret, downloaded_size try: logger.debug("DOWNLOAD: %s" % url) total_length = response.headers.get('content-length') or len(response.content) total_length = int(total_length) description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None progress = progress_bar.Progress(total_length, self.output, description, print_dot=False) chunk_size = 1024 if not file_path else 1024 * 100 encoding = response.headers.get('content-encoding') gzip = (encoding == "gzip") written_chunks, total_downloaded_size = write_chunks( progress.update(read_response(chunk_size), chunk_size), file_path ) response.close() if total_downloaded_size != total_length and not gzip: raise ConanException("Transfer interrupted before " "complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) return written_chunks except Exception as e: logger.debug(e.__class__) logger.debug(traceback.format_exc()) # If this part failed, it means problems with the connection to server raise ConanConnectionError("Download failed, check server, possibly try again\n%s" % str(e))