def upload(self, url, abs_path, auth=None, dedup=False, retry=1, retry_wait=0, headers=None):
# Send always the header with the Sha1
headers = headers or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers,
auth=auth)
if response.status_code == 401:
raise AuthenticationException(response.content)
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response.content)
raise ForbiddenException(response.content)
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
self.output.info("")
# Actual transfer of the real content
it = load_in_chunks(abs_path, self.chunk_size)
# Now it is a chunked read file
file_size = os.stat(abs_path).st_size
it = upload_with_progress(file_size, it, self.chunk_size, self.output)
# Now it will print progress in each iteration
iterable_to_file = IterableToFileAdapter(it, file_size)
# Now it is prepared to work with request
ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url,
data=iterable_to_file, headers=headers, auth=auth)
return ret
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None,
headers=None):
retry = retry if retry is not None else self.requester.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = copy(headers) or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url,
abs_path=abs_path, headers=headers, auth=auth)
return ret
def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url, stream=True, verify=self.verify, auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" % (url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" % (response.status_code, url))
try:
logger.debug("DOWNLOAD: %s" % url)
data = self._download_data(response, file_path)
duration = time.time() - t1
log_download(url, duration)
return data
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError("Download failed, check server, possibly try again\n%s"
% str(e))
def _upload_file(self, url, data, headers, auth):
try:
response = self.requester.put(url,
data=data,
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
response.raise_for_status(
) # Raise HTTPError for bad http response status
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def _handle_400_response(response, auth):
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
def _upload_file(self, url, abs_path, headers, auth):
file_size = os.stat(abs_path).st_size
file_name = os.path.basename(abs_path)
description = "Uploading {}".format(file_name)
def load_in_chunks(_file, size):
"""Lazy function (generator) to read a file piece by piece.
Default chunk size: 1k."""
while True:
chunk = _file.read(size)
if not chunk:
break
yield chunk
with open(abs_path, mode='rb') as file_handler:
progress = progress_bar.Progress(file_size,
self.output,
description,
print_dot=True)
chunk_size = 1024
data = progress.update(load_in_chunks(file_handler, chunk_size),
chunk_size)
iterable_to_file = IterableToFileAdapter(data, file_size)
try:
response = self.requester.put(url,
data=iterable_to_file,
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(
response_to_str(response))
raise ForbiddenException(response_to_str(response))
response.raise_for_status(
) # Raise HTTPError for bad http response status
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def publish_build_info(build_info_file, url, user, password, apikey):
with open(build_info_file) as json_data:
parsed_uri = urlparse(url)
request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format(
uri=parsed_uri)
if user and password:
response = requests.put(
request_url,
headers={"Content-Type": "application/json"},
data=json_data,
auth=(user, password))
elif apikey:
response = requests.put(request_url,
headers={
"Content-Type": "application/json",
"X-JFrog-Art-Api": apikey
},
data=json_data)
else:
response = requests.put(request_url)
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
elif response.status_code != 204:
raise RequestErrorException(response_to_str(response))
def check_credentials(auth_user):
"""Just check if valid token. It not exception
is raised from Bottle plugin"""
if not auth_user:
raise AuthenticationException("Logged user needed!")
response.content_type = 'text/plain'
return auth_user
def _check_error_response(ret):
if ret.status_code == 401:
raise AuthenticationException("Wrong user or password")
# Cannot check content-type=text/html, conan server is doing it wrong
if not ret.ok or "html>" in str(ret.content):
raise ConanException("%s\n\nInvalid server response, check remote URL and "
"try again" % str(ret.content))
def retry_with_new_token(self, *args, **kwargs):
"""Try LOGIN_RETRIES to obtain a password from user input for which
we can get a valid token from api_client. If a token is returned,
credentials are stored in localdb and rest method is called"""
for _ in range(LOGIN_RETRIES):
user, password = self._user_io.request_login(
self._remote.name, self.user)
token = None
try:
token = self.authenticate(user, password)
except AuthenticationException:
if self.user is None:
self._user_io.out.error('Wrong user or password')
else:
self._user_io.out.error('Wrong password for user "%s"' %
self.user)
self._user_io.out.info(
'You can change username with "conan user <username>"')
if token:
logger.debug("Got token: %s" % str(token))
self._rest_client.token = token
self.user = user
self._store_login((user, token))
# Set custom headers of mac_digest and username
self.set_custom_headers(user)
return wrapper(self, *args, **kwargs)
raise AuthenticationException("Too many failed login attempts, bye!")
def get(url, **kwargs):
resp_basic_auth = Response()
resp_basic_auth.status_code = 200
if "authenticate" in url:
if kwargs["auth"].password != "PASSWORD!":
raise Exception("Bad password")
resp_basic_auth._content = b"TOKEN"
resp_basic_auth.headers = {"Content-Type": "text/plain"}
elif "ping" in url:
token = getattr(kwargs["auth"], "token", None)
password = getattr(kwargs["auth"], "password", None)
if token and token != "TOKEN":
raise Exception("Bad JWT Token")
if not token and not password:
raise AuthenticationException(
"I'm an Artifactory without anonymous access that "
"requires authentication for the ping endpoint and "
"I don't return the capabilities")
elif "search" in url:
if kwargs["auth"].token != "TOKEN":
raise Exception("Bad JWT Token")
resp_basic_auth._content = b'{"results": []}'
resp_basic_auth.headers = {
"Content-Type": "application/json"
}
else:
raise Exception("Shouldn't be more remote calls")
return resp_basic_auth
def authenticate(self, user, password):
if user is None: # The user is already in DB, just need the password
prev_user = self._localdb.get_username(self._remote.url)
if prev_user is None:
raise ConanException("User for remote '%s' is not defined" %
self._remote.name)
else:
user = prev_user
# Create a connection to the org
organization_url, _ = self._get_organization_url_and_feed(self._remote)
credentials = BasicAuthentication(user, password)
connection = Connection(base_url=organization_url, creds=credentials)
# Authenticate the connection
try:
connection.authenticate()
except Exception as ex:
raise AuthenticationException(ex)
# Generate token
token_raw_bytes = password.encode()
token_bytes = base64.b64encode(token_raw_bytes)
# Store result in DB
remote_name, prev_user, user = update_localdb(
self._local_db, user, token_bytes.decode(),
self._sxor(user, password), self._remote)
return remote_name, prev_user, user
def _check_rule_ok(self, username, rule, conan_reference):
"""Checks if a rule specified in config file applies to current conans
reference and current user"""
try:
rule_ref = ConanFileReference.loads(rule[0])
except Exception:
# TODO: Log error
raise InternalErrorException("Invalid server configuration. "
"Contact the administrator.")
authorized_users = [_.strip() for _ in rule[1].split(",")]
if len(authorized_users) < 1:
raise InternalErrorException("Invalid server configuration. "
"Contact the administrator.")
# Check if rule apply conan_reference
if self._check_ref_apply_for_rule(rule_ref, conan_reference):
if authorized_users[0] == "*" or username in authorized_users:
return True # Ok, applies and match username
else:
if username:
if authorized_users[0] == "?":
return True #Ok, applies and match any authenticated username
else:
raise ForbiddenException("Permission denied")
else:
raise AuthenticationException()
return False
def _upload_file(self, url, data, headers, auth):
try:
response = self.requester.put(url, data=data, verify=self.verify,
headers=headers, auth=auth)
if response.status_code == 401:
raise AuthenticationException(response.content)
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response.content)
raise ForbiddenException(response.content)
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def authenticate(self, username, password):
valid = self.authenticator.valid_user(username, password)
# If user is valid returns a token
if valid:
token = self.credentials_manager.get_token_for(username)
return token
else:
raise AuthenticationException("Wrong user or password")
def check_credentials(self):
organization_url, _ = self._get_organization_url_and_feed(self._remote)
user, token, refresh_token = self._local_db.get_login(self._remote.url)
if not user or not refresh_token:
raise AuthenticationException('Username or password not valid.')
# Decrypt password
password = self._sxor(user, refresh_token)
# Create a connection to the org
credentials = BasicAuthentication(user, password)
self._connection = Connection(base_url=organization_url,
creds=credentials)
# Authenticate the connection
try:
self._connection.authenticate()
except Exception as ex:
raise AuthenticationException(ex)
def _check_any_rule_ok(self, username, rules, *args, **kwargs):
for rule in rules:
# raises if don't
ret = self._check_rule_ok(username, rule, *args, **kwargs)
if ret: # A rule is applied ok, if not apply keep looking
return True
if username:
raise ForbiddenException("Permission denied")
else:
raise AuthenticationException()
def authenticate(http_basic_credentials):
if not http_basic_credentials:
raise AuthenticationException("Wrong user or password")
user_service = UserService(app.authenticator,
app.credentials_manager)
token = user_service.authenticate(http_basic_credentials.user,
http_basic_credentials.password)
return token
def authenticate(self, user, password):
"""Sends user + password to get a token"""
auth = HTTPBasicAuth(user, password)
url = self.router.common_authenticate()
logger.debug("REST: Authenticate: %s" % url)
ret = self.requester.get(url, auth=auth, headers=self.custom_headers,
verify=self.verify_ssl)
if ret.status_code == 401:
raise AuthenticationException("Wrong user or password")
# Cannot check content-type=text/html, conan server is doing it wrong
if not ret.ok or "html>" in str(ret.content):
raise ConanException("%s\n\nInvalid server response, check remote URL and "
"try again" % str(ret.content))
return ret
def get(self, url, *args, **kwargs):
if "authenticate" in url:
resp_basic_auth = Response()
resp_basic_auth._content = b"TOKEN"
resp_basic_auth.status_code = 200
resp_basic_auth.headers = {"Content-Type": "text/plain"}
return resp_basic_auth
if "ping" in url and not kwargs["auth"].token:
raise AuthenticationException(
"I'm an Artifactory without anonymous access that "
"requires authentication for the ping endpoint and "
"I don't return the capabilities")
raise Exception("Shouldn't be more remote calls")
def authenticate(self, user, password):
"""Sends user + password to get a token"""
auth = HTTPBasicAuth(user, password)
url = "%s/users/authenticate" % self._remote_api_url
t1 = time.time()
ret = self.requester.get(url, auth=auth, headers=self.custom_headers,
verify=self.verify_ssl)
if ret.status_code == 401:
raise AuthenticationException("Wrong user or password")
# Cannot check content-type=text/html, conan server is doing it wrong
if not ret.ok or "html>" in str(ret.content):
raise ConanException("%s\n\nInvalid server response, check remote URL and "
"try again" % str(ret.content))
duration = time.time() - t1
log_client_rest_api_call(url, "GET", duration, self.custom_headers)
return ret
def _retry_with_new_token(self, user, remote, method_name, *args, **kwargs):
"""Try LOGIN_RETRIES to obtain a password from user input for which
we can get a valid token from api_client. If a token is returned,
credentials are stored in localdb and rest method is called"""
for _ in range(LOGIN_RETRIES):
input_user, input_password = self._user_io.request_login(remote.name, user)
try:
self._authenticate(remote, input_user, input_password)
except AuthenticationException:
if user is None:
self._user_io.out.error('Wrong user or password')
else:
self._user_io.out.error('Wrong password for user "%s"' % user)
self._user_io.out.info('You can change username with "conan user <username>"')
else:
return self.call_rest_api_method(remote, method_name, *args, **kwargs)
raise AuthenticationException("Too many failed login attempts, bye!")
def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}",
package_id=None):
ret = {}
need_sources = False
if package_id:
data = metadata.packages[package_id].checksums
else:
data = metadata.recipe.checksums
need_sources = not ("conan_sources.tgz" in data)
for name, value in data.items():
name_or_id = name_format.format(name)
ret[value["sha1"]] = {"md5": value["md5"],
"name": name_or_id if not use_id else None,
"id": name_or_id if use_id else None}
if need_sources:
remote_name = metadata.recipe.remote
remotes = self._conan_cache.registry.load_remotes()
remote_url = remotes[remote_name].url
parsed_uri = urlparse(remote_url)
base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format(
uri=parsed_uri)
request_url = urljoin(base_url, "{}/conan_sources.tgz".format(request_path))
if self._user and self._password:
response = requests.get(request_url, auth=(self._user, self._password))
elif self._apikey:
response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey})
else:
response = requests.get(request_url)
if response.status_code == 200:
data = response.json()
ret[data["checksums"]["sha1"]] = {"md5": data["checksums"],
"name": "conan_sources.tgz",
"id": None}
elif response.status_code == 401:
raise AuthenticationException(response_to_str(response))
else:
raise RequestErrorException(response_to_str(response))
return set([Artifact(k, **v) for k, v in ret.items()])
def _download_file(self, url, auth, headers, file_path, try_resume=False):
t1 = time.time()
if try_resume and file_path and os.path.exists(file_path):
range_start = os.path.getsize(file_path)
headers = headers.copy() if headers else {}
headers["range"] = "bytes={}-".format(range_start)
else:
range_start = 0
try:
response = self._requester.get(url,
stream=True,
verify=self._verify_ssl,
auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" %
(url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth is None or (hasattr(auth, "token")
and auth.token is None):
# TODO: This is a bit weird, why this conversion? Need to investigate
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
def read_response(size):
for chunk in response.iter_content(size):
yield chunk
def write_chunks(chunks, path):
ret = None
downloaded_size = range_start
if path:
mkdir(os.path.dirname(path))
mode = "ab" if range_start else "wb"
with open(path, mode) as file_handler:
for chunk in chunks:
assert ((six.PY3 and isinstance(chunk, bytes))
or (six.PY2 and isinstance(chunk, str)))
file_handler.write(chunk)
downloaded_size += len(chunk)
else:
ret_data = bytearray()
for chunk in chunks:
ret_data.extend(chunk)
downloaded_size += len(chunk)
ret = bytes(ret_data)
return ret, downloaded_size
def get_total_length():
if range_start:
content_range = response.headers.get("Content-Range", "")
match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range)
if not match or range_start != int(match.group(1)):
raise ConanException("Error in resumed download from %s\n"
"Incorrect Content-Range header %s" %
(url, content_range))
return int(match.group(3))
else:
total_size = response.headers.get('Content-Length') or len(
response.content)
return int(total_size)
try:
logger.debug("DOWNLOAD: %s" % url)
total_length = get_total_length()
action = "Downloading" if range_start == 0 else "Continuing download of"
description = "{} {}".format(
action, os.path.basename(file_path)) if file_path else None
progress = progress_bar.Progress(total_length, self._output,
description)
progress.initial_value(range_start)
chunk_size = 1024 if not file_path else 1024 * 100
written_chunks, total_downloaded_size = write_chunks(
progress.update(read_response(chunk_size)), file_path)
gzip = (response.headers.get("content-encoding") == "gzip")
response.close()
# it seems that if gzip we don't know the size, cannot resume and shouldn't raise
if total_downloaded_size != total_length and not gzip:
if (file_path
and total_length > total_downloaded_size > range_start
and response.headers.get("Accept-Ranges") == "bytes"):
written_chunks = self._download_file(url,
auth,
headers,
file_path,
try_resume=True)
else:
raise ConanException(
"Transfer interrupted before complete: %s < %s" %
(total_downloaded_size, total_length))
duration = time.time() - t1
log_download(url, duration)
return written_chunks
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url, stream=True, verify=self.verify, auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" % (url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth is None or (hasattr(auth, "token") and auth.token is None):
# TODO: This is a bit weird, why this conversion? Need to investigate
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" % (response.status_code, url))
def read_response(size):
for chunk in response.iter_content(size):
yield chunk
def write_chunks(chunks, path):
ret = None
downloaded_size = 0
if path:
mkdir(os.path.dirname(path))
with open(path, 'wb') as file_handler:
for chunk in chunks:
assert ((six.PY3 and isinstance(chunk, bytes)) or
(six.PY2 and isinstance(chunk, str)))
file_handler.write(chunk)
downloaded_size += len(chunk)
else:
ret_data = bytearray()
for chunk in chunks:
ret_data.extend(chunk)
downloaded_size += len(chunk)
ret = bytes(ret_data)
return ret, downloaded_size
try:
logger.debug("DOWNLOAD: %s" % url)
total_length = response.headers.get('content-length') or len(response.content)
total_length = int(total_length)
description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None
progress = progress_bar.Progress(total_length, self.output, description, print_dot=False)
chunk_size = 1024 if not file_path else 1024 * 100
encoding = response.headers.get('content-encoding')
gzip = (encoding == "gzip")
written_chunks, total_downloaded_size = write_chunks(
progress.update(read_response(chunk_size), chunk_size),
file_path
)
response.close()
if total_downloaded_size != total_length and not gzip:
raise ConanException("Transfer interrupted before "
"complete: %s < %s" % (total_downloaded_size, total_length))
duration = time.time() - t1
log_download(url, duration)
return written_chunks
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError("Download failed, check server, possibly try again\n%s"
% str(e))