def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url, stream=True, verify=self.verify, auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" % (url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" % (response.status_code, url))
try:
logger.debug("DOWNLOAD: %s" % url)
data = self._download_data(response, file_path)
duration = time.time() - t1
log_download(url, duration)
return data
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError("Download failed, check server, possibly try again\n%s"
% str(e))
def publish_build_info(build_info_file, url, user, password, apikey):
with open(build_info_file) as json_data:
parsed_uri = urlparse(url)
request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format(
uri=parsed_uri)
if user and password:
response = requests.put(
request_url,
headers={"Content-Type": "application/json"},
data=json_data,
auth=(user, password))
elif apikey:
response = requests.put(request_url,
headers={
"Content-Type": "application/json",
"X-JFrog-Art-Api": apikey
},
data=json_data)
else:
response = requests.put(request_url)
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
elif response.status_code != 204:
raise RequestErrorException(response_to_str(response))
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None,
headers=None):
retry = retry if retry is not None else self.requester.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = copy(headers) or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url,
abs_path=abs_path, headers=headers, auth=auth)
return ret
def _upload_file(self, url, data, headers, auth):
try:
response = self.requester.put(url,
data=data,
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
response.raise_for_status(
) # Raise HTTPError for bad http response status
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def upload(self,
url,
abs_path,
auth=None,
dedup=False,
retry=None,
retry_wait=None,
headers=None):
retry = retry if retry is not None else self.requester.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = headers or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url,
data="",
verify=self.verify,
headers=dedup_headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
if not self.output.is_terminal:
self.output.info("")
# Actual transfer of the real content
it = load_in_chunks(abs_path, self.chunk_size)
# Now it is a chunked read file
file_size = os.stat(abs_path).st_size
file_name = os.path.basename(abs_path)
it = upload_with_progress(file_size, it, self.chunk_size, self.output,
file_name)
# Now it will print progress in each iteration
iterable_to_file = IterableToFileAdapter(it, file_size)
# Now it is prepared to work with request
ret = call_with_retry(self.output,
retry,
retry_wait,
self._upload_file,
url,
data=iterable_to_file,
headers=headers,
auth=auth)
return ret
def _handle_400_response(response, auth):
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
def _upload_file(self, url, abs_path, headers, auth):
file_size = os.stat(abs_path).st_size
file_name = os.path.basename(abs_path)
description = "Uploading {}".format(file_name)
def load_in_chunks(_file, size):
"""Lazy function (generator) to read a file piece by piece.
Default chunk size: 1k."""
while True:
chunk = _file.read(size)
if not chunk:
break
yield chunk
with open(abs_path, mode='rb') as file_handler:
progress = progress_bar.Progress(file_size,
self.output,
description,
print_dot=True)
chunk_size = 1024
data = progress.update(load_in_chunks(file_handler, chunk_size),
chunk_size)
iterable_to_file = IterableToFileAdapter(data, file_size)
try:
response = self.requester.put(url,
data=iterable_to_file,
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(
response_to_str(response))
raise ForbiddenException(response_to_str(response))
response.raise_for_status(
) # Raise HTTPError for bad http response status
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def get_json(self, url, data=None):
headers = self.custom_headers
if data: # POST request
headers.update({'Content-type': 'application/json',
'Accept': 'text/plain',
'Accept': 'application/json'})
logger.debug("REST: post: %s" % url)
response = self.requester.post(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True,
data=json.dumps(data))
else:
logger.debug("REST: get: %s" % url)
response = self.requester.get(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True)
if response.status_code != 200: # Error message is text
response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
raise get_exception_from_error(response.status_code)(response_to_str(response))
content = decode_text(response.content)
content_type = response.headers.get("Content-Type")
if content_type != 'application/json':
raise ConanException("%s\n\nResponse from remote is not json, but '%s'"
% (content, content_type))
try: # This can fail, if some proxy returns 200 and an html message
result = json.loads(content)
except Exception:
raise ConanException("Remote responded with broken json: %s" % content)
if not isinstance(result, dict):
raise ConanException("Unexpected server response %s" % result)
return result
def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}",
package_id=None):
ret = {}
need_sources = False
if package_id:
data = metadata.packages[package_id].checksums
else:
data = metadata.recipe.checksums
need_sources = not ("conan_sources.tgz" in data)
for name, value in data.items():
name_or_id = name_format.format(name)
ret[value["sha1"]] = {"md5": value["md5"],
"name": name_or_id if not use_id else None,
"id": name_or_id if use_id else None}
if need_sources:
remote_name = metadata.recipe.remote
remotes = self._conan_cache.registry.load_remotes()
remote_url = remotes[remote_name].url
parsed_uri = urlparse(remote_url)
base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format(
uri=parsed_uri)
request_url = urljoin(base_url, "{}/conan_sources.tgz".format(request_path))
if self._user and self._password:
response = requests.get(request_url, auth=(self._user, self._password))
elif self._apikey:
response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey})
else:
response = requests.get(request_url)
if response.status_code == 200:
data = response.json()
ret[data["checksums"]["sha1"]] = {"md5": data["checksums"],
"name": "conan_sources.tgz",
"id": None}
elif response.status_code == 401:
raise AuthenticationException(response_to_str(response))
else:
raise RequestErrorException(response_to_str(response))
return set([Artifact(k, **v) for k, v in ret.items()])
def server_capabilities(self):
"""Get information about the server: status, version, type and capabilities"""
url = self.router.ping()
logger.debug("REST: ping: %s" % url)
ret = self.requester.get(url, auth=self.auth, headers=self.custom_headers,
verify=self.verify_ssl)
server_capabilities = ret.headers.get('X-Conan-Server-Capabilities', "")
if not server_capabilities and not ret.ok:
# Old Artifactory might return 401/403 without capabilities, we don't want
# to cache them #5687, so raise the exception and force authentication
raise get_exception_from_error(ret.status_code)(response_to_str(ret))
return [cap.strip() for cap in server_capabilities.split(",") if cap]
def server_capabilities(self, user=None, password=None):
"""Get information about the server: status, version, type and capabilities"""
url = self.router.ping()
logger.debug("REST: ping: %s" % url)
if user and password:
# This can happen in "conan user" cmd. Instead of empty token, use HttpBasic
auth = HTTPBasicAuth(user, password)
else:
auth = self.auth
ret = self.requester.get(url, auth=auth, headers=self.custom_headers, verify=self.verify_ssl)
server_capabilities = ret.headers.get('X-Conan-Server-Capabilities', "")
if not server_capabilities and not ret.ok:
# Old Artifactory might return 401/403 without capabilities, we don't want
# to cache them #5687, so raise the exception and force authentication
raise get_exception_from_error(ret.status_code)(response_to_str(ret))
return [cap.strip() for cap in server_capabilities.split(",") if cap]
def _dedup(self, url, headers, auth):
""" send the headers to see if it is possible to skip uploading the file, because it
is already in the server. Artifactory support file deduplication
"""
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self._requester.put(url,
data="",
verify=self._verify_ssl,
headers=dedup_headers,
auth=auth)
if response.status_code == 500:
raise InternalErrorException(response_to_str(response))
self._handle_400_response(response, auth)
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
def _download_file(self, url, auth, headers, file_path, try_resume=False):
t1 = time.time()
if try_resume and file_path and os.path.exists(file_path):
range_start = os.path.getsize(file_path)
headers = headers.copy() if headers else {}
headers["range"] = "bytes={}-".format(range_start)
else:
range_start = 0
try:
response = self._requester.get(url,
stream=True,
verify=self._verify_ssl,
auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" %
(url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth is None or (hasattr(auth, "token")
and auth.token is None):
# TODO: This is a bit weird, why this conversion? Need to investigate
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
def read_response(size):
for chunk in response.iter_content(size):
yield chunk
def write_chunks(chunks, path):
ret = None
downloaded_size = range_start
if path:
mkdir(os.path.dirname(path))
mode = "ab" if range_start else "wb"
with open(path, mode) as file_handler:
for chunk in chunks:
assert ((six.PY3 and isinstance(chunk, bytes))
or (six.PY2 and isinstance(chunk, str)))
file_handler.write(chunk)
downloaded_size += len(chunk)
else:
ret_data = bytearray()
for chunk in chunks:
ret_data.extend(chunk)
downloaded_size += len(chunk)
ret = bytes(ret_data)
return ret, downloaded_size
def get_total_length():
if range_start:
content_range = response.headers.get("Content-Range", "")
match = re.match(r"^bytes (\d+)-(\d+)/(\d+)", content_range)
if not match or range_start != int(match.group(1)):
raise ConanException("Error in resumed download from %s\n"
"Incorrect Content-Range header %s" %
(url, content_range))
return int(match.group(3))
else:
total_size = response.headers.get('Content-Length') or len(
response.content)
return int(total_size)
try:
logger.debug("DOWNLOAD: %s" % url)
total_length = get_total_length()
action = "Downloading" if range_start == 0 else "Continuing download of"
description = "{} {}".format(
action, os.path.basename(file_path)) if file_path else None
progress = progress_bar.Progress(total_length, self._output,
description)
progress.initial_value(range_start)
chunk_size = 1024 if not file_path else 1024 * 100
written_chunks, total_downloaded_size = write_chunks(
progress.update(read_response(chunk_size)), file_path)
gzip = (response.headers.get("content-encoding") == "gzip")
response.close()
# it seems that if gzip we don't know the size, cannot resume and shouldn't raise
if total_downloaded_size != total_length and not gzip:
if (file_path
and total_length > total_downloaded_size > range_start
and response.headers.get("Accept-Ranges") == "bytes"):
written_chunks = self._download_file(url,
auth,
headers,
file_path,
try_resume=True)
else:
raise ConanException(
"Transfer interrupted before complete: %s < %s" %
(total_downloaded_size, total_length))
duration = time.time() - t1
log_download(url, duration)
return written_chunks
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url, stream=True, verify=self.verify, auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" % (url, exc))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 403:
if auth is None or (hasattr(auth, "token") and auth.token is None):
# TODO: This is a bit weird, why this conversion? Need to investigate
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" % (response.status_code, url))
def read_response(size):
for chunk in response.iter_content(size):
yield chunk
def write_chunks(chunks, path):
ret = None
downloaded_size = 0
if path:
mkdir(os.path.dirname(path))
with open(path, 'wb') as file_handler:
for chunk in chunks:
assert ((six.PY3 and isinstance(chunk, bytes)) or
(six.PY2 and isinstance(chunk, str)))
file_handler.write(chunk)
downloaded_size += len(chunk)
else:
ret_data = bytearray()
for chunk in chunks:
ret_data.extend(chunk)
downloaded_size += len(chunk)
ret = bytes(ret_data)
return ret, downloaded_size
try:
logger.debug("DOWNLOAD: %s" % url)
total_length = response.headers.get('content-length') or len(response.content)
total_length = int(total_length)
description = "Downloading {}".format(os.path.basename(file_path)) if file_path else None
progress = progress_bar.Progress(total_length, self.output, description, print_dot=False)
chunk_size = 1024 if not file_path else 1024 * 100
encoding = response.headers.get('content-encoding')
gzip = (encoding == "gzip")
written_chunks, total_downloaded_size = write_chunks(
progress.update(read_response(chunk_size), chunk_size),
file_path
)
response.close()
if total_downloaded_size != total_length and not gzip:
raise ConanException("Transfer interrupted before "
"complete: %s < %s" % (total_downloaded_size, total_length))
duration = time.time() - t1
log_download(url, duration)
return written_chunks
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError("Download failed, check server, possibly try again\n%s"
% str(e))
def test_json_error(self):
response = self._get_json_response()
result = rest.response_to_str(response)
self.assertEqual("Could not find the artifact", result)
def test_html_error(self):
response = self._get_html_response()
result = rest.response_to_str(response)
self.assertEqual("404: Not Found", result)
