def publish_build_info(build_info_file, url, user, password, apikey):
with open(build_info_file) as json_data:
parsed_uri = urlparse(url)
request_url = "{uri.scheme}://{uri.netloc}/artifactory/api/build".format(
uri=parsed_uri)
if user and password:
response = requests.put(
request_url,
headers={"Content-Type": "application/json"},
data=json_data,
auth=(user, password))
elif apikey:
response = requests.put(request_url,
headers={
"Content-Type": "application/json",
"X-JFrog-Art-Api": apikey
},
data=json_data)
else:
response = requests.put(request_url)
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
elif response.status_code != 204:
raise RequestErrorException(response_to_str(response))
def put_file(self, file_saver, abs_filepath, token, upload_size):
"""
file_saver is an object with the save() method without parameters
"""
try:
encoded_path, filesize, user = self.updown_auth_manager.get_resource_info(
token)
# Check size
if upload_size != filesize:
logger.debug("Invalid size file!!: %s: %s" %
(user, abs_filepath))
raise RequestErrorException("Bad file size")
abs_encoded_path = os.path.abspath(
os.path.join(self.base_store_folder, encoded_path))
if not self._valid_path(abs_filepath, abs_encoded_path):
raise NotFoundException("File not found")
logger.debug("Put file: %s: %s" % (user, abs_filepath))
mkdir(os.path.dirname(abs_filepath))
if os.path.exists(abs_filepath):
os.remove(abs_filepath)
file_saver.save(os.path.dirname(abs_filepath))
except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError):
raise NotFoundException("File not found")
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None,
headers=None):
retry = retry if retry is not None else self.requester.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = copy(headers) or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url,
abs_path=abs_path, headers=headers, auth=auth)
return ret
def _upload_file(self, url, data, headers, auth):
try:
response = self.requester.put(url,
data=data,
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
response.raise_for_status(
) # Raise HTTPError for bad http response status
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def upload(self,
url,
abs_path,
auth=None,
dedup=False,
retry=None,
retry_wait=None,
headers=None):
retry = retry if retry is not None else self.requester.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = headers or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url,
data="",
verify=self.verify,
headers=dedup_headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
if not self.output.is_terminal:
self.output.info("")
# Actual transfer of the real content
it = load_in_chunks(abs_path, self.chunk_size)
# Now it is a chunked read file
file_size = os.stat(abs_path).st_size
file_name = os.path.basename(abs_path)
it = upload_with_progress(file_size, it, self.chunk_size, self.output,
file_name)
# Now it will print progress in each iteration
iterable_to_file = IterableToFileAdapter(it, file_size)
# Now it is prepared to work with request
ret = call_with_retry(self.output,
retry,
retry_wait,
self._upload_file,
url,
data=iterable_to_file,
headers=headers,
auth=auth)
return ret
def _handle_400_response(response, auth):
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
def _upload_file(self, url, abs_path, headers, auth):
file_size = os.stat(abs_path).st_size
file_name = os.path.basename(abs_path)
description = "Uploading {}".format(file_name)
def load_in_chunks(_file, size):
"""Lazy function (generator) to read a file piece by piece.
Default chunk size: 1k."""
while True:
chunk = _file.read(size)
if not chunk:
break
yield chunk
with open(abs_path, mode='rb') as file_handler:
progress = progress_bar.Progress(file_size,
self.output,
description,
print_dot=True)
chunk_size = 1024
data = progress.update(load_in_chunks(file_handler, chunk_size),
chunk_size)
iterable_to_file = IterableToFileAdapter(data, file_size)
try:
response = self.requester.put(url,
data=iterable_to_file,
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(
response_to_str(response))
raise ForbiddenException(response_to_str(response))
response.raise_for_status(
) # Raise HTTPError for bad http response status
except ConanException:
raise
except Exception as exc:
raise ConanException(exc)
return response
def _validate_conan_reg_filenames(files):
message = "Invalid conans request"
# Could be partial uploads, so we can't expect for all files to be present
# # conanfile and digest in files
# if CONANFILE not in files:
# # Log something
# raise RequestErrorException("Missing %s" % CONANFILE)
# if CONAN_MANIFEST not in files:
# # Log something
# raise RequestErrorException("Missing %s" % CONAN_MANIFEST)
# All contents in same directory (from conan_id)
for filename in files:
if ".." in filename:
# Log something
raise RequestErrorException(message)
def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}",
package_id=None):
ret = {}
need_sources = False
if package_id:
data = metadata.packages[package_id].checksums
else:
data = metadata.recipe.checksums
need_sources = not ("conan_sources.tgz" in data)
for name, value in data.items():
name_or_id = name_format.format(name)
ret[value["sha1"]] = {"md5": value["md5"],
"name": name_or_id if not use_id else None,
"id": name_or_id if use_id else None}
if need_sources:
remote_name = metadata.recipe.remote
remotes = self._conan_cache.registry.load_remotes()
remote_url = remotes[remote_name].url
parsed_uri = urlparse(remote_url)
base_url = "{uri.scheme}://{uri.netloc}/artifactory/api/storage/conan/".format(
uri=parsed_uri)
request_url = urljoin(base_url, "{}/conan_sources.tgz".format(request_path))
if self._user and self._password:
response = requests.get(request_url, auth=(self._user, self._password))
elif self._apikey:
response = requests.get(request_url, headers={"X-JFrog-Art-Api": self._apikey})
else:
response = requests.get(request_url)
if response.status_code == 200:
data = response.json()
ret[data["checksums"]["sha1"]] = {"md5": data["checksums"],
"name": "conan_sources.tgz",
"id": None}
elif response.status_code == 401:
raise AuthenticationException(response_to_str(response))
else:
raise RequestErrorException(response_to_str(response))
return set([Artifact(k, **v) for k, v in ret.items()])
