def upload(self, url, abs_path, auth=None, dedup=False):
if dedup:
headers = {
"X-Checksum-Deploy": "true",
"X-Checksum-Sha1": sha1sum(abs_path)
}
response = self.requester.put(url,
data="",
verify=self.verify,
headers=headers,
auth=auth)
if response.status_code != 404:
return response
self.output.info("")
# Actual transfer of the real content
it = load_in_chunks(abs_path, self.chunk_size)
# Now it is a chunked read file
file_size = os.stat(abs_path).st_size
it = upload_with_progress(file_size, it, self.chunk_size, self.output)
# Now it will print progress in each iteration
iterable_to_file = IterableToFileAdapter(it, file_size)
# Now it is prepared to work with request
ret = self.requester.put(url,
data=iterable_to_file,
verify=self.verify,
headers=None,
auth=auth)
return ret
def upload(self, url, abs_path, auth=None, dedup=False, retry=1, retry_wait=0, headers=None):
# Send always the header with the Sha1
headers = headers or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers,
auth=auth)
if response.status_code == 401:
raise AuthenticationException(response.content)
if response.status_code == 403:
if auth.token is None:
raise AuthenticationException(response.content)
raise ForbiddenException(response.content)
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
self.output.info("")
# Actual transfer of the real content
it = load_in_chunks(abs_path, self.chunk_size)
# Now it is a chunked read file
file_size = os.stat(abs_path).st_size
it = upload_with_progress(file_size, it, self.chunk_size, self.output)
# Now it will print progress in each iteration
iterable_to_file = IterableToFileAdapter(it, file_size)
# Now it is prepared to work with request
ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url,
data=iterable_to_file, headers=headers, auth=auth)
return ret
def _file_document(name, path):
return {
"name": name,
"path": path,
"md5": md5sum(path),
"sha1": sha1sum(path)
}
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None,
headers=None):
retry = retry if retry is not None else self.requester.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = copy(headers) or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
dedup_headers = {"X-Checksum-Deploy": "true"}
if headers:
dedup_headers.update(headers)
response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers,
auth=auth)
if response.status_code == 400:
raise RequestErrorException(response_to_str(response))
if response.status_code == 401:
raise AuthenticationException(response_to_str(response))
if response.status_code == 403:
if auth is None or auth.token is None:
raise AuthenticationException(response_to_str(response))
raise ForbiddenException(response_to_str(response))
if response.status_code == 201: # Artifactory returns 201 if the file is there
return response
ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url,
abs_path=abs_path, headers=headers, auth=auth)
return ret
def test_conan_data_as_source_newtools(self):
tgz_path = tgz_with_contents({"foo.txt": "foo"})
if sys.version_info.major == 3 and sys.version_info.minor >= 9:
# Python 3.9 changed the tar algorithm. Conan tgz will have different checksums
# https://github.com/conan-io/conan/issues/8020
md5_value = "7ebdc5ed79b7b72f3a6010da3671ae05"
sha1_value = "862c1b58de1dfadaad3206b453b4de731c1751af"
sha256_value = "25200fc2bd7f430358cd7a7c5ce4a84396e8ec68a1e9d8880994b1236f214972"
else:
md5_value = "2ef49b5a102db1abb775eaf1922d5662"
sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf"
sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338"
self.assertEqual(md5_value, md5sum(tgz_path))
self.assertEqual(sha1_value, sha1sum(tgz_path))
self.assertEqual(sha256_value, sha256sum(tgz_path))
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/myfile.tar.gz")
def get_file():
return static_file(os.path.basename(tgz_path),
root=os.path.dirname(tgz_path),
mimetype="")
thread.run_server()
client = TestClient()
conanfile = textwrap.dedent("""
from conans import ConanFile
from conan.tools.files import get
class Lib(ConanFile):
def source(self):
data = self.conan_data["sources"]["all"]
get(self, **data)
self.output.info("OK!")
""")
conandata = textwrap.dedent("""
sources:
all:
url: "http://*****:*****@user/testing")
client.run("create . {}".format(ref))
self.assertIn("OK!", client.out)
source_folder = client.cache.package_layout(ref).source()
downloaded_file = os.path.join(source_folder, "foo.txt")
self.assertEqual("foo", load(downloaded_file))
def calc_files_checksum(files):
return {
file_name: {
"md5": md5sum(path),
"sha1": sha1sum(path)
}
for file_name, path in files.items()
}
def conan_data_as_source_test(self):
tgz_path = tgz_with_contents({"foo.txt": "foo"})
md5_value = "2ef49b5a102db1abb775eaf1922d5662"
sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf"
sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338"
self.assertEqual(md5_value, md5sum(tgz_path))
self.assertEqual(sha1_value, sha1sum(tgz_path))
self.assertEqual(sha256_value, sha256sum(tgz_path))
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/myfile.tar.gz")
def get_file():
return static_file(os.path.basename(tgz_path),
root=os.path.dirname(tgz_path),
mimetype="")
thread.run_server()
client = TestClient()
conanfile = textwrap.dedent("""
from conans import ConanFile, tools
class Lib(ConanFile):
def source(self):
data = self.conan_data["sources"]["all"]
tools.get(**data)
self.output.info("OK!")
""")
conandata = textwrap.dedent("""
sources:
all:
url: "http://*****:*****@user/testing")
client.run("create . {}".format(ref))
self.assertIn("OK!", client.out)
source_folder = client.cache.package_layout(ref).source()
downloaded_file = os.path.join(source_folder, "foo.txt")
self.assertEqual("foo", load(downloaded_file))
def upload(self, url, abs_path, auth=None, dedup=False):
if dedup:
headers = {"X-Checksum-Deploy": "true",
"X-Checksum-Sha1": sha1sum(abs_path)}
response = self.requester.put(url, data="", verify=self.verify, headers=headers,
auth=auth)
if response.status_code != 404:
return response
self.output.info("")
# Actual transfer of the real content
it = load_in_chunks(abs_path, self.chunk_size)
# Now it is a chunked read file
file_size = os.stat(abs_path).st_size
it = upload_with_progress(file_size, it, self.chunk_size, self.output)
# Now it will print progress in each iteration
iterable_to_file = IterableToFileAdapter(it, file_size)
# Now it is prepared to work with request
ret = self.requester.put(url, data=iterable_to_file, verify=self.verify, headers=None, auth=auth)
return ret
def upload(self,
url,
abs_path,
auth=None,
dedup=False,
retry=None,
retry_wait=None,
headers=None,
display_name=None):
retry = retry if retry is not None else self._config.retry
retry = retry if retry is not None else 1
retry_wait = retry_wait if retry_wait is not None else self._config.retry_wait
retry_wait = retry_wait if retry_wait is not None else 5
# Send always the header with the Sha1
headers = copy(headers) or {}
headers["X-Checksum-Sha1"] = sha1sum(abs_path)
if dedup:
response = self._dedup(url, headers, auth)
if response:
return response
for counter in range(retry + 1):
try:
return self._upload_file(url, abs_path, headers, auth,
display_name)
except (NotFoundException, ForbiddenException,
AuthenticationException, RequestErrorException):
raise
except ConanException as exc:
if counter == retry:
raise
else:
if self._output:
self._output.error(exc)
self._output.info("Waiting %d seconds to retry..." %
retry_wait)
time.sleep(retry_wait)
def _file_document(name, path):
return {"name": name, "path": path, "md5": md5sum(path), "sha1": sha1sum(path)}
