def upload(self, url, abs_path, auth=None, dedup=False): if dedup: headers = { "X-Checksum-Deploy": "true", "X-Checksum-Sha1": sha1sum(abs_path) } response = self.requester.put(url, data="", verify=self.verify, headers=headers, auth=auth) if response.status_code != 404: return response self.output.info("") # Actual transfer of the real content it = load_in_chunks(abs_path, self.chunk_size) # Now it is a chunked read file file_size = os.stat(abs_path).st_size it = upload_with_progress(file_size, it, self.chunk_size, self.output) # Now it will print progress in each iteration iterable_to_file = IterableToFileAdapter(it, file_size) # Now it is prepared to work with request ret = self.requester.put(url, data=iterable_to_file, verify=self.verify, headers=None, auth=auth) return ret
def upload(self, url, abs_path, auth=None, dedup=False, retry=1, retry_wait=0, headers=None): # Send always the header with the Sha1 headers = headers or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 401: raise AuthenticationException(response.content) if response.status_code == 403: if auth.token is None: raise AuthenticationException(response.content) raise ForbiddenException(response.content) if response.status_code == 201: # Artifactory returns 201 if the file is there return response self.output.info("") # Actual transfer of the real content it = load_in_chunks(abs_path, self.chunk_size) # Now it is a chunked read file file_size = os.stat(abs_path).st_size it = upload_with_progress(file_size, it, self.chunk_size, self.output) # Now it will print progress in each iteration iterable_to_file = IterableToFileAdapter(it, file_size) # Now it is prepared to work with request ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, data=iterable_to_file, headers=headers, auth=auth) return ret
def _file_document(name, path): return { "name": name, "path": path, "md5": md5sum(path), "sha1": sha1sum(path) }
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None): retry = retry if retry is not None else self.requester.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self.requester.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = copy(headers) or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: dedup_headers = {"X-Checksum-Deploy": "true"} if headers: dedup_headers.update(headers) response = self.requester.put(url, data="", verify=self.verify, headers=dedup_headers, auth=auth) if response.status_code == 400: raise RequestErrorException(response_to_str(response)) if response.status_code == 401: raise AuthenticationException(response_to_str(response)) if response.status_code == 403: if auth is None or auth.token is None: raise AuthenticationException(response_to_str(response)) raise ForbiddenException(response_to_str(response)) if response.status_code == 201: # Artifactory returns 201 if the file is there return response ret = call_with_retry(self.output, retry, retry_wait, self._upload_file, url, abs_path=abs_path, headers=headers, auth=auth) return ret
def test_conan_data_as_source_newtools(self): tgz_path = tgz_with_contents({"foo.txt": "foo"}) if sys.version_info.major == 3 and sys.version_info.minor >= 9: # Python 3.9 changed the tar algorithm. Conan tgz will have different checksums # https://github.com/conan-io/conan/issues/8020 md5_value = "7ebdc5ed79b7b72f3a6010da3671ae05" sha1_value = "862c1b58de1dfadaad3206b453b4de731c1751af" sha256_value = "25200fc2bd7f430358cd7a7c5ce4a84396e8ec68a1e9d8880994b1236f214972" else: md5_value = "2ef49b5a102db1abb775eaf1922d5662" sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf" sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338" self.assertEqual(md5_value, md5sum(tgz_path)) self.assertEqual(sha1_value, sha1sum(tgz_path)) self.assertEqual(sha256_value, sha256sum(tgz_path)) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/myfile.tar.gz") def get_file(): return static_file(os.path.basename(tgz_path), root=os.path.dirname(tgz_path), mimetype="") thread.run_server() client = TestClient() conanfile = textwrap.dedent(""" from conans import ConanFile from conan.tools.files import get class Lib(ConanFile): def source(self): data = self.conan_data["sources"]["all"] get(self, **data) self.output.info("OK!") """) conandata = textwrap.dedent(""" sources: all: url: "http://*****:*****@user/testing") client.run("create . {}".format(ref)) self.assertIn("OK!", client.out) source_folder = client.cache.package_layout(ref).source() downloaded_file = os.path.join(source_folder, "foo.txt") self.assertEqual("foo", load(downloaded_file))
def calc_files_checksum(files): return { file_name: { "md5": md5sum(path), "sha1": sha1sum(path) } for file_name, path in files.items() }
def conan_data_as_source_test(self): tgz_path = tgz_with_contents({"foo.txt": "foo"}) md5_value = "2ef49b5a102db1abb775eaf1922d5662" sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf" sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338" self.assertEqual(md5_value, md5sum(tgz_path)) self.assertEqual(sha1_value, sha1sum(tgz_path)) self.assertEqual(sha256_value, sha256sum(tgz_path)) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/myfile.tar.gz") def get_file(): return static_file(os.path.basename(tgz_path), root=os.path.dirname(tgz_path), mimetype="") thread.run_server() client = TestClient() conanfile = textwrap.dedent(""" from conans import ConanFile, tools class Lib(ConanFile): def source(self): data = self.conan_data["sources"]["all"] tools.get(**data) self.output.info("OK!") """) conandata = textwrap.dedent(""" sources: all: url: "http://*****:*****@user/testing") client.run("create . {}".format(ref)) self.assertIn("OK!", client.out) source_folder = client.cache.package_layout(ref).source() downloaded_file = os.path.join(source_folder, "foo.txt") self.assertEqual("foo", load(downloaded_file))
def upload(self, url, abs_path, auth=None, dedup=False): if dedup: headers = {"X-Checksum-Deploy": "true", "X-Checksum-Sha1": sha1sum(abs_path)} response = self.requester.put(url, data="", verify=self.verify, headers=headers, auth=auth) if response.status_code != 404: return response self.output.info("") # Actual transfer of the real content it = load_in_chunks(abs_path, self.chunk_size) # Now it is a chunked read file file_size = os.stat(abs_path).st_size it = upload_with_progress(file_size, it, self.chunk_size, self.output) # Now it will print progress in each iteration iterable_to_file = IterableToFileAdapter(it, file_size) # Now it is prepared to work with request ret = self.requester.put(url, data=iterable_to_file, verify=self.verify, headers=None, auth=auth) return ret
def upload(self, url, abs_path, auth=None, dedup=False, retry=None, retry_wait=None, headers=None, display_name=None): retry = retry if retry is not None else self._config.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else self._config.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 # Send always the header with the Sha1 headers = copy(headers) or {} headers["X-Checksum-Sha1"] = sha1sum(abs_path) if dedup: response = self._dedup(url, headers, auth) if response: return response for counter in range(retry + 1): try: return self._upload_file(url, abs_path, headers, auth, display_name) except (NotFoundException, ForbiddenException, AuthenticationException, RequestErrorException): raise except ConanException as exc: if counter == retry: raise else: if self._output: self._output.error(exc) self._output.info("Waiting %d seconds to retry..." % retry_wait) time.sleep(retry_wait)
def _file_document(name, path): return {"name": name, "path": path, "md5": md5sum(path), "sha1": sha1sum(path)}