def test_download(self): http_server = StoppableThreadBottle() file_path = os.path.join(temp_folder(), "myfile.txt") save(file_path, "some content") @http_server.server.get("/myfile.txt") def get_file(): return static_file(os.path.basename(file_path), os.path.dirname(file_path)) http_server.run_server() profile = textwrap.dedent("""\ [conf] tools.files.download:retry=1 tools.files.download:retry_wait=0 """) conanfile = textwrap.dedent(""" import os from conans import ConanFile from conan.tools.files import download class Pkg(ConanFile): name = "mypkg" version = "1.0" def source(self): download(self, "http://localhost:{}/myfile.txt", "myfile.txt") assert os.path.exists("myfile.txt") """.format(http_server.port)) client = TestClient() client.save({"conanfile.py": conanfile}) client.save({"profile": profile}) client.run("create . -pr=profile")
def test_conan_data_as_source_newtools(self): tgz_path = tgz_with_contents({"foo.txt": "foo"}) if sys.version_info.major == 3 and sys.version_info.minor >= 9: # Python 3.9 changed the tar algorithm. Conan tgz will have different checksums # https://github.com/conan-io/conan/issues/8020 md5_value = "7ebdc5ed79b7b72f3a6010da3671ae05" sha1_value = "862c1b58de1dfadaad3206b453b4de731c1751af" sha256_value = "25200fc2bd7f430358cd7a7c5ce4a84396e8ec68a1e9d8880994b1236f214972" else: md5_value = "2ef49b5a102db1abb775eaf1922d5662" sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf" sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338" self.assertEqual(md5_value, md5sum(tgz_path)) self.assertEqual(sha1_value, sha1sum(tgz_path)) self.assertEqual(sha256_value, sha256sum(tgz_path)) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/myfile.tar.gz") def get_file(): return static_file(os.path.basename(tgz_path), root=os.path.dirname(tgz_path), mimetype="") thread.run_server() client = TestClient() conanfile = textwrap.dedent(""" from conans import ConanFile from conan.tools.files import get class Lib(ConanFile): def source(self): data = self.conan_data["sources"]["all"] get(self, **data) self.output.info("OK!") """) conandata = textwrap.dedent(""" sources: all: url: "http://*****:*****@user/testing") client.run("create . {}".format(ref)) self.assertIn("OK!", client.out) source_folder = client.cache.package_layout(ref).source() downloaded_file = os.path.join(source_folder, "foo.txt") self.assertEqual("foo", load(downloaded_file))
def conan_data_as_source_test(self): tgz_path = tgz_with_contents({"foo.txt": "foo"}) md5_value = "2ef49b5a102db1abb775eaf1922d5662" sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf" sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338" self.assertEqual(md5_value, md5sum(tgz_path)) self.assertEqual(sha1_value, sha1sum(tgz_path)) self.assertEqual(sha256_value, sha256sum(tgz_path)) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/myfile.tar.gz") def get_file(): return static_file(os.path.basename(tgz_path), root=os.path.dirname(tgz_path), mimetype="") thread.run_server() client = TestClient() conanfile = textwrap.dedent(""" from conans import ConanFile, tools class Lib(ConanFile): def source(self): data = self.conan_data["sources"]["all"] tools.get(**data) self.output.info("OK!") """) conandata = textwrap.dedent(""" sources: all: url: "http://*****:*****@user/testing") client.run("create . {}".format(ref)) self.assertIn("OK!", client.out) source_folder = client.cache.package_layout(ref).source() downloaded_file = os.path.join(source_folder, "foo.txt") self.assertEqual("foo", load(downloaded_file))
def get_gunzip_test(self): # Create a tar file to be downloaded from server tmp = temp_folder() filepath = os.path.join(tmp, "test.txt.gz") import gzip with gzip.open(filepath, "wb") as f: f.write(b"hello world zipped!") thread = StoppableThreadBottle() @thread.server.get("/test.txt.gz") def get_file(): return static_file(os.path.basename(filepath), root=os.path.dirname(filepath), mimetype="application/octet-stream") thread.run_server() out = TestBufferConanOutput() with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests, output=out) self.assertTrue(os.path.exists("test.txt")) self.assertEqual(load("test.txt"), "hello world zipped!") with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests, output=out, destination="myfile.doc") self.assertTrue(os.path.exists("myfile.doc")) self.assertEqual(load("myfile.doc"), "hello world zipped!") with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests, output=out, destination="mytemp/myfile.txt") self.assertTrue(os.path.exists("mytemp/myfile.txt")) self.assertEqual(load("mytemp/myfile.txt"), "hello world zipped!") thread.stop()
def bottle_server(): http_server = StoppableThreadBottle() @http_server.server.get('/forbidden') def get_forbidden(): return HTTPError(403, "Access denied.") folder = temp_folder() manual_file = os.path.join(folder, "manual.html") save(manual_file, "this is some content") @http_server.server.get("/manual.html") def get_manual(): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) def check_auth(user, password): # Check user/password here return user == "user" and password == "passwd" @http_server.server.get('/basic-auth/<user>/<password>') @auth_basic(check_auth) def get_manual_auth(user, password): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) @http_server.server.get("/error_url") def error_url(): from bottle import response response.status = 500 return 'This always fail' http_server.run_server() yield http_server http_server.stop()
def conan_data_as_source_test(self): tgz_path = tgz_with_contents({"foo.txt": "bar"}) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/myfile.tar.gz") def get_file(): return static_file(os.path.basename(tgz_path), root=os.path.dirname(tgz_path)) thread.run_server() client = TestClient() conanfile = """ from conans import ConanFile, tools class Lib(ConanFile): def source(self): tools.get(**self.conan_data["sources"]["all"]) self.output.info("OK!") """ client.save({"conanfile.py": conanfile, "conandata.yml": """ sources: all: url: "http://*****:*****@user/testing") client.run("create . {}".format(ref)) self.assertIn("OK!", client.out) source_folder = client.cache.package_layout(ref).source() self.assertTrue(os.path.exists(os.path.join(source_folder, "foo.txt")))
def test_config_install_requester(self): # https://github.com/conan-io/conan/issues/4169 http_server = StoppableThreadBottle() path = self._create_zip() from bottle import static_file @http_server.server.get("/myconfig.zip") def get_zip(): return static_file(os.path.basename(path), os.path.dirname(path)) http_server.run_server() self.client.run("config install http://localhost:%s/myconfig.zip" % http_server.port) self.assertIn("Unzipping", self.client.out) http_server.stop()
def bottle_server_zip(): http_server = StoppableThreadBottle() tmp_folder = temp_folder() file_path = os.path.join(tmp_folder, "sample.tar.gz") test_folder = os.path.join(tmp_folder, "test_folder") zipped_file = os.path.join(test_folder, "myfile.txt") save(zipped_file, "myfile contents!") import tarfile tar_file = tarfile.open(file_path, "w:gz") tar_file.add(test_folder, "test_folder") tar_file.add(zipped_file, "test_folder/myfile.txt") tar_file.close() assert (os.path.exists(file_path)) @http_server.server.get("/this_is_not_the_file_name") def get_file(): return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) @http_server.server.get("/") def get_file2(): return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) @http_server.server.get("/sample.tgz") def get_file3(): return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) tmp = temp_folder() filepath = os.path.join(tmp, "test.txt.gz") import gzip with gzip.open(filepath, "wb") as f: f.write(b"hello world zipped!") @http_server.server.get("/test.txt.gz") def get_file(): return static_file(os.path.basename(filepath), root=os.path.dirname(filepath), mimetype="application/octet-stream") http_server.run_server() yield http_server http_server.stop()
def download_unathorized_test(self, mock_config): http_server = StoppableThreadBottle() mock_config.return_value = ConfigMock() @http_server.server.get('/forbidden') def get_forbidden(): return HTTPError(403, "Access denied.") http_server.run_server() out = TestBufferConanOutput() dest = os.path.join(temp_folder(), "manual.html") # Not authorized with six.assertRaisesRegex(self, AuthenticationException, "403"): tools.download("http://localhost:%s/forbidden" % http_server.port, dest, requester=requests, out=out) http_server.stop()
def test_user_downloads_cached(self): http_server = StoppableThreadBottle() file_path = os.path.join(temp_folder(), "myfile.txt") save(file_path, "some content") file_path_query = os.path.join(temp_folder(), "myfile2.txt") save(file_path_query, "some query") @http_server.server.get("/myfile.txt") def get_file(): f = file_path_query if request.query else file_path return static_file(os.path.basename(f), os.path.dirname(f)) http_server.run_server() client = TestClient() cache_folder = temp_folder() client.run('config set storage.download_cache="%s"' % cache_folder) # badchecksums are not cached conanfile = textwrap.dedent(""" from conans import ConanFile, tools class Pkg(ConanFile): def source(self): tools.download("http://localhost:%s/myfile.txt", "myfile.txt", md5="kk") """ % http_server.port) client.save({"conanfile.py": conanfile}) client.run("source .", assert_error=True) self.assertIn("ConanException: md5 signature failed for", client.out) self.assertIn("Provided signature: kk", client.out) self.assertIn("Computed signature: 9893532233caff98cd083a116b013c0b", client.out) self.assertEqual(1, len(os.listdir(cache_folder))) # Nothing was cached # This is the right checksum conanfile = textwrap.dedent(""" from conans import ConanFile, tools class Pkg(ConanFile): def source(self): md5 = "9893532233caff98cd083a116b013c0b" md5_2 = "0dc8a17658b1c7cfa23657780742a353" tools.download("http://localhost:{0}/myfile.txt", "myfile.txt", md5=md5) tools.download("http://localhost:{0}/myfile.txt?q=2", "myfile2.txt", md5=md5_2) """).format(http_server.port) client.save({"conanfile.py": conanfile}) client.run("source .") local_path = os.path.join(client.current_folder, "myfile.txt") self.assertTrue(os.path.exists(local_path)) self.assertEqual("some content", client.load("myfile.txt")) local_path2 = os.path.join(client.current_folder, "myfile2.txt") self.assertTrue(os.path.exists(local_path2)) self.assertEqual("some query", client.load("myfile2.txt")) # 2 files cached, plus "locks" folder = 3 self.assertEqual(3, len(os.listdir(cache_folder))) # remove remote file os.remove(file_path) os.remove(local_path) os.remove(local_path2) # Will use the cached one client.run("source .") self.assertTrue(os.path.exists(local_path)) self.assertTrue(os.path.exists(local_path2)) self.assertEqual("some content", client.load("myfile.txt")) self.assertEqual("some query", client.load("myfile2.txt")) # disabling cache will make it fail os.remove(local_path) os.remove(local_path2) client.run("config rm storage.download_cache") client.run("source .", assert_error=True) self.assertIn("ERROR: conanfile.py: Error in source() method, line 7", client.out) self.assertIn("Not found: http://localhost", client.out)
def get_filename_download_test(self): # Create a tar file to be downloaded from server with tools.chdir(tools.mkdir_tmp()): import tarfile tar_file = tarfile.open("sample.tar.gz", "w:gz") mkdir("test_folder") tar_file.add(os.path.abspath("test_folder"), "test_folder") tar_file.close() file_path = os.path.abspath("sample.tar.gz") assert(os.path.exists(file_path)) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/this_is_not_the_file_name") def get_file(): return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) @thread.server.get("/") def get_file2(): self.assertEqual(request.query["file"], "1") return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) @thread.server.get("/error_url") def error_url(): from bottle import response response.status = 500 return 'This always fail' thread.run_server() out = TestBufferConanOutput() # Test: File name cannot be deduced from '?file=1' with six.assertRaisesRegex(self, ConanException, "Cannot deduce file name form url. Use 'filename' parameter."): tools.get("http://localhost:%s/?file=1" % thread.port, output=out) # Test: Works with filename parameter instead of '?file=1' with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/?file=1" % thread.port, filename="sample.tar.gz", requester=requests, output=out) self.assertTrue(os.path.exists("test_folder")) # Test: Use a different endpoint but still not the filename one with tools.chdir(tools.mkdir_tmp()): from zipfile import BadZipfile with self.assertRaises(BadZipfile): tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port, requester=requests, output=out) tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port, filename="sample.tar.gz", requester=requests, output=out) self.assertTrue(os.path.exists("test_folder")) thread.stop() with six.assertRaisesRegex(self, ConanException, "Error"): tools.get("http://localhost:%s/error_url" % thread.port, filename="fake_sample.tar.gz", requester=requests, output=out, verify=False, retry=2, retry_wait=0) # Not found error self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2)
def download_retries_test(self): http_server = StoppableThreadBottle() with tools.chdir(tools.mkdir_tmp()): with open("manual.html", "w") as fmanual: fmanual.write("this is some content") manual_file = os.path.abspath("manual.html") from bottle import auth_basic @http_server.server.get("/manual.html") def get_manual(): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) def check_auth(user, password): # Check user/password here return user == "user" and password == "passwd" @http_server.server.get('/basic-auth/<user>/<password>') @auth_basic(check_auth) def get_manual_auth(user, password): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) http_server.run_server() out = TestBufferConanOutput() # Connection error # Default behaviour with six.assertRaisesRegex(self, ConanException, "Error downloading"): tools.download("http://fakeurl3.es/nonexists", os.path.join(temp_folder(), "file.txt"), out=out, requester=requests) self.assertEqual(str(out).count("Waiting 5 seconds to retry..."), 1) # Retry arguments override defaults with six.assertRaisesRegex(self, ConanException, "Error downloading"): tools.download("http://fakeurl3.es/nonexists", os.path.join(temp_folder(), "file.txt"), out=out, requester=requests, retry=2, retry_wait=1) self.assertEqual(str(out).count("Waiting 1 seconds to retry..."), 2) # Retry default values from the config class MockRequester(object): retry = 2 retry_wait = 0 def get(self, *args, **kwargs): return requests.get(*args, **kwargs) with six.assertRaisesRegex(self, ConanException, "Error downloading"): tools.download("http://fakeurl3.es/nonexists", os.path.join(temp_folder(), "file.txt"), out=out, requester=MockRequester()) self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2) # Not found error with six.assertRaisesRegex(self, NotFoundException, "Not found: "): tools.download("http://google.es/FILE_NOT_FOUND", os.path.join(temp_folder(), "README.txt"), out=out, requester=requests, retry=2, retry_wait=0) # And OK dest = os.path.join(temp_folder(), "manual.html") tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=3, retry_wait=0, requester=requests) self.assertTrue(os.path.exists(dest)) content = load(dest) # overwrite = False with self.assertRaises(ConanException): tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=False, requester=requests) # overwrite = True tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=True, requester=requests) self.assertTrue(os.path.exists(dest)) content_new = load(dest) self.assertEqual(content, content_new) # Not authorized with self.assertRaises(ConanException): tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, overwrite=True, requester=requests, out=out) # Authorized tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, auth=("user", "passwd"), overwrite=True, requester=requests, out=out) # Authorized using headers tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="}, overwrite=True, requester=requests, out=out) http_server.stop()
def test_download_retries(self): http_server = StoppableThreadBottle() with tools.chdir(tools.mkdir_tmp()): with open("manual.html", "w") as fmanual: fmanual.write("this is some content") manual_file = os.path.abspath("manual.html") from bottle import auth_basic @http_server.server.get("/manual.html") def get_manual(): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) def check_auth(user, password): # Check user/password here return user == "user" and password == "passwd" @http_server.server.get('/basic-auth/<user>/<password>') @auth_basic(check_auth) def get_manual_auth(user, password): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) http_server.run_server() out = TestBufferConanOutput() dest = os.path.join(temp_folder(), "manual.html") tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=3, retry_wait=0, requester=requests) self.assertTrue(os.path.exists(dest)) content = load(dest) # overwrite = False with self.assertRaises(ConanException): tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=False, requester=requests) # overwrite = True tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=True, requester=requests) self.assertTrue(os.path.exists(dest)) content_new = load(dest) self.assertEqual(content, content_new) # Not authorized with self.assertRaises(ConanException): tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, overwrite=True, requester=requests, out=out, retry=0, retry_wait=0) # Authorized tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, auth=("user", "passwd"), overwrite=True, requester=requests, out=out, retry=0, retry_wait=0) # Authorized using headers tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="}, overwrite=True, requester=requests, out=out, retry=0, retry_wait=0) http_server.stop()