def get_gunzip_test(self): # Create a tar file to be downloaded from server tmp = temp_folder() filepath = os.path.join(tmp, "test.txt.gz") import gzip with gzip.open(filepath, "wb") as f: f.write(b"hello world zipped!") thread = StoppableThreadBottle() @thread.server.get("/test.txt.gz") def get_file(): return static_file(os.path.basename(filepath), root=os.path.dirname(filepath), mimetype="application/octet-stream") thread.run_server() out = TestBufferConanOutput() with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests, output=out) self.assertTrue(os.path.exists("test.txt")) self.assertEqual(load("test.txt"), "hello world zipped!") with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests, output=out, destination="myfile.doc") self.assertTrue(os.path.exists("myfile.doc")) self.assertEqual(load("myfile.doc"), "hello world zipped!") with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests, output=out, destination="mytemp/myfile.txt") self.assertTrue(os.path.exists("mytemp/myfile.txt")) self.assertEqual(load("mytemp/myfile.txt"), "hello world zipped!") thread.stop()
def test_skip_toolset(self): settings = MockSettings({"build_type": "Debug", "compiler": "Visual Studio", "compiler.version": "15", "arch": "x86_64"}) class Runner(object): def __init__(self): self.commands = [] def __call__(self, *args, **kwargs): self.commands.append(args[0]) with chdir(tools.mkdir_tmp()): runner = Runner() conanfile = MockConanfile(settings, runner=runner) msbuild = MSBuild(conanfile) msbuild.build("myproject", toolset=False) self.assertEqual(len(runner.commands), 1) self.assertNotIn("PlatformToolset", runner.commands[0]) runner = Runner() conanfile = MockConanfile(settings, runner=runner) msbuild = MSBuild(conanfile) msbuild.build("myproject", toolset="mytoolset") self.assertEqual(len(runner.commands), 1) self.assertIn('/p:PlatformToolset="mytoolset"', runner.commands[0])
def get_filename_download_test(self): # Create a tar file to be downloaded from server with tools.chdir(tools.mkdir_tmp()): import tarfile tar_file = tarfile.open("sample.tar.gz", "w:gz") mkdir("test_folder") tar_file.add(os.path.abspath("test_folder"), "test_folder") tar_file.close() file_path = os.path.abspath("sample.tar.gz") assert(os.path.exists(file_path)) # Instance stoppable thread server and add endpoints thread = StoppableThreadBottle() @thread.server.get("/this_is_not_the_file_name") def get_file(): return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) @thread.server.get("/") def get_file2(): self.assertEqual(request.query["file"], "1") return static_file(os.path.basename(file_path), root=os.path.dirname(file_path)) @thread.server.get("/error_url") def error_url(): from bottle import response response.status = 500 return 'This always fail' thread.run_server() out = TestBufferConanOutput() # Test: File name cannot be deduced from '?file=1' with six.assertRaisesRegex(self, ConanException, "Cannot deduce file name form url. Use 'filename' parameter."): tools.get("http://localhost:%s/?file=1" % thread.port, output=out) # Test: Works with filename parameter instead of '?file=1' with tools.chdir(tools.mkdir_tmp()): tools.get("http://localhost:%s/?file=1" % thread.port, filename="sample.tar.gz", requester=requests, output=out) self.assertTrue(os.path.exists("test_folder")) # Test: Use a different endpoint but still not the filename one with tools.chdir(tools.mkdir_tmp()): from zipfile import BadZipfile with self.assertRaises(BadZipfile): tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port, requester=requests, output=out) tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port, filename="sample.tar.gz", requester=requests, output=out) self.assertTrue(os.path.exists("test_folder")) thread.stop() with six.assertRaisesRegex(self, ConanException, "Error"): tools.get("http://localhost:%s/error_url" % thread.port, filename="fake_sample.tar.gz", requester=requests, output=out, verify=False, retry=2, retry_wait=0) # Not found error self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2)
def download_retries_test(self): http_server = StoppableThreadBottle() with tools.chdir(tools.mkdir_tmp()): with open("manual.html", "w") as fmanual: fmanual.write("this is some content") manual_file = os.path.abspath("manual.html") from bottle import auth_basic @http_server.server.get("/manual.html") def get_manual(): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) def check_auth(user, password): # Check user/password here return user == "user" and password == "passwd" @http_server.server.get('/basic-auth/<user>/<password>') @auth_basic(check_auth) def get_manual_auth(user, password): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) http_server.run_server() out = TestBufferConanOutput() # Connection error # Default behaviour with six.assertRaisesRegex(self, ConanException, "Error downloading"): tools.download("http://fakeurl3.es/nonexists", os.path.join(temp_folder(), "file.txt"), out=out, requester=requests) self.assertEqual(str(out).count("Waiting 5 seconds to retry..."), 1) # Retry arguments override defaults with six.assertRaisesRegex(self, ConanException, "Error downloading"): tools.download("http://fakeurl3.es/nonexists", os.path.join(temp_folder(), "file.txt"), out=out, requester=requests, retry=2, retry_wait=1) self.assertEqual(str(out).count("Waiting 1 seconds to retry..."), 2) # Retry default values from the config class MockRequester(object): retry = 2 retry_wait = 0 def get(self, *args, **kwargs): return requests.get(*args, **kwargs) with six.assertRaisesRegex(self, ConanException, "Error downloading"): tools.download("http://fakeurl3.es/nonexists", os.path.join(temp_folder(), "file.txt"), out=out, requester=MockRequester()) self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2) # Not found error with six.assertRaisesRegex(self, NotFoundException, "Not found: "): tools.download("http://google.es/FILE_NOT_FOUND", os.path.join(temp_folder(), "README.txt"), out=out, requester=requests, retry=2, retry_wait=0) # And OK dest = os.path.join(temp_folder(), "manual.html") tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=3, retry_wait=0, requester=requests) self.assertTrue(os.path.exists(dest)) content = load(dest) # overwrite = False with self.assertRaises(ConanException): tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=False, requester=requests) # overwrite = True tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=True, requester=requests) self.assertTrue(os.path.exists(dest)) content_new = load(dest) self.assertEqual(content, content_new) # Not authorized with self.assertRaises(ConanException): tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, overwrite=True, requester=requests, out=out) # Authorized tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, auth=("user", "passwd"), overwrite=True, requester=requests, out=out) # Authorized using headers tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="}, overwrite=True, requester=requests, out=out) http_server.stop()
def test_get_mirror(self, _): """ tools.get must supports a list of URLs. However, only one must be downloaded. """ class MockRequester(object): def __init__(self): self.count = 0 self.fail_first = False self.fail_all = False def get(self, *args, **kwargs): self.count += 1 resp = Response() resp._content = b'{"results": []}' resp.headers = {"Content-Type": "application/json"} resp.status_code = 200 if (self.fail_first and self.count == 1) or self.fail_all: resp.status_code = 408 return resp file = "test.txt.gz" out = TestBufferConanOutput() urls = [ "http://localhost:{}/{}".format(8000 + i, file) for i in range(3) ] # Only the first file must be downloaded with tools.chdir(tools.mkdir_tmp()): requester = MockRequester() tools.get(urls, requester=requester, output=out, retry=0, retry_wait=0) self.assertEqual(1, requester.count) # Fail the first, download only the second with tools.chdir(tools.mkdir_tmp()): requester = MockRequester() requester.fail_first = True tools.get(urls, requester=requester, output=out, retry=0, retry_wait=0) self.assertEqual(2, requester.count) self.assertIn( "WARN: Could not download from the URL {}: Error 408 downloading file {}." " Trying another mirror.".format(urls[0], urls[0]), out) # Fail all downloads with tools.chdir(tools.mkdir_tmp()): requester = MockRequester() requester.fail_all = True with self.assertRaises(ConanException) as error: tools.get(urls, requester=requester, output=out, retry=0, retry_wait=0) self.assertEqual(3, requester.count) self.assertIn("All downloads from (3) URLs have failed.", str(error.exception))
def test_download_retries(self): http_server = StoppableThreadBottle() with tools.chdir(tools.mkdir_tmp()): with open("manual.html", "w") as fmanual: fmanual.write("this is some content") manual_file = os.path.abspath("manual.html") from bottle import auth_basic @http_server.server.get("/manual.html") def get_manual(): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) def check_auth(user, password): # Check user/password here return user == "user" and password == "passwd" @http_server.server.get('/basic-auth/<user>/<password>') @auth_basic(check_auth) def get_manual_auth(user, password): return static_file(os.path.basename(manual_file), os.path.dirname(manual_file)) http_server.run_server() out = TestBufferConanOutput() dest = os.path.join(temp_folder(), "manual.html") tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=3, retry_wait=0, requester=requests) self.assertTrue(os.path.exists(dest)) content = load(dest) # overwrite = False with self.assertRaises(ConanException): tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=False, requester=requests) # overwrite = True tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2, retry_wait=0, overwrite=True, requester=requests) self.assertTrue(os.path.exists(dest)) content_new = load(dest) self.assertEqual(content, content_new) # Not authorized with self.assertRaises(ConanException): tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, overwrite=True, requester=requests, out=out, retry=0, retry_wait=0) # Authorized tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, auth=("user", "passwd"), overwrite=True, requester=requests, out=out, retry=0, retry_wait=0) # Authorized using headers tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest, headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="}, overwrite=True, requester=requests, out=out, retry=0, retry_wait=0) http_server.stop()