def get_gunzip_test(self):
# Create a tar file to be downloaded from server
tmp = temp_folder()
filepath = os.path.join(tmp, "test.txt.gz")
import gzip
with gzip.open(filepath, "wb") as f:
f.write(b"hello world zipped!")
thread = StoppableThreadBottle()
@thread.server.get("/test.txt.gz")
def get_file():
return static_file(os.path.basename(filepath), root=os.path.dirname(filepath),
mimetype="application/octet-stream")
thread.run_server()
out = TestBufferConanOutput()
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests,
output=out)
self.assertTrue(os.path.exists("test.txt"))
self.assertEqual(load("test.txt"), "hello world zipped!")
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests,
output=out, destination="myfile.doc")
self.assertTrue(os.path.exists("myfile.doc"))
self.assertEqual(load("myfile.doc"), "hello world zipped!")
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests,
output=out, destination="mytemp/myfile.txt")
self.assertTrue(os.path.exists("mytemp/myfile.txt"))
self.assertEqual(load("mytemp/myfile.txt"), "hello world zipped!")
thread.stop()
def test_skip_toolset(self):
settings = MockSettings({"build_type": "Debug",
"compiler": "Visual Studio",
"compiler.version": "15",
"arch": "x86_64"})
class Runner(object):
def __init__(self):
self.commands = []
def __call__(self, *args, **kwargs):
self.commands.append(args[0])
with chdir(tools.mkdir_tmp()):
runner = Runner()
conanfile = MockConanfile(settings, runner=runner)
msbuild = MSBuild(conanfile)
msbuild.build("myproject", toolset=False)
self.assertEqual(len(runner.commands), 1)
self.assertNotIn("PlatformToolset", runner.commands[0])
runner = Runner()
conanfile = MockConanfile(settings, runner=runner)
msbuild = MSBuild(conanfile)
msbuild.build("myproject", toolset="mytoolset")
self.assertEqual(len(runner.commands), 1)
self.assertIn('/p:PlatformToolset="mytoolset"', runner.commands[0])
def get_filename_download_test(self):
# Create a tar file to be downloaded from server
with tools.chdir(tools.mkdir_tmp()):
import tarfile
tar_file = tarfile.open("sample.tar.gz", "w:gz")
mkdir("test_folder")
tar_file.add(os.path.abspath("test_folder"), "test_folder")
tar_file.close()
file_path = os.path.abspath("sample.tar.gz")
assert(os.path.exists(file_path))
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/this_is_not_the_file_name")
def get_file():
return static_file(os.path.basename(file_path), root=os.path.dirname(file_path))
@thread.server.get("/")
def get_file2():
self.assertEqual(request.query["file"], "1")
return static_file(os.path.basename(file_path), root=os.path.dirname(file_path))
@thread.server.get("/error_url")
def error_url():
from bottle import response
response.status = 500
return 'This always fail'
thread.run_server()
out = TestBufferConanOutput()
# Test: File name cannot be deduced from '?file=1'
with six.assertRaisesRegex(self, ConanException,
"Cannot deduce file name form url. Use 'filename' parameter."):
tools.get("http://localhost:%s/?file=1" % thread.port, output=out)
# Test: Works with filename parameter instead of '?file=1'
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/?file=1" % thread.port, filename="sample.tar.gz",
requester=requests, output=out)
self.assertTrue(os.path.exists("test_folder"))
# Test: Use a different endpoint but still not the filename one
with tools.chdir(tools.mkdir_tmp()):
from zipfile import BadZipfile
with self.assertRaises(BadZipfile):
tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port,
requester=requests, output=out)
tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port,
filename="sample.tar.gz", requester=requests, output=out)
self.assertTrue(os.path.exists("test_folder"))
thread.stop()
with six.assertRaisesRegex(self, ConanException, "Error"):
tools.get("http://localhost:%s/error_url" % thread.port,
filename="fake_sample.tar.gz", requester=requests, output=out, verify=False,
retry=2, retry_wait=0)
# Not found error
self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2)
def download_retries_test(self):
http_server = StoppableThreadBottle()
with tools.chdir(tools.mkdir_tmp()):
with open("manual.html", "w") as fmanual:
fmanual.write("this is some content")
manual_file = os.path.abspath("manual.html")
from bottle import auth_basic
@http_server.server.get("/manual.html")
def get_manual():
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
def check_auth(user, password):
# Check user/password here
return user == "user" and password == "passwd"
@http_server.server.get('/basic-auth/<user>/<password>')
@auth_basic(check_auth)
def get_manual_auth(user, password):
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
http_server.run_server()
out = TestBufferConanOutput()
# Connection error
# Default behaviour
with six.assertRaisesRegex(self, ConanException, "Error downloading"):
tools.download("http://fakeurl3.es/nonexists",
os.path.join(temp_folder(), "file.txt"), out=out,
requester=requests)
self.assertEqual(str(out).count("Waiting 5 seconds to retry..."), 1)
# Retry arguments override defaults
with six.assertRaisesRegex(self, ConanException, "Error downloading"):
tools.download("http://fakeurl3.es/nonexists",
os.path.join(temp_folder(), "file.txt"), out=out,
requester=requests,
retry=2, retry_wait=1)
self.assertEqual(str(out).count("Waiting 1 seconds to retry..."), 2)
# Retry default values from the config
class MockRequester(object):
retry = 2
retry_wait = 0
def get(self, *args, **kwargs):
return requests.get(*args, **kwargs)
with six.assertRaisesRegex(self, ConanException, "Error downloading"):
tools.download("http://fakeurl3.es/nonexists",
os.path.join(temp_folder(), "file.txt"), out=out,
requester=MockRequester())
self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2)
# Not found error
with six.assertRaisesRegex(self, NotFoundException, "Not found: "):
tools.download("http://google.es/FILE_NOT_FOUND",
os.path.join(temp_folder(), "README.txt"), out=out,
requester=requests,
retry=2, retry_wait=0)
# And OK
dest = os.path.join(temp_folder(), "manual.html")
tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=3,
retry_wait=0, requester=requests)
self.assertTrue(os.path.exists(dest))
content = load(dest)
# overwrite = False
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out,
retry=2, retry_wait=0, overwrite=False, requester=requests)
# overwrite = True
tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2,
retry_wait=0, overwrite=True, requester=requests)
self.assertTrue(os.path.exists(dest))
content_new = load(dest)
self.assertEqual(content, content_new)
# Not authorized
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest,
overwrite=True, requester=requests, out=out)
# Authorized
tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest,
auth=("user", "passwd"), overwrite=True, requester=requests, out=out)
# Authorized using headers
tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest,
headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="}, overwrite=True,
requester=requests, out=out)
http_server.stop()
def test_get_mirror(self, _):
""" tools.get must supports a list of URLs. However, only one must be downloaded.
"""
class MockRequester(object):
def __init__(self):
self.count = 0
self.fail_first = False
self.fail_all = False
def get(self, *args, **kwargs):
self.count += 1
resp = Response()
resp._content = b'{"results": []}'
resp.headers = {"Content-Type": "application/json"}
resp.status_code = 200
if (self.fail_first and self.count == 1) or self.fail_all:
resp.status_code = 408
return resp
file = "test.txt.gz"
out = TestBufferConanOutput()
urls = [
"http://localhost:{}/{}".format(8000 + i, file) for i in range(3)
]
# Only the first file must be downloaded
with tools.chdir(tools.mkdir_tmp()):
requester = MockRequester()
tools.get(urls,
requester=requester,
output=out,
retry=0,
retry_wait=0)
self.assertEqual(1, requester.count)
# Fail the first, download only the second
with tools.chdir(tools.mkdir_tmp()):
requester = MockRequester()
requester.fail_first = True
tools.get(urls,
requester=requester,
output=out,
retry=0,
retry_wait=0)
self.assertEqual(2, requester.count)
self.assertIn(
"WARN: Could not download from the URL {}: Error 408 downloading file {}."
" Trying another mirror.".format(urls[0], urls[0]), out)
# Fail all downloads
with tools.chdir(tools.mkdir_tmp()):
requester = MockRequester()
requester.fail_all = True
with self.assertRaises(ConanException) as error:
tools.get(urls,
requester=requester,
output=out,
retry=0,
retry_wait=0)
self.assertEqual(3, requester.count)
self.assertIn("All downloads from (3) URLs have failed.",
str(error.exception))
def test_download_retries(self):
http_server = StoppableThreadBottle()
with tools.chdir(tools.mkdir_tmp()):
with open("manual.html", "w") as fmanual:
fmanual.write("this is some content")
manual_file = os.path.abspath("manual.html")
from bottle import auth_basic
@http_server.server.get("/manual.html")
def get_manual():
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
def check_auth(user, password):
# Check user/password here
return user == "user" and password == "passwd"
@http_server.server.get('/basic-auth/<user>/<password>')
@auth_basic(check_auth)
def get_manual_auth(user, password):
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
http_server.run_server()
out = TestBufferConanOutput()
dest = os.path.join(temp_folder(), "manual.html")
tools.download("http://localhost:%s/manual.html" % http_server.port,
dest,
out=out,
retry=3,
retry_wait=0,
requester=requests)
self.assertTrue(os.path.exists(dest))
content = load(dest)
# overwrite = False
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/manual.html" %
http_server.port,
dest,
out=out,
retry=2,
retry_wait=0,
overwrite=False,
requester=requests)
# overwrite = True
tools.download("http://localhost:%s/manual.html" % http_server.port,
dest,
out=out,
retry=2,
retry_wait=0,
overwrite=True,
requester=requests)
self.assertTrue(os.path.exists(dest))
content_new = load(dest)
self.assertEqual(content, content_new)
# Not authorized
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/basic-auth/user/passwd" %
http_server.port,
dest,
overwrite=True,
requester=requests,
out=out,
retry=0,
retry_wait=0)
# Authorized
tools.download("http://localhost:%s/basic-auth/user/passwd" %
http_server.port,
dest,
auth=("user", "passwd"),
overwrite=True,
requester=requests,
out=out,
retry=0,
retry_wait=0)
# Authorized using headers
tools.download("http://localhost:%s/basic-auth/user/passwd" %
http_server.port,
dest,
headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="},
overwrite=True,
requester=requests,
out=out,
retry=0,
retry_wait=0)
http_server.stop()
