def test_download(self):
http_server = StoppableThreadBottle()
file_path = os.path.join(temp_folder(), "myfile.txt")
save(file_path, "some content")
@http_server.server.get("/myfile.txt")
def get_file():
return static_file(os.path.basename(file_path),
os.path.dirname(file_path))
http_server.run_server()
profile = textwrap.dedent("""\
[conf]
tools.files.download:retry=1
tools.files.download:retry_wait=0
""")
conanfile = textwrap.dedent("""
import os
from conans import ConanFile
from conan.tools.files import download
class Pkg(ConanFile):
name = "mypkg"
version = "1.0"
def source(self):
download(self, "http://localhost:{}/myfile.txt", "myfile.txt")
assert os.path.exists("myfile.txt")
""".format(http_server.port))
client = TestClient()
client.save({"conanfile.py": conanfile})
client.save({"profile": profile})
client.run("create . -pr=profile")
def get_gunzip_test(self):
# Create a tar file to be downloaded from server
tmp = temp_folder()
filepath = os.path.join(tmp, "test.txt.gz")
import gzip
with gzip.open(filepath, "wb") as f:
f.write(b"hello world zipped!")
thread = StoppableThreadBottle()
@thread.server.get("/test.txt.gz")
def get_file():
return static_file(os.path.basename(filepath), root=os.path.dirname(filepath),
mimetype="application/octet-stream")
thread.run_server()
out = TestBufferConanOutput()
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests,
output=out)
self.assertTrue(os.path.exists("test.txt"))
self.assertEqual(load("test.txt"), "hello world zipped!")
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests,
output=out, destination="myfile.doc")
self.assertTrue(os.path.exists("myfile.doc"))
self.assertEqual(load("myfile.doc"), "hello world zipped!")
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/test.txt.gz" % thread.port, requester=requests,
output=out, destination="mytemp/myfile.txt")
self.assertTrue(os.path.exists("mytemp/myfile.txt"))
self.assertEqual(load("mytemp/myfile.txt"), "hello world zipped!")
thread.stop()
def bottle_server():
http_server = StoppableThreadBottle()
@http_server.server.get('/forbidden')
def get_forbidden():
return HTTPError(403, "Access denied.")
folder = temp_folder()
manual_file = os.path.join(folder, "manual.html")
save(manual_file, "this is some content")
@http_server.server.get("/manual.html")
def get_manual():
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
def check_auth(user, password):
# Check user/password here
return user == "user" and password == "passwd"
@http_server.server.get('/basic-auth/<user>/<password>')
@auth_basic(check_auth)
def get_manual_auth(user, password):
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
@http_server.server.get("/error_url")
def error_url():
from bottle import response
response.status = 500
return 'This always fail'
http_server.run_server()
yield http_server
http_server.stop()
def test_conan_data_as_source_newtools(self):
tgz_path = tgz_with_contents({"foo.txt": "foo"})
if sys.version_info.major == 3 and sys.version_info.minor >= 9:
# Python 3.9 changed the tar algorithm. Conan tgz will have different checksums
# https://github.com/conan-io/conan/issues/8020
md5_value = "7ebdc5ed79b7b72f3a6010da3671ae05"
sha1_value = "862c1b58de1dfadaad3206b453b4de731c1751af"
sha256_value = "25200fc2bd7f430358cd7a7c5ce4a84396e8ec68a1e9d8880994b1236f214972"
else:
md5_value = "2ef49b5a102db1abb775eaf1922d5662"
sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf"
sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338"
self.assertEqual(md5_value, md5sum(tgz_path))
self.assertEqual(sha1_value, sha1sum(tgz_path))
self.assertEqual(sha256_value, sha256sum(tgz_path))
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/myfile.tar.gz")
def get_file():
return static_file(os.path.basename(tgz_path),
root=os.path.dirname(tgz_path),
mimetype="")
thread.run_server()
client = TestClient()
conanfile = textwrap.dedent("""
from conans import ConanFile
from conan.tools.files import get
class Lib(ConanFile):
def source(self):
data = self.conan_data["sources"]["all"]
get(self, **data)
self.output.info("OK!")
""")
conandata = textwrap.dedent("""
sources:
all:
url: "http://*****:*****@user/testing")
client.run("create . {}".format(ref))
self.assertIn("OK!", client.out)
source_folder = client.cache.package_layout(ref).source()
downloaded_file = os.path.join(source_folder, "foo.txt")
self.assertEqual("foo", load(downloaded_file))
def conan_data_as_source_test(self):
tgz_path = tgz_with_contents({"foo.txt": "foo"})
md5_value = "2ef49b5a102db1abb775eaf1922d5662"
sha1_value = "18dbea2d9a97bb9e9948604a41976bba5b5940bf"
sha256_value = "9619013c1f7b83cca4bf3f336f8b4525a23d5463e0768599fe5339e02dd0a338"
self.assertEqual(md5_value, md5sum(tgz_path))
self.assertEqual(sha1_value, sha1sum(tgz_path))
self.assertEqual(sha256_value, sha256sum(tgz_path))
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/myfile.tar.gz")
def get_file():
return static_file(os.path.basename(tgz_path),
root=os.path.dirname(tgz_path),
mimetype="")
thread.run_server()
client = TestClient()
conanfile = textwrap.dedent("""
from conans import ConanFile, tools
class Lib(ConanFile):
def source(self):
data = self.conan_data["sources"]["all"]
tools.get(**data)
self.output.info("OK!")
""")
conandata = textwrap.dedent("""
sources:
all:
url: "http://*****:*****@user/testing")
client.run("create . {}".format(ref))
self.assertIn("OK!", client.out)
source_folder = client.cache.package_layout(ref).source()
downloaded_file = os.path.join(source_folder, "foo.txt")
self.assertEqual("foo", load(downloaded_file))
def test_config_install_requester(self):
# https://github.com/conan-io/conan/issues/4169
http_server = StoppableThreadBottle()
path = self._create_zip()
from bottle import static_file
@http_server.server.get("/myconfig.zip")
def get_zip():
return static_file(os.path.basename(path), os.path.dirname(path))
http_server.run_server()
self.client.run("config install http://localhost:%s/myconfig.zip" % http_server.port)
self.assertIn("Unzipping", self.client.out)
http_server.stop()
def bottle_server_zip():
http_server = StoppableThreadBottle()
tmp_folder = temp_folder()
file_path = os.path.join(tmp_folder, "sample.tar.gz")
test_folder = os.path.join(tmp_folder, "test_folder")
zipped_file = os.path.join(test_folder, "myfile.txt")
save(zipped_file, "myfile contents!")
import tarfile
tar_file = tarfile.open(file_path, "w:gz")
tar_file.add(test_folder, "test_folder")
tar_file.add(zipped_file, "test_folder/myfile.txt")
tar_file.close()
assert (os.path.exists(file_path))
@http_server.server.get("/this_is_not_the_file_name")
def get_file():
return static_file(os.path.basename(file_path),
root=os.path.dirname(file_path))
@http_server.server.get("/")
def get_file2():
return static_file(os.path.basename(file_path),
root=os.path.dirname(file_path))
@http_server.server.get("/sample.tgz")
def get_file3():
return static_file(os.path.basename(file_path),
root=os.path.dirname(file_path))
tmp = temp_folder()
filepath = os.path.join(tmp, "test.txt.gz")
import gzip
with gzip.open(filepath, "wb") as f:
f.write(b"hello world zipped!")
@http_server.server.get("/test.txt.gz")
def get_file():
return static_file(os.path.basename(filepath),
root=os.path.dirname(filepath),
mimetype="application/octet-stream")
http_server.run_server()
yield http_server
http_server.stop()
def download_unathorized_test(self, mock_config):
http_server = StoppableThreadBottle()
mock_config.return_value = ConfigMock()
@http_server.server.get('/forbidden')
def get_forbidden():
return HTTPError(403, "Access denied.")
http_server.run_server()
out = TestBufferConanOutput()
dest = os.path.join(temp_folder(), "manual.html")
# Not authorized
with six.assertRaisesRegex(self, AuthenticationException, "403"):
tools.download("http://localhost:%s/forbidden" % http_server.port, dest,
requester=requests, out=out)
http_server.stop()
def conan_data_as_source_test(self):
tgz_path = tgz_with_contents({"foo.txt": "bar"})
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/myfile.tar.gz")
def get_file():
return static_file(os.path.basename(tgz_path), root=os.path.dirname(tgz_path))
thread.run_server()
client = TestClient()
conanfile = """
from conans import ConanFile, tools
class Lib(ConanFile):
def source(self):
tools.get(**self.conan_data["sources"]["all"])
self.output.info("OK!")
"""
client.save({"conanfile.py": conanfile,
"conandata.yml": """
sources:
all:
url: "http://*****:*****@user/testing")
client.run("create . {}".format(ref))
self.assertIn("OK!", client.out)
source_folder = client.cache.package_layout(ref).source()
self.assertTrue(os.path.exists(os.path.join(source_folder, "foo.txt")))
def test_user_downloads_cached(self):
http_server = StoppableThreadBottle()
file_path = os.path.join(temp_folder(), "myfile.txt")
save(file_path, "some content")
file_path_query = os.path.join(temp_folder(), "myfile2.txt")
save(file_path_query, "some query")
@http_server.server.get("/myfile.txt")
def get_file():
f = file_path_query if request.query else file_path
return static_file(os.path.basename(f), os.path.dirname(f))
http_server.run_server()
client = TestClient()
cache_folder = temp_folder()
client.run('config set storage.download_cache="%s"' % cache_folder)
# badchecksums are not cached
conanfile = textwrap.dedent("""
from conans import ConanFile, tools
class Pkg(ConanFile):
def source(self):
tools.download("http://localhost:%s/myfile.txt", "myfile.txt", md5="kk")
""" % http_server.port)
client.save({"conanfile.py": conanfile})
client.run("source .", assert_error=True)
self.assertIn("ConanException: md5 signature failed for", client.out)
self.assertIn("Provided signature: kk", client.out)
self.assertIn("Computed signature: 9893532233caff98cd083a116b013c0b",
client.out)
self.assertEqual(1,
len(os.listdir(cache_folder))) # Nothing was cached
# This is the right checksum
conanfile = textwrap.dedent("""
from conans import ConanFile, tools
class Pkg(ConanFile):
def source(self):
md5 = "9893532233caff98cd083a116b013c0b"
md5_2 = "0dc8a17658b1c7cfa23657780742a353"
tools.download("http://localhost:{0}/myfile.txt", "myfile.txt", md5=md5)
tools.download("http://localhost:{0}/myfile.txt?q=2", "myfile2.txt", md5=md5_2)
""").format(http_server.port)
client.save({"conanfile.py": conanfile})
client.run("source .")
local_path = os.path.join(client.current_folder, "myfile.txt")
self.assertTrue(os.path.exists(local_path))
self.assertEqual("some content", client.load("myfile.txt"))
local_path2 = os.path.join(client.current_folder, "myfile2.txt")
self.assertTrue(os.path.exists(local_path2))
self.assertEqual("some query", client.load("myfile2.txt"))
# 2 files cached, plus "locks" folder = 3
self.assertEqual(3, len(os.listdir(cache_folder)))
# remove remote file
os.remove(file_path)
os.remove(local_path)
os.remove(local_path2)
# Will use the cached one
client.run("source .")
self.assertTrue(os.path.exists(local_path))
self.assertTrue(os.path.exists(local_path2))
self.assertEqual("some content", client.load("myfile.txt"))
self.assertEqual("some query", client.load("myfile2.txt"))
# disabling cache will make it fail
os.remove(local_path)
os.remove(local_path2)
client.run("config rm storage.download_cache")
client.run("source .", assert_error=True)
self.assertIn("ERROR: conanfile.py: Error in source() method, line 7",
client.out)
self.assertIn("Not found: http://localhost", client.out)
def get_filename_download_test(self):
# Create a tar file to be downloaded from server
with tools.chdir(tools.mkdir_tmp()):
import tarfile
tar_file = tarfile.open("sample.tar.gz", "w:gz")
mkdir("test_folder")
tar_file.add(os.path.abspath("test_folder"), "test_folder")
tar_file.close()
file_path = os.path.abspath("sample.tar.gz")
assert(os.path.exists(file_path))
# Instance stoppable thread server and add endpoints
thread = StoppableThreadBottle()
@thread.server.get("/this_is_not_the_file_name")
def get_file():
return static_file(os.path.basename(file_path), root=os.path.dirname(file_path))
@thread.server.get("/")
def get_file2():
self.assertEqual(request.query["file"], "1")
return static_file(os.path.basename(file_path), root=os.path.dirname(file_path))
@thread.server.get("/error_url")
def error_url():
from bottle import response
response.status = 500
return 'This always fail'
thread.run_server()
out = TestBufferConanOutput()
# Test: File name cannot be deduced from '?file=1'
with six.assertRaisesRegex(self, ConanException,
"Cannot deduce file name form url. Use 'filename' parameter."):
tools.get("http://localhost:%s/?file=1" % thread.port, output=out)
# Test: Works with filename parameter instead of '?file=1'
with tools.chdir(tools.mkdir_tmp()):
tools.get("http://localhost:%s/?file=1" % thread.port, filename="sample.tar.gz",
requester=requests, output=out)
self.assertTrue(os.path.exists("test_folder"))
# Test: Use a different endpoint but still not the filename one
with tools.chdir(tools.mkdir_tmp()):
from zipfile import BadZipfile
with self.assertRaises(BadZipfile):
tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port,
requester=requests, output=out)
tools.get("http://localhost:%s/this_is_not_the_file_name" % thread.port,
filename="sample.tar.gz", requester=requests, output=out)
self.assertTrue(os.path.exists("test_folder"))
thread.stop()
with six.assertRaisesRegex(self, ConanException, "Error"):
tools.get("http://localhost:%s/error_url" % thread.port,
filename="fake_sample.tar.gz", requester=requests, output=out, verify=False,
retry=2, retry_wait=0)
# Not found error
self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2)
def download_retries_test(self):
http_server = StoppableThreadBottle()
with tools.chdir(tools.mkdir_tmp()):
with open("manual.html", "w") as fmanual:
fmanual.write("this is some content")
manual_file = os.path.abspath("manual.html")
from bottle import auth_basic
@http_server.server.get("/manual.html")
def get_manual():
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
def check_auth(user, password):
# Check user/password here
return user == "user" and password == "passwd"
@http_server.server.get('/basic-auth/<user>/<password>')
@auth_basic(check_auth)
def get_manual_auth(user, password):
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
http_server.run_server()
out = TestBufferConanOutput()
# Connection error
# Default behaviour
with six.assertRaisesRegex(self, ConanException, "Error downloading"):
tools.download("http://fakeurl3.es/nonexists",
os.path.join(temp_folder(), "file.txt"), out=out,
requester=requests)
self.assertEqual(str(out).count("Waiting 5 seconds to retry..."), 1)
# Retry arguments override defaults
with six.assertRaisesRegex(self, ConanException, "Error downloading"):
tools.download("http://fakeurl3.es/nonexists",
os.path.join(temp_folder(), "file.txt"), out=out,
requester=requests,
retry=2, retry_wait=1)
self.assertEqual(str(out).count("Waiting 1 seconds to retry..."), 2)
# Retry default values from the config
class MockRequester(object):
retry = 2
retry_wait = 0
def get(self, *args, **kwargs):
return requests.get(*args, **kwargs)
with six.assertRaisesRegex(self, ConanException, "Error downloading"):
tools.download("http://fakeurl3.es/nonexists",
os.path.join(temp_folder(), "file.txt"), out=out,
requester=MockRequester())
self.assertEqual(str(out).count("Waiting 0 seconds to retry..."), 2)
# Not found error
with six.assertRaisesRegex(self, NotFoundException, "Not found: "):
tools.download("http://google.es/FILE_NOT_FOUND",
os.path.join(temp_folder(), "README.txt"), out=out,
requester=requests,
retry=2, retry_wait=0)
# And OK
dest = os.path.join(temp_folder(), "manual.html")
tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=3,
retry_wait=0, requester=requests)
self.assertTrue(os.path.exists(dest))
content = load(dest)
# overwrite = False
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out,
retry=2, retry_wait=0, overwrite=False, requester=requests)
# overwrite = True
tools.download("http://localhost:%s/manual.html" % http_server.port, dest, out=out, retry=2,
retry_wait=0, overwrite=True, requester=requests)
self.assertTrue(os.path.exists(dest))
content_new = load(dest)
self.assertEqual(content, content_new)
# Not authorized
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest,
overwrite=True, requester=requests, out=out)
# Authorized
tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest,
auth=("user", "passwd"), overwrite=True, requester=requests, out=out)
# Authorized using headers
tools.download("http://localhost:%s/basic-auth/user/passwd" % http_server.port, dest,
headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="}, overwrite=True,
requester=requests, out=out)
http_server.stop()
def test_download_retries(self):
http_server = StoppableThreadBottle()
with tools.chdir(tools.mkdir_tmp()):
with open("manual.html", "w") as fmanual:
fmanual.write("this is some content")
manual_file = os.path.abspath("manual.html")
from bottle import auth_basic
@http_server.server.get("/manual.html")
def get_manual():
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
def check_auth(user, password):
# Check user/password here
return user == "user" and password == "passwd"
@http_server.server.get('/basic-auth/<user>/<password>')
@auth_basic(check_auth)
def get_manual_auth(user, password):
return static_file(os.path.basename(manual_file),
os.path.dirname(manual_file))
http_server.run_server()
out = TestBufferConanOutput()
dest = os.path.join(temp_folder(), "manual.html")
tools.download("http://localhost:%s/manual.html" % http_server.port,
dest,
out=out,
retry=3,
retry_wait=0,
requester=requests)
self.assertTrue(os.path.exists(dest))
content = load(dest)
# overwrite = False
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/manual.html" %
http_server.port,
dest,
out=out,
retry=2,
retry_wait=0,
overwrite=False,
requester=requests)
# overwrite = True
tools.download("http://localhost:%s/manual.html" % http_server.port,
dest,
out=out,
retry=2,
retry_wait=0,
overwrite=True,
requester=requests)
self.assertTrue(os.path.exists(dest))
content_new = load(dest)
self.assertEqual(content, content_new)
# Not authorized
with self.assertRaises(ConanException):
tools.download("http://localhost:%s/basic-auth/user/passwd" %
http_server.port,
dest,
overwrite=True,
requester=requests,
out=out,
retry=0,
retry_wait=0)
# Authorized
tools.download("http://localhost:%s/basic-auth/user/passwd" %
http_server.port,
dest,
auth=("user", "passwd"),
overwrite=True,
requester=requests,
out=out,
retry=0,
retry_wait=0)
# Authorized using headers
tools.download("http://localhost:%s/basic-auth/user/passwd" %
http_server.port,
dest,
headers={"Authorization": "Basic dXNlcjpwYXNzd2Q="},
overwrite=True,
requester=requests,
out=out,
retry=0,
retry_wait=0)
http_server.stop()
