def test_error_package_xz(self):
server = TestServer()
ref = ConanFileReference.loads("Pkg/0.1@user/channel")
ref = ref.copy_with_rev(DEFAULT_REVISION_V1)
client = TestClient(servers={"default": server},
users={"default": [("lasote", "mypass")]})
server.server_store.update_last_revision(ref)
export = server.server_store.export(
ref) # *1 the path can't be known before upload a revision
conanfile = """from conans import ConanFile
class Pkg(ConanFile):
exports_sources = "*"
"""
save_files(export, {
"conanfile.py": conanfile,
"conanmanifest.txt": "1"
})
pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID,
DEFAULT_REVISION_V1)
server.server_store.update_last_package_revision(
pref.copy_with_revs(DEFAULT_REVISION_V1, DEFAULT_REVISION_V1))
package = server.server_store.package(pref)
save_files(
package, {
"conaninfo.txt": "#",
"conanmanifest.txt": "1",
"conan_package.txz": "#"
})
client.run("install Pkg/0.1@user/channel", assert_error=True)
self.assertIn(
"ERROR: This Conan version is not prepared to handle "
"'conan_package.txz' file format", client.out)
def _get_latest_pref(self, pref):
ref = self._get_latest_ref(pref.ref)
pref = PackageReference(ref, pref.id)
tmp = self._server_store.get_last_package_revision(pref)
if not tmp:
prev = DEFAULT_REVISION_V1
else:
prev = tmp.revision
return pref.copy_with_revs(ref.revision, prev)
def remove_packages(self, ref, package_ids):
""" Remove any packages specified by package_ids"""
self.check_credentials()
if ref.revision is None:
# Remove the packages from all the RREVs
revisions = self.get_recipe_revisions(ref)
refs = [ref.copy_with_rev(rev["revision"]) for rev in revisions]
else:
refs = [ref]
for ref in refs:
assert ref.revision is not None, "remove_packages needs RREV"
if not package_ids:
url = self.router.remove_all_packages(ref)
response = self.requester.delete(url,
auth=self.auth,
verify=self.verify_ssl,
headers=self.custom_headers)
if response.status_code == 404:
# Double check if it is a 404 because there are no packages
try:
package_search_url = self.router.search_packages(ref)
if not self.get_json(package_search_url):
return
except Exception as e:
logger.warning("Unexpected error searching {} packages"
" in remote {}: {}".format(
ref, self.remote_url, e))
if response.status_code != 200: # Error message is text
# To be able to access ret.text (ret.content are bytes)
response.charset = "utf-8"
raise get_exception_from_error(response.status_code)(
response.text)
else:
for pid in package_ids:
pref = PackageReference(ref, pid)
revisions = self.get_package_revisions(pref)
prefs = [
pref.copy_with_revs(ref.revision, rev["revision"])
for rev in revisions
]
for pref in prefs:
url = self.router.remove_package(pref)
response = self.requester.delete(
url,
auth=self.auth,
headers=self.custom_headers,
verify=self.verify_ssl)
if response.status_code == 404:
raise PackageNotFoundException(pref)
if response.status_code != 200: # Error message is text
# To be able to access ret.text (ret.content are bytes)
response.charset = "utf-8"
raise get_exception_from_error(
response.status_code)(response.text)
def export_pkg(self, ref, conanfile=GenConanfile(), args=None, assert_error=False):
if conanfile:
self.save({"conanfile.py": conanfile})
self.run("export-pkg . {} {} --json {}".format(ref.full_str(),
args or "", self.tmp_json_name),
assert_error=assert_error)
rrev = self.cache.package_layout(ref).recipe_revision()
data = json.loads(self.load(self.tmp_json_name))
if assert_error:
return None
package_id = data["installed"][0]["packages"][0]["id"]
package_ref = PackageReference(ref, package_id)
prev = self.cache.package_layout(ref.copy_clear_rev()).package_revision(package_ref)
return package_ref.copy_with_revs(rrev, prev)
def create(self, ref, conanfile=None, args=None, assert_error=False):
conanfile = str(conanfile) if conanfile else str(GenConanfile())
self.save({"conanfile.py": conanfile})
self.run("create . {} {} --json {}".format(ref.full_str(),
args or "", self.tmp_json_name),
assert_error=assert_error)
rrev = self.cache.package_layout(ref).recipe_revision()
json_path = os.path.join(self.current_folder, self.tmp_json_name)
data = json.loads(load(json_path))
if assert_error:
return None
package_id = data["installed"][0]["packages"][0]["id"]
package_ref = PackageReference(ref, package_id)
prev = self.cache.package_layout(ref.copy_clear_rev()).package_revision(package_ref)
return package_ref.copy_with_revs(rrev, prev)
def remove_packages(self, ref, package_ids=None):
""" Remove any packages specified by package_ids"""
self.check_credentials()
if ref.revision is None:
# Remove the packages from all the RREVs
revisions = self.get_recipe_revisions(ref)
refs = [ref.copy_with_rev(rev["revision"]) for rev in revisions]
else:
refs = [ref]
for ref in refs:
assert ref.revision is not None, "remove_packages needs RREV"
if not package_ids:
url = self.router.remove_all_packages(ref)
response = self.requester.delete(url,
auth=self.auth,
headers=self.custom_headers,
verify=self.verify_ssl)
if response.status_code != 200: # Error message is text
# To be able to access ret.text (ret.content are bytes)
response.charset = "utf-8"
raise get_exception_from_error(response.status_code)(
response.text)
else:
for pid in package_ids:
pref = PackageReference(ref, pid)
revisions = self.get_package_revisions(pref)
prefs = [
pref.copy_with_revs(ref.revision, rev["revision"])
for rev in revisions
]
for pref in prefs:
url = self.router.remove_package(pref)
response = self.requester.delete(
url,
auth=self.auth,
headers=self.custom_headers,
verify=self.verify_ssl)
if response.status_code == 404:
raise PackageNotFoundException(pref)
if response.status_code != 200: # Error message is text
# To be able to access ret.text (ret.content are bytes)
response.charset = "utf-8"
raise get_exception_from_error(
response.status_code)(response.text)
def p_ref_with_rev(self, pref):
if pref.revision and pref.ref.revision:
return pref
if not pref.ref.revision:
# Search the latest recipe revision with the requested package
pref = self.get_latest_package_reference(pref)
return pref
ref = self.ref_with_rev(pref.ref)
ret = PackageReference(ref, pref.id)
latest_p = self.get_last_package_revision(ret)
if not latest_p:
raise NotFoundException("Package not found: '%s'" % str(pref))
return ret.copy_with_revs(ref.revision, latest_p.revision)
def p_ref_with_rev(self, p_reference):
if p_reference.revision and p_reference.conan.revision:
return p_reference
if not p_reference.conan.revision:
# Search the latest recipe revision with the requested package
p_reference = self.get_latest_package_reference(p_reference)
return p_reference
reference = self.ref_with_rev(p_reference.conan)
ret = PackageReference(reference, p_reference.package_id)
latest_p = self.get_last_package_revision(ret)
if not latest_p:
raise NotFoundException("Package not found: '%s'" %
str(p_reference))
return ret.copy_with_revs(reference.revision, latest_p.revision)
def get_latest_package_reference(self, pref):
assert (isinstance(pref, PackageReference))
rev_file_path = self._recipe_revisions_file(pref.ref)
revs = self._get_revisions(rev_file_path)
if not revs:
raise NotFoundException("Recipe not found: '%s'" % str(pref.ref))
for rev in revs.items():
pref = PackageReference(pref.ref.copy_with_rev(rev.revision),
pref.id)
tmp = self.get_last_package_revision(pref)
if tmp:
pref = pref.copy_with_revs(rev.revision, tmp.revision)
try:
folder = self.package(pref)
if self._storage_adapter.path_exists(folder):
return pref
except NotFoundException:
pass
raise NotFoundException("Package not found: '%s'" % str(pref))
def upload_test(self):
client = TestClient(servers={"default": TestServer()},
users={"default": [("lasote", "mypass")]})
ref = ConanFileReference.loads("Hello0/0.1@lasote/stable#%s" %
DEFAULT_REVISION_V1)
files = cpp_hello_conan_files("Hello0", "0.1", build=False)
files["to_be_deleted.txt"] = "delete me"
files["to_be_deleted2.txt"] = "delete me2"
remote_paths = client.servers["default"].server_store
client.save(files)
client.run("export . lasote/stable")
# Upload conan file
client.run("upload %s" % str(ref))
# Verify the files are there
if client.cache.config.revisions_enabled:
rev = client.cache.package_layout(ref).recipe_revision()
ref = ref.copy_with_rev(rev)
server_conan_path = remote_paths.export(ref)
self.assertTrue(
os.path.exists(os.path.join(server_conan_path, EXPORT_TGZ_NAME)))
tmp = temp_folder()
untargz(os.path.join(server_conan_path, EXPORT_TGZ_NAME), tmp)
self.assertTrue(load(os.path.join(tmp, "to_be_deleted.txt")),
"delete me")
self.assertTrue(load(os.path.join(tmp, "to_be_deleted2.txt")),
"delete me2")
# Now delete local files export and upload and check that they are not in server
os.remove(os.path.join(client.current_folder, "to_be_deleted.txt"))
client.run("export . lasote/stable")
client.run("upload %s" % str(ref))
if client.cache.config.revisions_enabled:
rev = client.cache.package_layout(ref).recipe_revision()
ref = ref.copy_with_rev(rev)
server_conan_path = remote_paths.export(ref)
self.assertTrue(
os.path.exists(os.path.join(server_conan_path, EXPORT_TGZ_NAME)))
tmp = temp_folder()
untargz(os.path.join(server_conan_path, EXPORT_TGZ_NAME), tmp)
self.assertFalse(os.path.exists(os.path.join(tmp,
"to_be_deleted.txt")))
self.assertTrue(os.path.exists(os.path.join(tmp,
"to_be_deleted2.txt")))
# Now modify a file, and delete other, and put a new one.
files["to_be_deleted2.txt"] = "modified content"
files["new_file.lib"] = "new file"
del files["to_be_deleted.txt"]
client.save(files)
client.run("export . lasote/stable")
client.run("upload %s" % str(ref))
if client.cache.config.revisions_enabled:
rev = client.cache.package_layout(ref).recipe_revision()
ref = ref.copy_with_rev(rev)
server_conan_path = remote_paths.export(ref)
# Verify all is correct
self.assertTrue(
os.path.exists(os.path.join(server_conan_path, EXPORT_TGZ_NAME)))
tmp = temp_folder()
untargz(os.path.join(server_conan_path, EXPORT_TGZ_NAME), tmp)
self.assertTrue(load(os.path.join(tmp, "to_be_deleted2.txt")),
"modified content")
self.assertTrue(load(os.path.join(tmp, "new_file.lib")), "new file")
self.assertFalse(os.path.exists(os.path.join(tmp,
"to_be_deleted.txt")))
##########################
# Now try with the package
##########################
client.run("install %s --build missing" % str(ref))
# Upload package
package_ids = client.cache.package_layout(ref).conan_packages()
client.run("upload %s -p %s" % (str(ref), str(package_ids[0])))
# Check that package exists on server
pref = PackageReference(ref, str(package_ids[0]))
prev = remote_paths.get_last_package_revision(pref)
pref = pref.copy_with_revs(pref.ref.revision, prev.revision)
package_server_path = remote_paths.package(pref)
self.assertTrue(os.path.exists(package_server_path))
# Add a new file to package (artificially), upload again and check
pack_path = client.cache.package_layout(pref.ref).package(pref)
new_file_source_path = os.path.join(pack_path, "newlib.lib")
save(new_file_source_path, "newlib")
os.unlink(os.path.join(pack_path, PACKAGE_TGZ_NAME)) # Force new tgz
self._create_manifest(client, pref)
client.run("upload %s -p %s" % (str(ref), str(package_ids[0])))
folder = uncompress_packaged_files(remote_paths, pref)
remote_file_path = os.path.join(folder, "newlib.lib")
self.assertTrue(os.path.exists(remote_file_path))
# Now modify the file and check again
save(new_file_source_path, "othercontent")
self._create_manifest(client, pref)
client.run("upload %s -p %s" % (str(ref), str(package_ids[0])))
folder = uncompress_packaged_files(remote_paths, pref)
remote_file_path = os.path.join(folder, "newlib.lib")
self.assertTrue(os.path.exists(remote_file_path))
self.assertTrue(load(remote_file_path), "othercontent")
# Now delete the file and check again
os.remove(new_file_source_path)
self._create_manifest(client, pref)
os.unlink(os.path.join(pack_path, PACKAGE_TGZ_NAME)) # Force new tgz
client.run("upload %s -p %s" % (str(ref), str(package_ids[0])))
folder = uncompress_packaged_files(remote_paths, pref)
remote_file_path = os.path.join(folder, "newlib.lib")
# With revisions makes no sense because there is a new revision always that sources change
if not client.cache.config.revisions_enabled:
self.assertFalse(os.path.exists(remote_file_path))
self.assertNotEqual(remote_file_path, new_file_source_path)
class UploadTest(unittest.TestCase):
def _get_client(self, requester=None):
servers = {}
# All can write (for avoid authentication until we mock user_io)
self.test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")],
users={"lasote": "mypass"})
servers["default"] = self.test_server
return TestClient(servers=servers,
users={"default": [("lasote", "mypass")]},
requester_class=requester)
def setUp(self):
self.client = self._get_client()
self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" %
DEFAULT_REVISION_V1)
self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1)
reg_folder = self.client.cache.package_layout(self.ref).export()
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("ERROR: Recipe not found: '%s'" % str(self.ref),
self.client.out)
files = hello_source_files()
fake_metadata = PackageMetadata()
fake_metadata.recipe.revision = DEFAULT_REVISION_V1
fake_metadata.packages[self.pref.id].revision = DEFAULT_REVISION_V1
self.client.save({"metadata.json": fake_metadata.dumps()},
path=self.client.cache.package_layout(
self.ref).base_folder())
self.client.save(files, path=reg_folder)
self.client.save(
{
CONANFILE: myconan1,
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"
},
path=reg_folder)
mkdir(self.client.cache.package_layout(self.ref).export_sources())
manifest = FileTreeManifest.create(reg_folder)
manifest.time = '123123123'
manifest.save(reg_folder)
self.test_server.server_store.update_last_revision(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
package_folder = self.client.cache.package_layout(self.ref).package(
self.pref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"),
"//bin")
save(os.path.join(package_folder, CONANINFO),
"""[recipe_hash]\n%s""" % manifest.summary_hash)
FileTreeManifest.create(package_folder).save(package_folder)
self.test_server.server_store.update_last_package_revision(self.pref)
os.chmod(
os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin",
"executable")).st_mode | stat.S_IRWXU)
expected_manifest = FileTreeManifest.create(package_folder)
expected_manifest.save(package_folder)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def try_upload_bad_recipe_test(self):
files = hello_conan_files("Hello0", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
ref = ConanFileReference.loads("Hello0/1.2.1@frodo/stable")
os.unlink(
os.path.join(
self.client.cache.package_layout(ref).export(),
CONAN_MANIFEST))
with six.assertRaisesRegex(self, Exception, "Command failed"):
self.client.run("upload %s" % str(ref))
self.assertIn("Cannot upload corrupted recipe", self.client.out)
def upload_with_pattern_test(self):
for num in range(5):
files = hello_conan_files("Hello%s" % num, "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
self.client.run("upload Hello* --confirm")
for num in range(5):
self.assertIn("Uploading Hello%s/1.2.1@frodo/stable" % num,
self.client.out)
self.client.run("upload Hello0* --confirm")
self.assertIn("Uploading Hello0/1.2.1@frodo/stable", self.client.out)
self.assertIn("Recipe is up to date, upload skipped", self.client.out)
self.assertNotIn("Hello1", self.client.out)
self.assertNotIn("Hello2", self.client.out)
self.assertNotIn("Hello3", self.client.out)
def upload_error_test(self):
"""Cause an error in the transfer and see some message"""
# Check for the default behaviour
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm")
self.assertIn("Can't connect because of the evil mock", client.out)
self.assertIn("Waiting 5 seconds to retry...", client.out)
# This will fail in the first put file, so, as we need to
# upload 3 files (conanmanifest, conanfile and tgz) will do it with 2 retries
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry-wait=0")
self.assertIn("Can't connect because of the evil mock", client.out)
self.assertIn("Waiting 0 seconds to retry...", client.out)
# but not with 0
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry 0 --retry-wait=1",
assert_error=True)
self.assertNotIn("Waiting 1 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files: "
"conan_export.tgz. [Remote: default]", client.out)
# Try with broken connection even with 10 retries
client = self._get_client(TerribleConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry 10 --retry-wait=0",
assert_error=True)
self.assertIn("Waiting 0 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files",
client.out)
# For each file will fail the first time and will success in the second one
client = self._get_client(FailPairFilesUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("install Hello0/1.2.1@frodo/stable --build")
client.run("upload Hello* --confirm --retry 3 --retry-wait=0 --all")
self.assertEqual(str(client.out).count("ERROR: Pair file, error!"), 6)
def upload_error_with_config_test(self):
"""Cause an error in the transfer and see some message"""
# This will fail in the first put file, so, as we need to
# upload 3 files (conanmanifest, conanfile and tgz) will do it with 2 retries
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run('config set general.retry_wait=0')
client.run("upload Hello* --confirm")
self.assertIn("Can't connect because of the evil mock", client.out)
self.assertIn("Waiting 0 seconds to retry...", client.out)
# but not with 0
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run('config set general.retry=0')
client.run('config set general.retry_wait=1')
client.run("upload Hello* --confirm", assert_error=True)
self.assertNotIn("Waiting 1 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files: "
"conan_export.tgz. [Remote: default]", client.out)
# Try with broken connection even with 10 retries
client = self._get_client(TerribleConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run('config set general.retry=10')
client.run('config set general.retry_wait=0')
client.run("upload Hello* --confirm", assert_error=True)
self.assertIn("Waiting 0 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files",
client.out)
# For each file will fail the first time and will success in the second one
client = self._get_client(FailPairFilesUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("install Hello0/1.2.1@frodo/stable --build")
client.run('config set general.retry=3')
client.run('config set general.retry_wait=0')
client.run("upload Hello* --confirm --all")
self.assertEqual(str(client.out).count("ERROR: Pair file, error!"), 6)
def upload_parallel_error_test(self):
"""Cause an error in the parallel transfer and see some message"""
client = TestClient(requester_class=FailOnReferencesUploader,
default_server_user=True)
client.save({"conanfile.py": GenConanfile()})
client.run('user -p password -r default user')
for index in range(4):
client.run('create . lib{}/1.0@user/channel'.format(index))
client.run('upload lib* --parallel -c --all -r default',
assert_error=True)
self.assertIn("Connection fails with lib2 and lib4 references!",
client.out)
self.assertIn("Execute upload again to retry upload the failed files",
client.out)
def upload_parallel_success_test(self):
"""Upload 2 packages in parallel with success"""
client = TestClient(default_server_user=True)
client.save({"conanfile.py": GenConanfile()})
client.run('create . lib0/1.0@user/channel')
self.assertIn(
"lib0/1.0@user/channel: Package '{}' created".format(
NO_SETTINGS_PACKAGE_ID), client.out)
client.run('create . lib1/1.0@user/channel')
self.assertIn(
"lib1/1.0@user/channel: Package '{}' created".format(
NO_SETTINGS_PACKAGE_ID), client.out)
client.run('user -p password -r default user')
client.run('upload lib* --parallel -c --all -r default')
self.assertIn("Uploading lib0/1.0@user/channel to remote 'default'",
client.out)
self.assertIn("Uploading lib1/1.0@user/channel to remote 'default'",
client.out)
client.run('search lib0/1.0@user/channel -r default')
self.assertIn("lib0/1.0@user/channel", client.out)
client.run('search lib1/1.0@user/channel -r default')
self.assertIn("lib1/1.0@user/channel", client.out)
def upload_parallel_fail_on_interaction_test(self):
"""Upload 2 packages in parallel and fail because non_interactive forced"""
client = TestClient(default_server_user=True)
client.save({"conanfile.py": GenConanfile()})
num_references = 2
for index in range(num_references):
client.run('create . lib{}/1.0@user/channel'.format(index))
self.assertIn(
"lib{}/1.0@user/channel: Package '{}' created".format(
index, NO_SETTINGS_PACKAGE_ID), client.out)
client.run('user -c')
client.run('upload lib* --parallel -c --all -r default',
assert_error=True)
self.assertIn(
"ERROR: lib0/1.0@user/channel: Upload recipe to 'default' failed: "
"Conan interactive mode disabled. [Remote: default]", client.out)
def recipe_upload_fail_on_generic_exception_test(self):
# Make the upload fail with a generic Exception
client = TestClient(default_server_user=True)
conanfile = textwrap.dedent("""
import os
from conans import ConanFile
class Pkg(ConanFile):
exports = "*"
def package(self):
self.copy("*")
""")
client.save({
"conanfile.py": conanfile,
"myheader.h": "",
"conan_export.tgz/dummy": ""
})
client.run('create . lib/1.0@user/channel')
client.run('upload lib* -c --all -r default', assert_error=True)
self.assertIn(
"ERROR: lib/1.0@user/channel: Upload recipe to 'default' failed:",
client.out)
self.assertIn("ERROR: Errors uploading some packages", client.out)
def package_upload_fail_on_generic_exception_test(self):
# Make the upload fail with a generic Exception
client = TestClient(default_server_user=True)
conanfile = textwrap.dedent("""
import os
from conans import ConanFile
class Pkg(ConanFile):
exports = "*"
def package(self):
os.makedirs(os.path.join(self.package_folder, "conan_package.tgz"))
self.copy("*")
""")
client.save({"conanfile.py": conanfile, "myheader.h": ""})
client.run('create . lib/1.0@user/channel')
client.run('upload lib* -c --all -r default', assert_error=True)
self.assertNotIn("os.remove(tgz_path)", client.out)
self.assertNotIn("Traceback", client.out)
self.assertIn(
"ERROR: lib/1.0@user/channel:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9: "
"Upload package to 'default' failed:", client.out)
self.assertIn("ERROR: Errors uploading some packages", client.out)
with environment_append({"CONAN_VERBOSE_TRACEBACK": "True"}):
client.run('upload lib* -c --all -r default', assert_error=True)
self.assertIn("os.remove(tgz_path)", client.out)
self.assertIn("Traceback", client.out)
self.assertIn(
"ERROR: lib/1.0@user/channel:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9: "
"Upload package to 'default' failed:", client.out)
self.assertIn("ERROR: Errors uploading some packages", client.out)
def test_beat_character_long_upload(self):
client = TestClient(default_server_user=True)
slow_conanfile = textwrap.dedent("""
from conans import ConanFile
class MyPkg(ConanFile):
exports = "*"
def package(self):
self.copy("*")
""")
client.save({"conanfile.py": slow_conanfile, "hello.cpp": ""})
client.run("create . pkg/0.1@user/stable")
client.run("user user --password=password")
with patch("conans.util.progress_bar.TIMEOUT_BEAT_SECONDS", -1):
with patch("conans.util.progress_bar.TIMEOUT_BEAT_CHARACTER",
"%&$"):
client.run("upload pkg/0.1@user/stable --all")
out = "".join(str(client.out).splitlines())
self.assertIn(
"Compressing package...%&$%&$Uploading conan_package.tgz -> "
"pkg/0.1@user/stable:5ab8", out)
self.assertIn("%&$Uploading conan_export.tgz", out)
self.assertIn("%&$Uploading conaninfo.txt", out)
def upload_with_pattern_and_package_error_test(self):
files = hello_conan_files("Hello1", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
self.client.run("upload Hello* --confirm -p 234234234",
assert_error=True)
self.assertIn(
"-p parameter only allowed with a valid recipe reference",
self.client.out)
def check_upload_confirm_question_test(self):
user_io = MockedUserIO({"default": [("lasote", "mypass")]},
out=TestBufferConanOutput())
files = hello_conan_files("Hello1", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
user_io.request_string = lambda _: "y"
self.client.run("upload Hello*", user_io=user_io)
self.assertIn("Uploading Hello1/1.2.1@frodo/stable", self.client.out)
files = hello_conan_files("Hello2", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
user_io.request_string = lambda _: "n"
self.client.run("upload Hello*", user_io=user_io)
self.assertNotIn("Uploading Hello2/1.2.1@frodo/stable",
self.client.out)
def upload_same_package_dont_compress_test(self):
# Create a manifest for the faked package
pack_path = self.client.cache.package_layout(self.pref.ref).package(
self.pref)
package_path = self.client.cache.package_layout(self.pref.ref).package(
self.pref)
expected_manifest = FileTreeManifest.create(package_path)
expected_manifest.save(pack_path)
self.client.run("upload %s --all" % str(self.ref))
self.assertIn("Compressing recipe", self.client.out)
self.assertIn("Compressing package", str(self.client.out))
self.client.run("upload %s --all" % str(self.ref))
self.assertNotIn("Compressing recipe", self.client.out)
self.assertNotIn("Compressing package", str(self.client.out))
self.assertIn("Package is up to date", str(self.client.out))
def upload_with_no_valid_settings_test(self):
'''Check if upload is still working even if the specified setting is not valid.
If this test fails, will fail in Linux/OSx'''
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "Hello"
version = "1.2"
settings = {"os": ["Windows"]}
"""
files = {CONANFILE: conanfile}
self.client.save(files)
self.client.run("export . lasote/stable")
self.assertIn("WARN: Conanfile doesn't have 'license'",
self.client.out)
self.client.run("upload Hello/1.2@lasote/stable")
self.assertIn("Uploading conanmanifest.txt", self.client.out)
def single_binary_test(self):
""" basic installation of a new conans
"""
# Try to upload an package without upload conans first
self.client.run('upload %s -p %s' % (self.ref, str(self.pref.id)))
self.assertIn("Uploaded conan recipe '%s'" % str(self.ref),
self.client.out)
def simple_test(self):
""" basic installation of a new conans
"""
# Upload conans
self.client.run('upload %s' % str(self.ref))
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertTrue(os.path.exists(self.server_reg_folder))
if not self.client.cache.config.revisions_enabled:
self.assertFalse(os.path.exists(self.server_pack_folder))
# Upload package
self.client.run('upload %s -p %s' % (str(self.ref), str(self.pref.id)))
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Test the file in the downloaded conans
files = [
'CMakeLists.txt', 'my_lib/debug/libd.a', 'hello.cpp', 'hello0.h',
CONANFILE, CONAN_MANIFEST, 'main.cpp', 'include/math/lib1.h',
'my_data/readme.txt', 'my_bin/executable'
]
self.assertTrue(
os.path.exists(os.path.join(self.server_reg_folder, CONANFILE)))
self.assertTrue(
os.path.exists(
os.path.join(self.server_reg_folder, EXPORT_TGZ_NAME)))
tmp = temp_folder()
untargz(os.path.join(self.server_reg_folder, EXPORT_TGZ_NAME), tmp)
for f in files:
if f not in (CONANFILE, CONAN_MANIFEST):
self.assertTrue(os.path.exists(os.path.join(tmp, f)))
else:
self.assertFalse(os.path.exists(os.path.join(tmp, f)))
folder = uncompress_packaged_files(self.test_server.server_store,
self.pref)
self.assertTrue(
os.path.exists(os.path.join(folder, "include", "lib1.h")))
self.assertTrue(
os.path.exists(os.path.join(folder, "lib", "my_lib/libd.a")))
self.assertTrue(
os.path.exists(os.path.join(folder, "res", "shares/readme.txt")))
if platform.system() != "Windows":
self.assertEqual(
os.stat(os.path.join(folder, "bin",
"my_bin/executable")).st_mode
& stat.S_IRWXU, stat.S_IRWXU)
def upload_all_test(self):
"""Upload conans and package together"""
# Try to upload all conans and packages
self.client.run('user -p mypass -r default lasote')
self.client.run('upload %s --all' % str(self.ref))
lines = [
line.strip() for line in str(self.client.out).splitlines()
if line.startswith("Uploading")
]
self.assertEqual(lines, [
"Uploading to remote 'default':",
"Uploading Hello/1.2.1@frodo/stable to remote 'default'",
"Uploading conan_export.tgz -> Hello/1.2.1@frodo/stable",
"Uploading conanfile.py -> Hello/1.2.1@frodo/stable",
"Uploading conanmanifest.txt -> Hello/1.2.1@frodo/stable",
"Uploading package 1/1: myfakeid to 'default'",
"Uploading conan_package.tgz -> Hello/1.2.1@frodo/stable:myfa",
"Uploading conaninfo.txt -> Hello/1.2.1@frodo/stable:myfa",
"Uploading conanmanifest.txt -> Hello/1.2.1@frodo/stable:myfa",
])
if self.client.cache.config.revisions_enabled:
layout = self.client.cache.package_layout(self.ref)
rev = layout.recipe_revision()
self.ref = self.ref.copy_with_rev(rev)
prev = layout.package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
server_reg_folder = self.test_server.server_store.export(self.ref)
server_pack_folder = self.test_server.server_store.package(self.pref)
self.assertTrue(os.path.exists(server_reg_folder))
self.assertTrue(os.path.exists(server_pack_folder))
def force_test(self):
'''Tries to upload a conans exported after than remote version.'''
# Upload all conans and packages
self.client.run('upload %s --all' % str(self.ref))
if self.client.cache.config.revisions_enabled:
layout = self.client.cache.package_layout(self.ref)
rev = layout.recipe_revision()
self.ref = self.ref.copy_with_rev(rev)
prev = layout.package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Fake datetime from exported date and upload again
old_digest = self.client.cache.package_layout(
self.ref).recipe_manifest()
old_digest.file_sums["new_file"] = "012345"
fake_digest = FileTreeManifest(2, old_digest.file_sums)
fake_digest.save(self.client.cache.package_layout(self.ref).export())
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("Remote recipe is newer than local recipe",
self.client.out)
self.client.run('upload %s --force' % str(self.ref))
self.assertIn("Uploading %s" % str(self.ref), self.client.out)
# Repeat transfer, to make sure it is uploading again
self.client.run('upload %s --force' % str(self.ref))
self.assertIn("Uploading conan_export.tgz", self.client.out)
self.assertIn("Uploading conanfile.py", self.client.out)
def upload_json_test(self):
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "test"
version = "0.1"
def package(self):
self.copy("mylib.so", dst="lib")
"""
client = self._get_client()
client.save({"conanfile.py": conanfile, "mylib.so": ""})
client.run("create . danimtb/testing")
# Test conflict parameter error
client.run(
"upload test/0.1@danimtb/* --all -p ewvfw --json upload.json",
assert_error=True)
json_path = os.path.join(client.current_folder, "upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
self.assertTrue(output["error"])
self.assertEqual(0, len(output["uploaded"]))
# Test invalid reference error
client.run("upload fake/0.1@danimtb/testing --all --json upload.json",
assert_error=True)
json_path = os.path.join(client.current_folder, "upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
self.assertTrue(output["error"])
self.assertEqual(0, len(output["uploaded"]))
# Test normal upload
client.run("upload test/0.1@danimtb/testing --all --json upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
output_expected = {
"error":
False,
"uploaded": [{
"recipe": {
"id": "test/0.1@danimtb/testing",
"remote_url": "unknown",
"remote_name": "default",
"time": "unknown"
},
"packages": [{
"id": NO_SETTINGS_PACKAGE_ID,
"time": "unknown"
}]
}]
}
self.assertEqual(output_expected["error"], output["error"])
self.assertEqual(len(output_expected["uploaded"]),
len(output["uploaded"]))
for i, item in enumerate(output["uploaded"]):
self.assertEqual(output_expected["uploaded"][i]["recipe"]["id"],
item["recipe"]["id"])
self.assertEqual(
output_expected["uploaded"][i]["recipe"]["remote_name"],
item["recipe"]["remote_name"])
for j, subitem in enumerate(item["packages"]):
self.assertEqual(
output_expected["uploaded"][i]["packages"][j]["id"],
subitem["id"])
class UploadTest(unittest.TestCase):
def _get_client(self, requester=None):
servers = {}
# All can write (for avoid authentication until we mock user_io)
self.test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")],
users={"lasote": "mypass"})
servers["default"] = self.test_server
return TestClient(servers=servers,
users={"default": [("lasote", "mypass")]},
requester_class=requester)
def setUp(self):
self.client = self._get_client()
self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" %
DEFAULT_REVISION_V1)
reg_folder = self.client.cache.export(self.ref)
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn(
"There is no local conanfile exported as %s" % str(self.ref),
self.client.user_io.out)
files = hello_source_files()
self.client.save(files, path=reg_folder)
self.client.save(
{
CONANFILE: myconan1,
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"
},
path=reg_folder)
mkdir(self.client.cache.export_sources(self.ref))
manifest = FileTreeManifest.create(reg_folder)
manifest.time = '123123123'
manifest.save(reg_folder)
self.test_server.server_store.update_last_revision(self.ref)
self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
package_folder = self.client.cache.package(self.pref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"),
"//bin")
save(os.path.join(package_folder, CONANINFO),
"""[recipe_hash]\n%s""" % manifest.summary_hash)
FileTreeManifest.create(package_folder).save(package_folder)
self.test_server.server_store.update_last_package_revision(self.pref)
os.chmod(
os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin",
"executable")).st_mode | stat.S_IRWXU)
package_path = self.client.cache.package(self.pref)
expected_manifest = FileTreeManifest.create(package_path)
expected_manifest.save(package_folder)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def try_upload_bad_recipe_test(self):
files = hello_conan_files("Hello0", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
ref = ConanFileReference.loads("Hello0/1.2.1@frodo/stable")
os.unlink(os.path.join(self.client.cache.export(ref), CONAN_MANIFEST))
with self.assertRaisesRegexp(Exception, "Command failed"):
self.client.run("upload %s" % str(ref))
self.assertIn("Cannot upload corrupted recipe",
self.client.user_io.out)
def upload_with_pattern_test(self):
for num in range(5):
files = hello_conan_files("Hello%s" % num, "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
self.client.run("upload Hello* --confirm")
for num in range(5):
self.assertIn("Uploading Hello%s/1.2.1@frodo/stable" % num,
self.client.user_io.out)
self.client.run("upload Hello0* --confirm")
self.assertIn("Uploading Hello0/1.2.1@frodo/stable",
self.client.user_io.out)
self.assertIn("Recipe is up to date, upload skipped", self.client.out)
self.assertNotIn("Hello1", self.client.user_io.out)
self.assertNotIn("Hello2", self.client.user_io.out)
self.assertNotIn("Hello3", self.client.user_io.out)
def upload_error_test(self):
"""Cause an error in the transfer and see some message"""
# This will fail in the first put file, so, as we need to
# upload 3 files (conanmanifest, conanfile and tgz) will do it with 2 retries
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry-wait=0")
self.assertIn("Can't connect because of the evil mock",
client.user_io.out)
self.assertIn("Waiting 0 seconds to retry...", client.user_io.out)
# but not with 1
client = self._get_client(BadConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry 1 --retry-wait=1",
assert_error=True)
self.assertNotIn("Waiting 1 seconds to retry...", client.user_io.out)
self.assertIn(
"ERROR: Execute upload again to retry upload the failed files: "
"conanmanifest.txt. [Remote: default]", client.user_io.out)
# Try with broken connection even with 10 retries
client = self._get_client(TerribleConnectionUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry 10 --retry-wait=0",
assert_error=True)
self.assertIn("Waiting 0 seconds to retry...", client.user_io.out)
self.assertIn(
"ERROR: Execute upload again to retry upload the failed files",
client.out)
# For each file will fail the first time and will success in the second one
client = self._get_client(FailPairFilesUploader)
files = cpp_hello_conan_files("Hello0", "1.2.1", build=False)
client.save(files)
client.run("export . frodo/stable")
client.run("install Hello0/1.2.1@frodo/stable --build")
client.run("upload Hello* --confirm --retry 3 --retry-wait=0 --all")
self.assertEquals(
str(client.user_io.out).count("ERROR: Pair file, error!"), 6)
def upload_with_pattern_and_package_error_test(self):
files = hello_conan_files("Hello1", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
self.client.run("upload Hello* --confirm -p 234234234",
assert_error=True)
self.assertIn(
"-p parameter only allowed with a valid recipe reference",
self.client.user_io.out)
def check_upload_confirm_question_test(self):
user_io = self.client.user_io
files = hello_conan_files("Hello1", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
user_io.request_string = lambda _: "y"
self.client.run("upload Hello*", user_io=user_io)
self.assertIn("Uploading Hello1/1.2.1@frodo/stable",
self.client.user_io.out)
files = hello_conan_files("Hello2", "1.2.1")
self.client.save(files)
self.client.run("export . frodo/stable")
user_io.request_string = lambda _: "n"
self.client.run("upload Hello*", user_io=user_io)
self.assertNotIn("Uploading Hello2/1.2.1@frodo/stable",
self.client.user_io.out)
def upload_same_package_dont_compress_test(self):
# Create a manifest for the faked package
pack_path = self.client.cache.package(self.pref)
package_path = self.client.cache.package(self.pref)
expected_manifest = FileTreeManifest.create(package_path)
expected_manifest.save(pack_path)
self.client.run("upload %s --all" % str(self.ref))
self.assertIn("Compressing recipe", self.client.user_io.out)
self.assertIn("Compressing package", str(self.client.user_io.out))
self.client.run("upload %s --all" % str(self.ref))
self.assertNotIn("Compressing recipe", self.client.user_io.out)
self.assertNotIn("Compressing package", str(self.client.user_io.out))
self.assertIn("Package is up to date", str(self.client.user_io.out))
def upload_with_no_valid_settings_test(self):
'''Check if upload is still working even if the specified setting is not valid.
If this test fails, will fail in Linux/OSx'''
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "Hello"
version = "1.2"
settings = {"os": ["Windows"]}
"""
files = {CONANFILE: conanfile}
self.client.save(files)
self.client.run("export . lasote/stable")
self.assertIn("WARN: Conanfile doesn't have 'license'",
self.client.user_io.out)
self.client.run("upload Hello/1.2@lasote/stable")
self.assertIn("Uploading conanmanifest.txt", self.client.user_io.out)
def single_binary_test(self):
""" basic installation of a new conans
"""
# Try to upload an package without upload conans first
self.client.run('upload %s -p %s' % (self.ref, str(self.pref.id)))
self.assertIn("Uploaded conan recipe '%s'" % str(self.ref),
self.client.out)
def simple_test(self):
""" basic installation of a new conans
"""
# Upload conans
self.client.run('upload %s' % str(self.ref))
if not self.client.block_v2:
rev = self.client.get_revision(self.ref)
self.ref = self.ref.copy_with_rev(rev)
prev = self.client.get_package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertTrue(os.path.exists(self.server_reg_folder))
if self.client.block_v2:
self.assertFalse(os.path.exists(self.server_pack_folder))
# Upload package
self.client.run('upload %s -p %s' % (str(self.ref), str(self.pref.id)))
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Test the file in the downloaded conans
files = [
'CMakeLists.txt', 'my_lib/debug/libd.a', 'hello.cpp', 'hello0.h',
CONANFILE, CONAN_MANIFEST, 'main.cpp', 'include/math/lib1.h',
'my_data/readme.txt', 'my_bin/executable'
]
self.assertTrue(
os.path.exists(os.path.join(self.server_reg_folder, CONANFILE)))
self.assertTrue(
os.path.exists(
os.path.join(self.server_reg_folder, EXPORT_TGZ_NAME)))
tmp = temp_folder()
untargz(os.path.join(self.server_reg_folder, EXPORT_TGZ_NAME), tmp)
for f in files:
if f not in (CONANFILE, CONAN_MANIFEST):
self.assertTrue(os.path.exists(os.path.join(tmp, f)))
else:
self.assertFalse(os.path.exists(os.path.join(tmp, f)))
folder = uncompress_packaged_files(self.test_server.server_store,
self.pref)
self.assertTrue(
os.path.exists(os.path.join(folder, "include", "lib1.h")))
self.assertTrue(
os.path.exists(os.path.join(folder, "lib", "my_lib/libd.a")))
self.assertTrue(
os.path.exists(os.path.join(folder, "res", "shares/readme.txt")))
if platform.system() != "Windows":
self.assertEqual(
os.stat(os.path.join(folder, "bin",
"my_bin/executable")).st_mode
& stat.S_IRWXU, stat.S_IRWXU)
def upload_all_test(self):
'''Upload conans and package together'''
# Try to upload all conans and packages
self.client.run('upload %s --all' % str(self.ref))
lines = [
line.strip() for line in str(self.client.user_io.out).splitlines()
if line.startswith("Uploading")
]
self.assertEqual(lines, [
"Uploading Hello/1.2.1@frodo/stable to remote 'default'",
"Uploading conanmanifest.txt",
"Uploading conanfile.py",
"Uploading conan_export.tgz",
"Uploading package 1/1: myfakeid to 'default'",
"Uploading conanmanifest.txt",
"Uploading conaninfo.txt",
"Uploading conan_package.tgz",
])
if not self.client.block_v2:
rev = self.client.get_revision(self.ref)
self.ref = self.ref.copy_with_rev(rev)
prev = self.client.get_package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
server_reg_folder = self.test_server.server_store.export(self.ref)
server_pack_folder = self.test_server.server_store.package(self.pref)
self.assertTrue(os.path.exists(server_reg_folder))
self.assertTrue(os.path.exists(server_pack_folder))
def force_test(self):
'''Tries to upload a conans exported after than remote version.'''
# Upload all conans and packages
self.client.run('upload %s --all' % str(self.ref))
if not self.client.block_v2:
rev = self.client.get_revision(self.ref)
self.ref = self.ref.copy_with_rev(rev)
prev = self.client.get_package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Fake datetime from exported date and upload again
old_digest = self.client.cache.package_layout(self.ref).load_manifest()
old_digest.file_sums["new_file"] = "012345"
fake_digest = FileTreeManifest(2, old_digest.file_sums)
fake_digest.save(self.client.cache.export(self.ref))
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("Remote recipe is newer than local recipe",
self.client.user_io.out)
self.client.run('upload %s --force' % str(self.ref))
self.assertIn("Uploading %s" % str(self.ref), self.client.user_io.out)
# Repeat transfer, to make sure it is uploading again
self.client.run('upload %s --force' % str(self.ref))
self.assertIn("Uploading conan_export.tgz", self.client.out)
self.assertIn("Uploading conanfile.py", self.client.out)
def upload_json_test(self):
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "test"
version = "0.1"
def package(self):
self.copy("mylib.so", dst="lib")
"""
client = self._get_client()
client.save({"conanfile.py": conanfile, "mylib.so": ""})
client.run("create . danimtb/testing")
# Test conflict parameter error
client.run(
"upload test/0.1@danimtb/* --all -p ewvfw --json upload.json",
assert_error=True)
json_path = os.path.join(client.current_folder, "upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
self.assertTrue(output["error"])
self.assertEqual(0, len(output["uploaded"]))
# Test invalid reference error
client.run("upload fake/0.1@danimtb/testing --all --json upload.json",
assert_error=True)
json_path = os.path.join(client.current_folder, "upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
self.assertTrue(output["error"])
self.assertEqual(0, len(output["uploaded"]))
# Test normal upload
client.run("upload test/0.1@danimtb/testing --all --json upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
output_expected = {
"error":
False,
"uploaded": [{
"recipe": {
"id": "test/0.1@danimtb/testing",
"remote_url": "unknown",
"remote_name": "default",
"time": "unknown"
},
"packages": [{
"id": NO_SETTINGS_PACKAGE_ID,
"time": "unknown"
}]
}]
}
self.assertEqual(output_expected["error"], output["error"])
self.assertEqual(len(output_expected["uploaded"]),
len(output["uploaded"]))
for i, item in enumerate(output["uploaded"]):
self.assertEqual(output_expected["uploaded"][i]["recipe"]["id"],
item["recipe"]["id"])
self.assertEqual(
output_expected["uploaded"][i]["recipe"]["remote_name"],
item["recipe"]["remote_name"])
for j, subitem in enumerate(item["packages"]):
self.assertEqual(
output_expected["uploaded"][i]["packages"][j]["id"],
subitem["id"])
class UploadTest(unittest.TestCase):
def _get_client(self, requester=None):
servers = {}
# All can write (for avoid authentication until we mock user_io)
self.test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")],
users={"lasote": "mypass"})
servers["default"] = self.test_server
test_client = TestClient(servers=servers,
users={"default": [("lasote", "mypass")]},
requester_class=requester)
save(test_client.cache.default_profile_path, "")
return test_client
def setUp(self):
self.client = self._get_client()
self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" %
DEFAULT_REVISION_V1)
self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1)
reg_folder = self.client.cache.package_layout(self.ref).export()
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("ERROR: Recipe not found: '%s'" % str(self.ref),
self.client.out)
files = {}
fake_metadata = PackageMetadata()
fake_metadata.recipe.revision = DEFAULT_REVISION_V1
fake_metadata.packages[self.pref.id].revision = DEFAULT_REVISION_V1
self.client.save({"metadata.json": fake_metadata.dumps()},
path=self.client.cache.package_layout(
self.ref).base_folder())
self.client.save(files, path=reg_folder)
self.client.save(
{
CONANFILE:
GenConanfile().with_name("Hello").with_version("1.2.1"),
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"
},
path=reg_folder)
mkdir(self.client.cache.package_layout(self.ref).export_sources())
manifest = FileTreeManifest.create(reg_folder)
manifest.time = '123123123'
manifest.save(reg_folder)
self.test_server.server_store.update_last_revision(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
package_folder = self.client.cache.package_layout(self.ref).package(
self.pref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"),
"//bin")
save(os.path.join(package_folder, CONANINFO),
"""[recipe_hash]\n%s""" % manifest.summary_hash)
FileTreeManifest.create(package_folder).save(package_folder)
self.test_server.server_store.update_last_package_revision(self.pref)
os.chmod(
os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin",
"executable")).st_mode | stat.S_IRWXU)
expected_manifest = FileTreeManifest.create(package_folder)
expected_manifest.save(package_folder)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def test_upload_error(self):
"""Cause an error in the transfer and see some message"""
# Check for the default behaviour
client = self._get_client(BadConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm")
self.assertIn("Can't connect because of the evil mock", client.out)
self.assertIn("Waiting 5 seconds to retry...", client.out)
# This will fail in the first put file, so, as we need to
# upload 3 files (conanmanifest, conanfile and tgz) will do it with 2 retries
client = self._get_client(BadConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry-wait=0")
self.assertIn("Can't connect because of the evil mock", client.out)
self.assertIn("Waiting 0 seconds to retry...", client.out)
# but not with 0
client = self._get_client(BadConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*"),
"somefile.txt": ""
}
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry 0 --retry-wait=1",
assert_error=True)
self.assertNotIn("Waiting 1 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files: "
"conan_export.tgz. [Remote: default]", client.out)
# Try with broken connection even with 10 retries
client = self._get_client(TerribleConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run("upload Hello* --confirm --retry 10 --retry-wait=0",
assert_error=True)
self.assertIn("Waiting 0 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files",
client.out)
# For each file will fail the first time and will success in the second one
client = self._get_client(FailPairFilesUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run("install Hello0/1.2.1@frodo/stable --build")
client.run("upload Hello* --confirm --retry 3 --retry-wait=0 --all")
self.assertEqual(str(client.out).count("ERROR: Pair file, error!"), 5)
def test_upload_error_with_config(self):
"""Cause an error in the transfer and see some message"""
# This will fail in the first put file, so, as we need to
# upload 3 files (conanmanifest, conanfile and tgz) will do it with 2 retries
client = self._get_client(BadConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run('config set general.retry_wait=0')
client.run("upload Hello* --confirm")
self.assertIn("Can't connect because of the evil mock", client.out)
self.assertIn("Waiting 0 seconds to retry...", client.out)
# but not with 0
client = self._get_client(BadConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*"),
"somefile.txt": ""
}
client.save(files)
client.run("export . frodo/stable")
client.run('config set general.retry=0')
client.run('config set general.retry_wait=1')
client.run("upload Hello* --confirm", assert_error=True)
self.assertNotIn("Waiting 1 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files: "
"conan_export.tgz. [Remote: default]", client.out)
# Try with broken connection even with 10 retries
client = self._get_client(TerribleConnectionUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run('config set general.retry=10')
client.run('config set general.retry_wait=0')
client.run("upload Hello* --confirm", assert_error=True)
self.assertIn("Waiting 0 seconds to retry...", client.out)
self.assertIn(
"ERROR: Hello0/1.2.1@frodo/stable: Upload recipe to 'default' failed: "
"Execute upload again to retry upload the failed files",
client.out)
# For each file will fail the first time and will success in the second one
client = self._get_client(FailPairFilesUploader)
files = {
"conanfile.py": GenConanfile("Hello0", "1.2.1").with_exports("*")
}
client.save(files)
client.run("export . frodo/stable")
client.run("install Hello0/1.2.1@frodo/stable --build")
client.run('config set general.retry=3')
client.run('config set general.retry_wait=0')
client.run("upload Hello* --confirm --all")
self.assertEqual(str(client.out).count("ERROR: Pair file, error!"), 5)
def test_upload_same_package_dont_compress(self):
# Create a manifest for the faked package
pack_path = self.client.cache.package_layout(self.pref.ref).package(
self.pref)
package_path = self.client.cache.package_layout(self.pref.ref).package(
self.pref)
expected_manifest = FileTreeManifest.create(package_path)
expected_manifest.save(pack_path)
self.client.run("upload %s --all" % str(self.ref))
self.assertIn("Compressing recipe", self.client.out)
self.assertIn("Compressing package", str(self.client.out))
self.client.run("upload %s --all" % str(self.ref))
self.assertNotIn("Compressing recipe", self.client.out)
self.assertNotIn("Compressing package", str(self.client.out))
self.assertIn("Package is up to date", str(self.client.out))
def test_upload_with_no_valid_settings(self):
# Check if upload is still working even if the specified setting is not valid.
# If this test fails, will fail in Linux/OSx
conanfile = textwrap.dedent("""
from conans import ConanFile
class TestConan(ConanFile):
name = "Hello"
version = "1.2"
settings = {"os": ["Windows"]}
""")
self.client.save({CONANFILE: conanfile})
self.client.run("export . lasote/stable")
self.assertIn("WARN: Conanfile doesn't have 'license'",
self.client.out)
self.client.run("upload Hello/1.2@lasote/stable")
self.assertIn("Uploading conanmanifest.txt", self.client.out)
def test_single_binary(self):
# Try to upload an package without upload conans first
self.client.run('upload %s -p %s' % (self.ref, str(self.pref.id)))
self.assertIn("Uploading %s to remote" % str(self.ref),
self.client.out)
def test_simple(self):
# Upload package
self.client.run('upload %s' % str(self.ref))
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertTrue(os.path.exists(self.server_reg_folder))
if not self.client.cache.config.revisions_enabled:
self.assertFalse(os.path.exists(self.server_pack_folder))
# Upload package
self.client.run('upload %s -p %s' % (str(self.ref), str(self.pref.id)))
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Test the file in the downloaded conans
files = [
'my_lib/debug/libd.a', CONANFILE, CONAN_MANIFEST,
'include/math/lib1.h', 'my_data/readme.txt', 'my_bin/executable'
]
self.assertTrue(
os.path.exists(os.path.join(self.server_reg_folder, CONANFILE)))
self.assertTrue(
os.path.exists(
os.path.join(self.server_reg_folder, EXPORT_TGZ_NAME)))
tmp = temp_folder()
untargz(os.path.join(self.server_reg_folder, EXPORT_TGZ_NAME), tmp)
for f in files:
if f not in (CONANFILE, CONAN_MANIFEST):
self.assertTrue(os.path.exists(os.path.join(tmp, f)))
else:
self.assertFalse(os.path.exists(os.path.join(tmp, f)))
folder = uncompress_packaged_files(self.test_server.server_store,
self.pref)
self.assertTrue(
os.path.exists(os.path.join(folder, "include", "lib1.h")))
self.assertTrue(
os.path.exists(os.path.join(folder, "lib", "my_lib/libd.a")))
self.assertTrue(
os.path.exists(os.path.join(folder, "res", "shares/readme.txt")))
if platform.system() != "Windows":
self.assertEqual(
os.stat(os.path.join(folder, "bin",
"my_bin/executable")).st_mode
& stat.S_IRWXU, stat.S_IRWXU)
def test_upload_all(self):
"""Upload recipe and package together"""
# Try to upload all conans and packages
self.client.run('user -p mypass -r default lasote')
self.client.run('upload %s --all' % str(self.ref))
lines = [
line.strip() for line in str(self.client.out).splitlines()
if line.startswith("Uploading")
]
self.assertEqual(lines, [
"Uploading to remote 'default':",
"Uploading Hello/1.2.1@frodo/stable to remote 'default'",
"Uploading conan_export.tgz -> Hello/1.2.1@frodo/stable",
"Uploading conanfile.py -> Hello/1.2.1@frodo/stable",
"Uploading conanmanifest.txt -> Hello/1.2.1@frodo/stable",
"Uploading package 1/1: myfakeid to 'default'",
"Uploading conan_package.tgz -> Hello/1.2.1@frodo/stable:myfa",
"Uploading conaninfo.txt -> Hello/1.2.1@frodo/stable:myfa",
"Uploading conanmanifest.txt -> Hello/1.2.1@frodo/stable:myfa",
])
if self.client.cache.config.revisions_enabled:
layout = self.client.cache.package_layout(self.ref)
rev = layout.recipe_revision()
self.ref = self.ref.copy_with_rev(rev)
prev = layout.package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
server_reg_folder = self.test_server.server_store.export(self.ref)
server_pack_folder = self.test_server.server_store.package(self.pref)
self.assertTrue(os.path.exists(server_reg_folder))
self.assertTrue(os.path.exists(server_pack_folder))
def test_force(self):
# Tries to upload a package exported after than remote version.
# Upload all recipes and packages
self.client.run('upload %s --all' % str(self.ref))
if self.client.cache.config.revisions_enabled:
layout = self.client.cache.package_layout(self.ref)
rev = layout.recipe_revision()
self.ref = self.ref.copy_with_rev(rev)
prev = layout.package_revision(self.pref)
self.pref = self.pref.copy_with_revs(rev, prev)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Fake datetime from exported date and upload again
old_digest = self.client.cache.package_layout(
self.ref).recipe_manifest()
old_digest.file_sums["new_file"] = "012345"
fake_digest = FileTreeManifest(2, old_digest.file_sums)
fake_digest.save(self.client.cache.package_layout(self.ref).export())
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("Remote recipe is newer than local recipe",
self.client.out)
self.client.run('upload %s --force' % str(self.ref))
self.assertIn("Uploading %s" % str(self.ref), self.client.out)
# Repeat transfer, to make sure it is uploading again
self.client.run('upload %s --force' % str(self.ref))
self.assertIn("Uploading conan_export.tgz", self.client.out)
self.assertIn("Uploading conanfile.py", self.client.out)
def test_upload_json(self):
conanfile = textwrap.dedent("""
from conans import ConanFile
class TestConan(ConanFile):
name = "test"
version = "0.1"
def package(self):
self.copy("mylib.so", dst="lib")
""")
client = self._get_client()
client.save({"conanfile.py": conanfile, "mylib.so": ""})
client.run("create . danimtb/testing")
# Test conflict parameter error
client.run(
"upload test/0.1@danimtb/* --all -p ewvfw --json upload.json",
assert_error=True)
json_path = os.path.join(client.current_folder, "upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
self.assertTrue(output["error"])
self.assertEqual(0, len(output["uploaded"]))
# Test invalid reference error
client.run("upload fake/0.1@danimtb/testing --all --json upload.json",
assert_error=True)
json_path = os.path.join(client.current_folder, "upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
self.assertTrue(output["error"])
self.assertEqual(0, len(output["uploaded"]))
# Test normal upload
client.run("upload test/0.1@danimtb/testing --all --json upload.json")
self.assertTrue(os.path.exists(json_path))
json_content = load(json_path)
output = json.loads(json_content)
output_expected = {
"error":
False,
"uploaded": [{
"recipe": {
"id": "test/0.1@danimtb/testing",
"remote_url": "unknown",
"remote_name": "default",
"time": "unknown"
},
"packages": [{
"id": NO_SETTINGS_PACKAGE_ID,
"time": "unknown"
}]
}]
}
self.assertEqual(output_expected["error"], output["error"])
self.assertEqual(len(output_expected["uploaded"]),
len(output["uploaded"]))
for i, item in enumerate(output["uploaded"]):
self.assertEqual(output_expected["uploaded"][i]["recipe"]["id"],
item["recipe"]["id"])
self.assertEqual(
output_expected["uploaded"][i]["recipe"]["remote_name"],
item["recipe"]["remote_name"])
for j, subitem in enumerate(item["packages"]):
self.assertEqual(
output_expected["uploaded"][i]["packages"][j]["id"],
subitem["id"])
