def _download(self, downloads, processed_package_refs):
""" executes the download of packages (both download and update), only once for a given
PREF, even if node duplicated
:param downloads: all nodes to be downloaded or updated, included repetitions
"""
if not downloads:
return
download_nodes = []
for node in downloads:
pref = node.pref
bare_pref = PackageReference(pref.ref, pref.id)
if bare_pref in processed_package_refs:
continue
processed_package_refs[bare_pref] = pref.revision
assert node.prev, "PREV for %s is None" % str(node.pref)
download_nodes.append(node)
def _download(n):
layout = self._cache.package_layout(n.pref.ref,
n.conanfile.short_paths)
# We cannot embed the package_lock inside the remote.get_package()
# because the handle_node_cache has its own lock
with layout.package_lock(n.pref):
self._download_pkg(layout, n)
parallel = self._cache.config.parallel_download
if parallel is not None:
self._out.info(
"Downloading binary packages in %s parallel threads" %
parallel)
thread_pool = ThreadPool(parallel)
thread_pool.map(_download, [n for n in download_nodes])
thread_pool.close()
thread_pool.join()
else:
for node in download_nodes:
_download(node)
def export_test(self):
if platform.system() == "Windows":
return
lib_name = "libtest.so.2"
lib_contents = "TestLib"
link_name = "libtest.so"
client = TestClient()
client.save({
"conanfile.py": conanfile,
"conanfile.txt": test_conanfile,
lib_name: lib_contents
})
pre_export_link = os.path.join(client.current_folder, link_name)
os.symlink(lib_name, pre_export_link)
client.run("export lasote/stable")
client.run("install --build -f=conanfile.txt")
conan_ref = ConanFileReference.loads("Hello/0.1@lasote/stable")
package_ref = PackageReference(
conan_ref, "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
for folder in [
client.paths.export(conan_ref),
client.paths.source(conan_ref),
client.paths.build(package_ref),
client.paths.package(package_ref)
]:
exported_lib = os.path.join(folder, lib_name)
exported_link = os.path.join(folder, link_name)
self.assertEqual(os.readlink(exported_link), lib_name)
self.assertEqual(load(exported_lib), load(exported_link))
self.assertTrue(os.path.islink(exported_link))
self._check(client, package_ref)
def upload(self,
conan_reference_or_pattern,
package_id=None,
all_packages=None,
force=False,
confirm=False,
retry=0,
retry_wait=0,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""If package_id is provided, conan_reference_or_pattern is a ConanFileReference"""
if package_id and not _is_a_reference(conan_reference_or_pattern):
raise ConanException(
"-p parameter only allowed with a valid recipe reference, "
"not with a pattern")
t1 = time.time()
if package_id: # Upload package
ref = ConanFileReference.loads(conan_reference_or_pattern)
self._check_reference(ref)
self.upload_package(PackageReference(ref, package_id),
retry=retry,
retry_wait=retry_wait,
skip_upload=skip_upload,
integrity_check=integrity_check,
no_overwrite=no_overwrite)
else: # Upload conans
self._run_upload(conan_reference_or_pattern,
all_packages=all_packages,
force=force,
confirm=confirm,
retry=retry,
retry_wait=retry_wait,
skip_upload=skip_upload,
integrity_check=integrity_check,
no_overwrite=no_overwrite)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def test_short_paths(self):
folder = temp_folder(False)
short_folder = os.path.join(folder, ".cn")
with tools.environment_append({"CONAN_USER_HOME_SHORT": short_folder}):
client = TestClient(base_folder=folder)
client.save({CONANFILE: conanfile_py.replace("False", "True")})
client.run("export %s" % self.user_channel)
client.run("info --paths %s" % (self.conan_ref))
base_path = os.path.join("MyPackage", "0.1.0", "myUser", "testing")
output = client.user_io.out
self.assertIn(os.path.join(base_path, "export"), output)
if long_paths_support:
self.assertIn(os.path.join(base_path, "source"), output)
self.assertIn(os.path.join(base_path, "build"), output)
self.assertIn(os.path.join(base_path, "package"), output)
self.assertNotIn(short_folder, output)
else:
self.assertNotIn(os.path.join(base_path, "source"), output)
self.assertNotIn(os.path.join(base_path, "build"), output)
self.assertNotIn(os.path.join(base_path, "package"), output)
self.assertIn("source_folder: %s" % short_folder, output)
self.assertIn("build_folder: %s" % short_folder, output)
self.assertIn("package_folder: %s" % short_folder, output)
# Ensure that the inner folders are not created (that could affect
# pkg creation flow
ref = ConanFileReference.loads(self.conan_ref)
id_ = re.search('ID:\s*([a-z0-9]*)',
str(client.user_io.out)).group(1)
pkg_ref = PackageReference(ref, id_)
for path in (client.client_cache.source(ref, True),
client.client_cache.build(pkg_ref, True),
client.client_cache.package(pkg_ref, True)):
self.assertFalse(os.path.exists(path))
if not long_paths_support: # The parent has been created for .conan_link
self.assertTrue(os.path.exists(os.path.dirname(path)))
def remove_packages_test(self):
ref = ConanFileReference.loads(
"MySecondConan/2.0.0@private_user/testing#%s" %
DEFAULT_REVISION_V1)
self._upload_recipe(ref)
folders = {}
for sha in ["1", "2", "3", "4", "5"]:
# Upload an package
pref = PackageReference(ref, sha, DEFAULT_REVISION_V1)
self._upload_package(pref)
folder = self.server.server_store.package(pref)
self.assertTrue(os.path.exists(folder))
folders[sha] = folder
self.api.remove_packages(ref, ["1"])
self.assertTrue(
os.path.exists(self.server.server_store.base_folder(ref)))
self.assertFalse(os.path.exists(folders["1"]))
self.assertTrue(os.path.exists(folders["2"]))
self.assertTrue(os.path.exists(folders["3"]))
self.assertTrue(os.path.exists(folders["4"]))
self.assertTrue(os.path.exists(folders["5"]))
self.api.remove_packages(ref, ["2", "3"])
self.assertTrue(
os.path.exists(self.server.server_store.base_folder(ref)))
self.assertFalse(os.path.exists(folders["1"]))
self.assertFalse(os.path.exists(folders["2"]))
self.assertFalse(os.path.exists(folders["3"]))
self.assertTrue(os.path.exists(folders["4"]))
self.assertTrue(os.path.exists(folders["5"]))
self.api.remove_packages(ref, [])
self.assertTrue(
os.path.exists(self.server.server_store.base_folder(ref)))
for sha in ["1", "2", "3", "4", "5"]:
self.assertFalse(os.path.exists(folders[sha]))
def test_cmake_lib_template():
client = TestClient(path_with_spaces=False)
client.run("new hello/0.1 --template=cmake_lib")
# Local flow works
client.run("install . -if=install")
client.run("build . -if=install")
client.run("export-pkg . hello/0.1@ -if=install")
package_id = re.search(r"Packaging to (\S+)", str(client.out)).group(1)
pref = PackageReference(ConanFileReference.loads("hello/0.1"), package_id)
package_folder = client.cache.package_layout(pref.ref).package(pref)
assert os.path.exists(os.path.join(package_folder, "include", "hello.h"))
# Create works
client.run("create .")
assert "hello/0.1: Hello World Release!" in client.out
client.run("create . -s build_type=Debug")
assert "hello/0.1: Hello World Debug!" in client.out
# Create + shared works
client.run("create . -o hello:shared=True")
assert "hello/0.1: Hello World Release!" in client.out
def _get_local_infos_min(paths, reference):
result = {}
packages_path = paths.packages(reference)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(
paths.package(package_reference, short_paths=None), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
conan_vars_info = ConanInfo.loads(
conan_info_content).serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" %
str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def upload(self, conan_reference, package_id=None, remote=None, all_packages=None,
force=False):
remote_proxy = ConanProxy(self._paths, self._user_io, self._remote_manager, remote)
uploader = ConanUploader(self._paths, self._user_io, remote_proxy)
# Load conanfile to check if the build policy is set to always
try:
conanfile_path = self._paths.conanfile(conan_reference)
output = ScopedOutput(str(conan_reference), self._user_io.out)
conan_file = self._loader().load_conan(conanfile_path, output, consumer=False)
except NotFoundException:
raise NotFoundException("There is no local conanfile exported as %s"
% str(conan_reference))
if conan_file.build_policy_always and (all_packages or package_id):
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
if package_id: # Upload package
uploader.upload_package(PackageReference(conan_reference, package_id))
else: # Upload conans
uploader.upload_conan(conan_reference, all_packages=all_packages, force=force)
def get_path(self, path, package_id=None):
""" Return the contents for the given `path` inside current layout, it can
be a single file or the list of files in a directory
:param package_id: will retrieve the contents from the package directory
:param path: path relative to the cache reference or package folder
"""
assert not os.path.isabs(path)
if package_id is None: # Get the file in the exported files
folder = self.export()
else:
pref = PackageReference(self._ref, package_id)
folder = self.package(pref)
abs_path = os.path.join(folder, path)
if not os.path.exists(abs_path):
raise NotFoundException("The specified path doesn't exist")
if os.path.isdir(abs_path):
return sorted([path for path in os.listdir(abs_path) if not discarded_file(path)])
else:
return load(abs_path)
def package_copier_test(self):
client = TestClient()
files = {"conanfile.py": base}
client.save(files)
client.run("export . lasote/channel")
client.run("install lib/0.1@lasote/channel --build")
client.run("copy lib/0.1@lasote/channel memsharded/stable --all")
client.run("search")
self.assertIn("lib/0.1@lasote/channel", client.out)
self.assertIn("lib/0.1@memsharded/stable", client.out)
client.run("search lib/0.1@lasote/channel")
self.assertIn("Package_ID: %s" % NO_SETTINGS_PACKAGE_ID, client.out)
client.run("search lib/0.1@memsharded/stable")
self.assertIn("Package_ID: %s" % NO_SETTINGS_PACKAGE_ID, client.out)
ref = ConanFileReference.loads("lib/0.1@lasote/channel")
pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID)
package_folder = client.cache.package_layout(
pref.ref, short_paths=False).package(pref)
if platform.system() == "Windows":
package_folder = load(os.path.join(package_folder, ".conan_link"))
self.assertTrue(os.path.exists(package_folder))
def test_upload_dirty(self):
client = TestClient(default_server_user=True)
client.save({"conanfile.py": GenConanfile("Hello", "0.1")})
client.run("create . lasote/testing")
ref = ConanFileReference.loads("Hello/0.1@lasote/testing")
pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID)
layout = client.cache.package_layout(pref.ref)
pkg_folder = os.path.join(layout.base_folder(), PACKAGES_FOLDER,
pref.id)
set_dirty(pkg_folder)
client.run("upload * --all --confirm", assert_error=True)
self.assertIn(
"ERROR: Hello/0.1@lasote/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9: "
"Upload package to 'default' failed: Package %s is corrupted, aborting upload"
% str(pref), client.out)
self.assertIn(
"Remove it with 'conan remove Hello/0.1@lasote/testing -p=%s'" %
NO_SETTINGS_PACKAGE_ID, client.out)
client.run("remove Hello/0.1@lasote/testing -p=%s -f" %
NO_SETTINGS_PACKAGE_ID)
client.run("upload * --all --confirm")
def remove_packages(self, package_layout, ids_filter=None):
if not ids_filter: # Remove all
path = package_layout.packages()
# Necessary for short_paths removal
for package in package_layout.conan_packages():
self._remove(os.path.join(path, package), package_layout.ref,
"package folder:%s" % package)
self._remove(path, package_layout.ref, "packages")
self._remove_file(package_layout.system_reqs(), package_layout.ref,
SYSTEM_REQS)
else:
for id_ in ids_filter: # remove just the specified packages
pref = PackageReference(package_layout.ref, id_)
if not package_layout.package_exists(pref):
raise PackageNotFoundException(pref)
pkg_folder = package_layout.package(pref)
self._remove(pkg_folder, package_layout.ref,
"package:%s" % id_)
self._remove_file(pkg_folder + ".dirty", package_layout.ref,
"dirty flag")
self._remove_file(package_layout.system_reqs_package(pref),
package_layout.ref,
"%s/%s" % (id_, SYSTEM_REQS))
def upload(self, conan_reference_or_pattern, package_id=None, remote=None, all_packages=None,
force=False, confirm=False, retry=0, retry_wait=0, skip_upload=False,
integrity_check=False):
"""If package_id is provided, conan_reference_or_pattern is a ConanFileReference"""
t1 = time.time()
remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager,
remote)
uploader = ConanUploader(self._client_cache, self._user_io, remote_proxy,
self._search_manager)
if package_id: # Upload package
ref = ConanFileReference.loads(conan_reference_or_pattern)
uploader.check_reference(ref)
uploader.upload_package(PackageReference(ref, package_id), retry=retry,
retry_wait=retry_wait, skip_upload=skip_upload,
integrity_check=integrity_check)
else: # Upload conans
uploader.upload(conan_reference_or_pattern, all_packages=all_packages,
force=force, confirm=confirm,
retry=retry, retry_wait=retry_wait, skip_upload=skip_upload,
integrity_check=integrity_check)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def old_server_new_client_simple_test(self):
"""Use this test as an example to try some new behavior with revisions against a server
without revisions"""
old_server = TestServer(server_capabilities=[])
users = {"old_server": [("lasote", "mypass")]}
client = TestClient(servers={"old_server": old_server}, users=users)
conanfile = '''
from conans import ConanFile
class HelloConan(ConanFile):
def build(self):
self.output.warn("Revision 1")
'''
ref = ConanFileReference.loads("lib/1.0@lasote/testing")
client.save({"conanfile.py": conanfile + "\n"})
client.run("create . %s" % str(ref))
client.run("upload %s -r=old_server --all" % str(ref))
client.run("remove %s -f" % str(ref))
client.run("install %s" % str(ref))
package_ref = PackageReference(ref.copy_with_rev(DEFAULT_REVISION_V1),
NO_SETTINGS_PACKAGE_ID)
prev2 = client.servers["old_server"].paths.get_last_package_revision(package_ref)
self.assertEquals(prev2.revision, DEFAULT_REVISION_V1)
def _compute_private_nodes(self, deps_graph, build_mode):
""" computes a list of nodes that are not required to be built, as they are
private requirements of already available shared libraries as binaries
"""
private_closure = deps_graph.private_nodes()
skippable_nodes = []
for private_node, private_requirers in private_closure:
for private_requirer in private_requirers:
conan_ref, conan_file = private_requirer
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
package_folder = self._paths.package(package_reference)
if not path_exists(package_folder, self._paths.store):
if not self._force_build(
conan_ref, build_mode): # Not download package
self._user_io.out.info(
'Package for %s does not exist' % str(conan_ref))
if not self._retrieve_remote_package(
package_reference):
break
else:
skippable_nodes.append(private_node)
return skippable_nodes
def _get_local_infos_min(package_layout):
result = OrderedDict()
package_ids = package_layout.package_ids()
for package_id in package_ids:
# Read conaninfo
pref = PackageReference(package_layout.ref, package_id)
info_path = os.path.join(package_layout.package(pref), CONANINFO)
if not os.path.exists(info_path):
logger.error("There is no ConanInfo: %s" % str(info_path))
continue
conan_info_content = load(info_path)
info = ConanInfo.loads(conan_info_content)
if package_layout.ref.revision:
metadata = package_layout.load_metadata()
recipe_revision = metadata.packages[package_id].recipe_revision
if recipe_revision and recipe_revision != package_layout.ref.revision:
continue
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
return result
def test_excluded_repo_fies(self):
conanfile = base_svn.format(url=_quoted("auto"), revision="auto")
conanfile = conanfile.replace("short_paths = True", "short_paths = False")
# SVN ignores pyc files by default:
# http://blogs.collab.net/subversion/repository-dictated-configuration-day-3-global-ignores
project_url, _ = self.create_project(files={"myfile": "contents",
"ignored.pyc": "bin",
# ".gitignore": "*.pyc\n",
"myfile.txt": "My file!",
"conanfile.py": conanfile})
project_url = project_url.replace(" ", "%20")
self.client.run_command('svn co "{url}" "{path}"'.format(url=project_url,
path=self.client.current_folder))
self.client.run("create . user/channel")
self.assertIn("Copying sources to build folder", self.client.out)
pref = PackageReference(ConanFileReference.loads("lib/0.1@user/channel"),
NO_SETTINGS_PACKAGE_ID)
bf = self.client.cache.package_layout(pref.ref).build(pref)
self.assertTrue(os.path.exists(os.path.join(bf, "myfile.txt")))
self.assertTrue(os.path.exists(os.path.join(bf, "myfile")))
self.assertTrue(os.path.exists(os.path.join(bf, ".svn")))
self.assertFalse(os.path.exists(os.path.join(bf, "ignored.pyc")))
def _build(self, nodes_to_process, deps_graph, skip_nodes,
profile_build_requires, keep_build, root_conanfile):
""" The build assumes an input of conans ordered by degree, first level
should be independent from each other, the next-second level should have
dependencies only to first level conans.
param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...]
"""
inverse = deps_graph.inverse_levels()
flat = []
for level in inverse:
level = sorted(level, key=lambda x: x.conan_ref)
flat.extend(n for n in level if n not in skip_nodes)
for conan_ref, package_id, conan_file, build_needed in nodes_to_process:
output = ScopedOutput(str(conan_ref), self._out)
package_ref = PackageReference(conan_ref, package_id)
package_folder = self._client_cache.package(
package_ref, conan_file.short_paths)
if build_needed and (conan_ref,
package_id) not in self._built_packages:
self._build_package(conan_file, conan_ref, package_id,
package_ref, output, keep_build,
profile_build_requires, flat, deps_graph)
else:
# Get the package, we have a not outdated remote package
with self._client_cache.package_lock(package_ref):
self._get_remote_package(conan_file, package_ref, output,
package_folder)
self._propagate_info(conan_file, conan_ref, flat, deps_graph)
# Call the info method
self._call_package_info(conan_file, package_folder)
# Finally, propagate information to root node (conan_ref=None)
self._propagate_info(root_conanfile, None, flat, deps_graph)
def remove_packages_test(self):
conan_ref = ConanFileReference.loads(
"MySecondConan/2.0.0@private_user/testing")
self._upload_recipe(conan_ref)
folders = {}
for sha in ["1", "2", "3", "4", "5"]:
# Upload an package
package_ref = PackageReference(conan_ref, sha)
self._upload_package(package_ref)
folder = self.server.search_manager._paths.package(package_ref)
self.assertTrue(os.path.exists(folder))
folders[sha] = folder
self.api.remove_packages(conan_ref, ["1"])
self.assertTrue(
os.path.exists(self.server.search_manager._paths.conan(conan_ref)))
self.assertFalse(os.path.exists(folders["1"]))
self.assertTrue(os.path.exists(folders["2"]))
self.assertTrue(os.path.exists(folders["3"]))
self.assertTrue(os.path.exists(folders["4"]))
self.assertTrue(os.path.exists(folders["5"]))
self.api.remove_packages(conan_ref, ["2", "3"])
self.assertTrue(
os.path.exists(self.server.search_manager._paths.conan(conan_ref)))
self.assertFalse(os.path.exists(folders["1"]))
self.assertFalse(os.path.exists(folders["2"]))
self.assertFalse(os.path.exists(folders["3"]))
self.assertTrue(os.path.exists(folders["4"]))
self.assertTrue(os.path.exists(folders["5"]))
self.api.remove_packages(conan_ref, [])
self.assertTrue(
os.path.exists(self.server.search_manager._paths.conan(conan_ref)))
for sha in ["1", "2", "3", "4", "5"]:
self.assertFalse(os.path.exists(folders[sha]))
def test_build_folders(self):
client = TestClient()
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "Hello"
version = "0.1"
settings = "os"
def package(self):
self.copy("*.h", src="include", dst="inc")
self.copy("*.lib", src="lib", dst="lib")
"""
client.save(
{
CONANFILE: conanfile,
"include/header.h": "//Windows header",
"include/header.txt": "",
"libs/what": "",
"lib/hello.lib": "My Lib",
"lib/bye.txt": ""
},
clean_first=True)
client.run(
"export-pkg . Hello/0.1@lasote/stable -s os=Windows --build-folder=."
)
conan_ref = ConanFileReference.loads("Hello/0.1@lasote/stable")
package_ref = PackageReference(
conan_ref, "3475bd55b91ae904ac96fde0f106a136ab951a5e")
package_folder = client.client_cache.package(package_ref)
inc = os.path.join(package_folder, "inc")
self.assertEqual(os.listdir(inc), ["header.h"])
self.assertEqual(load(os.path.join(inc, "header.h")),
"//Windows header")
lib = os.path.join(package_folder, "lib")
self.assertEqual(os.listdir(lib), ["hello.lib"])
self.assertEqual(load(os.path.join(lib, "hello.lib")), "My Lib")
def _upload(self, conan_file, conan_ref, force, packages_ids, retry, retry_wait, skip_upload,
integrity_check, no_overwrite, remote_name, recorder):
"""Uploads the recipes and binaries identified by conan_ref"""
defined_remote = self._registry.get_ref(conan_ref)
if remote_name: # If remote_name is given, use it
upload_remote = self._registry.remote(remote_name)
elif defined_remote: # Else, if the package had defined a remote, use it
upload_remote = defined_remote
else: # Or use the default otherwise
upload_remote = self._registry.default_remote
if not force:
self._check_recipe_date(conan_ref, upload_remote)
self._user_io.out.info("Uploading %s to remote '%s'"
% (str(conan_ref), upload_remote.name))
self._upload_recipe(conan_ref, retry, retry_wait, skip_upload, no_overwrite, upload_remote)
recorder.add_recipe(str(conan_ref), upload_remote.name, upload_remote.url)
if packages_ids:
# Can't use build_policy_always here because it's not loaded (only load_class)
if conan_file.build_policy == "always":
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
total = len(packages_ids)
for index, package_id in enumerate(packages_ids):
ret_upload_package = self._upload_package(PackageReference(conan_ref, package_id),
index + 1, total, retry, retry_wait,
skip_upload, integrity_check,
no_overwrite, upload_remote)
if ret_upload_package:
recorder.add_package(str(conan_ref), package_id)
if not defined_remote and not skip_upload:
self._registry.set_ref(conan_ref, upload_remote)
def test_remove_old_sources():
# https://github.com/conan-io/conan/issues/1841
test_server = TestServer()
def upload(header_content):
c = TestClient(servers={"default": test_server},
users={"default": [("lasote", "mypass")]})
base = textwrap.dedent('''
from conans import ConanFile
class ConanLib(ConanFile):
exports_sources = "*"
def package(self):
self.copy("*")
''')
c.save({"conanfile.py": base, "header.h": header_content})
c.run("create . Pkg/0.1@lasote/channel")
c.run("upload * --confirm --all")
return c
client = upload("mycontent1")
time.sleep(1)
upload("mycontent2")
client.run("install Pkg/0.1@lasote/channel -u")
if client.cache.config.revisions_enabled:
# The binary package is not updated but downloaded, because the local one we have
# belongs to a different revision and it is removed
assert "Pkg/0.1@lasote/channel:%s - Download" % NO_SETTINGS_PACKAGE_ID in client.out
else:
assert "Pkg/0.1@lasote/channel:%s - Update" % NO_SETTINGS_PACKAGE_ID in client.out
assert "Pkg/0.1@lasote/channel: Retrieving package %s" % NO_SETTINGS_PACKAGE_ID in client.out
ref = ConanFileReference.loads("Pkg/0.1@lasote/channel")
pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID)
header = os.path.join(
client.cache.package_layout(pref.ref).package(pref), "header.h")
assert load(header) == "mycontent2"
def print_graph(deps_graph, out):
all_nodes = []
ids = set()
for node in sorted(n for n in deps_graph.nodes if n.conan_ref):
package_id = PackageReference(node.conan_ref, node.conanfile.info.package_id())
if package_id not in ids:
all_nodes.append(node)
ids.add(package_id)
requires = [n for n in all_nodes if not n.build_require]
out.writeln("Requirements", Color.BRIGHT_YELLOW)
def _recipes(nodes):
for node in nodes:
if node.remote == WORKSPACE_FILE:
from_text = "from '%s'" % WORKSPACE_FILE
else:
from_text = "from local cache" if not node.remote else "from '%s'" % node.remote.name
out.writeln(" %s %s - %s" % (repr(node.conan_ref), from_text, node.recipe), Color.BRIGHT_CYAN)
_recipes(requires)
out.writeln("Packages", Color.BRIGHT_YELLOW)
def _packages(nodes):
for node in nodes:
ref, conanfile = node.conan_ref, node.conanfile
ref = PackageReference(ref, conanfile.info.package_id())
out.writeln(" %s - %s" % (repr(ref), node.binary), Color.BRIGHT_CYAN)
_packages(requires)
build_requires = [n for n in all_nodes if n.build_require]
if build_requires:
out.writeln("Build requirements", Color.BRIGHT_YELLOW)
_recipes(build_requires)
out.writeln("Build requirements packages", Color.BRIGHT_YELLOW)
_packages(build_requires)
out.writeln("")
def test_build_folders(self):
client = TestClient()
conanfile = """
from conans import ConanFile
class TestConan(ConanFile):
name = "Hello"
version = "0.1"
settings = "os"
def package(self):
self.copy("*.h", src="include", dst="inc")
self.copy("*.lib", src="lib", dst="lib")
"""
client.save(
{
CONANFILE: conanfile,
"include/header.h": "//Windows header",
"include/header.txt": "",
"libs/what": "",
"lib/hello.lib": "My Lib",
"lib/bye.txt": ""
},
clean_first=True)
client.run(
"export-pkg . Hello/0.1@lasote/stable -s os=Windows --build-folder=."
)
package_id = re.search(r"Packaging to (\S+)", str(client.out)).group(1)
ref = ConanFileReference.loads("Hello/0.1@lasote/stable")
pref = PackageReference(ref, package_id)
package_folder = client.cache.package_layout(pref.ref).package(pref)
inc = os.path.join(package_folder, "inc")
self.assertEqual(os.listdir(inc), ["header.h"])
self.assertEqual(load(os.path.join(inc, "header.h")),
"//Windows header")
lib = os.path.join(package_folder, "lib")
self.assertEqual(os.listdir(lib), ["hello.lib"])
self.assertEqual(load(os.path.join(lib, "hello.lib")), "My Lib")
def propagate_info(self):
""" takes the exports from upper level and updates the imports
right now also the imports are propagated, but should be checked
E.g. Conan A, depends on B. A=>B
B exports an include directory "my_dir", with root "/...../0123efe"
A imports are the exports of B, plus any other conans it depends on
A.imports.include_dirs = B.export.include_paths.
Note the difference, include_paths used to compute full paths as the user
defines export relative to its folder
"""
ordered = self.by_levels()
for level in ordered:
for node in level:
_, conanfile = node
neighbors = self.neighbors(node)
direct_reqs = [] # of PackageReference
indirect_reqs = set() # of PackageReference, avoid duplicates
for nref, nconan in neighbors:
package_id = nconan.info.package_id()
package_reference = PackageReference(nref, package_id)
direct_reqs.append(package_reference)
indirect_reqs.update(nconan.info.requires.refs())
conanfile.options.propagate_downstream(nref, nconan.info.full_options)
# Might be never used, but update original requirement, just in case
conanfile.requires[nref.name].conan_reference = nref
# There might be options that are not upstream
conanfile.options.clear_unused(indirect_reqs.union(direct_reqs))
conanfile.info = ConanInfo.create(conanfile.settings.values,
conanfile.options.values,
direct_reqs)
conanfile.info.requires.add(indirect_reqs)
conanfile.info.full_requires.extend(indirect_reqs)
# Once we are done, call conan_info() to narrow and change possible values
conanfile.conan_info()
return ordered
def upload(self,
conan_reference,
package_id=None,
remote=None,
all_packages=None,
force=False):
t1 = time.time()
remote_proxy = ConanProxy(self._client_cache, self._user_io,
self._remote_manager, remote)
uploader = ConanUploader(self._client_cache, self._user_io,
remote_proxy)
# Load conanfile to check if the build policy is set to always
try:
conanfile_path = self._client_cache.conanfile(conan_reference)
conan_file = self._loader().load_class(conanfile_path)
except NotFoundException:
raise NotFoundException(
"There is no local conanfile exported as %s" %
str(conan_reference))
# Can't use build_policy_always here because it's not loaded (only load_class)
if conan_file.build_policy == "always" and (all_packages
or package_id):
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
if package_id: # Upload package
uploader.upload_package(
PackageReference(conan_reference, package_id))
else: # Upload conans
uploader.upload_conan(conan_reference,
all_packages=all_packages,
force=force)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def _build(self, nodes_to_process, deps_graph, skip_nodes,
profile_build_requires, keep_build, root_node, update):
""" The build assumes an input of conans ordered by degree, first level
should be independent from each other, the next-second level should have
dependencies only to first level conans.
param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...]
"""
inverse_levels = deps_graph.inverse_levels()
for level in inverse_levels:
level[:] = [n for n in level if n not in skip_nodes]
for node, package_id, build_needed in nodes_to_process:
conan_ref, conan_file = node.conan_ref, node.conanfile
output = ScopedOutput(str(conan_ref), self._out)
package_ref = PackageReference(conan_ref, package_id)
package_folder = self._client_cache.package(
package_ref, conan_file.short_paths)
with self._client_cache.package_lock(package_ref):
set_dirty(package_folder)
if build_needed and (conan_ref,
package_id) not in self._built_packages:
self._build_package(node, package_id, package_ref, output,
keep_build, profile_build_requires,
inverse_levels, deps_graph, update)
else:
self._get_existing_package(conan_file, package_ref, output,
package_folder, update)
self._propagate_info(node, inverse_levels, deps_graph)
# Call the info method
self._call_package_info(conan_file, package_folder)
clean_dirty(package_folder)
# Finally, propagate information to root node (conan_ref=None)
self._propagate_info(root_node, inverse_levels, deps_graph)
def upload_dirty_test(self):
test_server = TestServer([], users={"lasote": "mypass"})
client = TestClient(servers={"default": test_server},
users={"default": [("lasote", "mypass")]})
client.save({"conanfile.py": str(TestConanFile())})
client.run("export . lasote/testing")
ref = ConanFileReference.loads("Hello/0.1@lasote/testing")
pref = PackageReference(ref, "12345")
package_folder = client.cache.package(pref)
recipe_rev = client.get_revision(ref)
p_rev = client.get_package_revision(pref)
with client.cache.package_layout(pref.ref).update_metadata() as metadata:
metadata.packages[pref.id].revision = p_rev
metadata.packages[pref.id].recipe_revision = recipe_rev
save(os.path.join(package_folder, "conanmanifest.txt"), "888")
set_dirty(package_folder)
client.run("upload * --all --confirm", assert_error=True)
self.assertIn("ERROR: Package Hello/0.1@lasote/testing:12345 is corrupted, aborting upload",
client.out)
self.assertIn("Remove it with 'conan remove Hello/0.1@lasote/testing -p=12345'", client.out)
client.run("remove Hello/0.1@lasote/testing -p=12345 -f")
client.run("upload * --all --confirm")
def search_test(self):
# Upload a conan1
conan_name1 = "HelloOnly/0.10@private_user/testing"
conan_reference1 = ConanFileReference.loads(conan_name1)
self._upload_recipe(conan_reference1)
# Upload a package
conan_info = """[settings]
arch=x86_64
compiler=gcc
os=Linux
[options]
386=False
[requires]
Hello
Bye/2.9
Say/2.1@user/testing
Chat/2.1@user/testing:SHA_ABC
"""
package_reference = PackageReference(conan_reference1, "1F23223EFDA")
self._upload_package(package_reference, {CONANINFO: conan_info})
# Upload a conan2
conan_name2 = "helloonlyToo/2.1@private_user/stable"
conan_reference2 = ConanFileReference.loads(conan_name2)
self._upload_recipe(conan_reference2)
# Get the info about this ConanFileReference
info = self.api.search_packages(conan_reference1, None)
self.assertEqual(
ConanInfo.loads(conan_info).serialize_min(), info["1F23223EFDA"])
# Search packages
results = self.api.search("HelloOnly*", ignorecase=False)
self.assertEqual(results, [conan_reference1])
def test_migration_tgz_location(self):
client = TestClient(default_server_user=True)
conanfile = textwrap.dedent("""
from conans import ConanFile
class Pkg(ConanFile):
exports = "*.txt"
exports_sources = "*.h"
""")
client.save({
"conanfile.py": conanfile,
"file.h": "contents",
"file.txt": "contents"
})
client.run("create . pkg/1.0@")
ref = ConanFileReference.loads("pkg/1.0")
layout = client.cache.package_layout(ref)
export_tgz = os.path.join(layout.export(), EXPORT_TGZ_NAME)
export_src_tgz = os.path.join(layout.export(), EXPORT_SOURCES_TGZ_NAME)
pref = PackageReference(ref, NO_SETTINGS_PACKAGE_ID)
pkg_tgz = os.path.join(layout.package(pref), PACKAGE_TGZ_NAME)
save(export_tgz, "")
save(export_src_tgz, "")
save(pkg_tgz, "")
self.assertTrue(os.path.isfile(export_tgz))
self.assertTrue(os.path.isfile(export_src_tgz))
self.assertTrue(os.path.isfile(pkg_tgz))
client2 = TestClient(client.cache_folder)
save(os.path.join(client.cache_folder, "version.txt"), "1.30.0")
client2.run("search")
self.assertIn(
"Removing temporary .tgz files, they are stored in a different location now",
client2.out)
self.assertFalse(os.path.isfile(export_tgz))
self.assertFalse(os.path.isfile(export_src_tgz))
self.assertFalse(os.path.isfile(pkg_tgz))
