def _get_recipe(self, layout, ref, check_updates, update, remotes,
recorder):
output = ScopedOutput(str(ref), self._out)
# check if it is in disk
conanfile_path = layout.conanfile()
# NOT in disk, must be retrieved from remotes
if not os.path.exists(conanfile_path):
remote, new_ref = self._download_recipe(layout, ref, output,
remotes, remotes.selected,
recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
metadata = layout.load_metadata()
cur_revision = metadata.recipe.revision
cur_remote = metadata.recipe.remote
cur_remote = remotes[cur_remote] if cur_remote else None
selected_remote = remotes.selected or cur_remote
check_updates = check_updates or update
requested_different_revision = (
ref.revision is not None) and cur_revision != ref.revision
if requested_different_revision:
if check_updates or self._cache.new_config[
"core:allow_explicit_revision_update"]:
remote, new_ref = self._download_recipe(
layout, ref, output, remotes, selected_remote, recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
else:
raise NotFoundException(
"The '%s' revision recipe in the local cache doesn't "
"match the requested '%s'."
" Use '--update' to check in the remote." %
(cur_revision, repr(ref)))
if not check_updates:
status = RECIPE_INCACHE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, cur_remote, ref
# Checking updates in the server
if not selected_remote:
status = RECIPE_NO_REMOTE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, None, ref
try: # get_recipe_manifest can fail, not in server
upstream_manifest, ref = self._remote_manager.get_recipe_manifest(
ref, selected_remote)
except NotFoundException:
status = RECIPE_NOT_IN_REMOTE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, selected_remote, ref
read_manifest = layout.recipe_manifest()
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
if update:
DiskRemover().remove_recipe(layout, output=output)
output.info("Retrieving from remote '%s'..." %
selected_remote.name)
self._download_recipe(layout, ref, output, remotes,
selected_remote, recorder)
status = RECIPE_UPDATED
return conanfile_path, status, selected_remote, ref
else:
status = RECIPE_UPDATEABLE
else:
status = RECIPE_NEWER
else:
status = RECIPE_INCACHE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, selected_remote, ref
def create(self, conanfile_path, name=None, version=None, user=None, channel=None,
profile_name=None, settings=None,
options=None, env=None, test_folder=None, not_export=False,
build_modes=None,
keep_source=False, verify=None,
manifests=None, manifests_interactive=None,
remote=None, update=False, cwd=None):
settings = settings or []
options = options or []
env = env or []
cwd = cwd or os.getcwd()
conanfile_path = _get_conanfile_path(conanfile_path, cwd, py=True)
if not name or not version:
conanfile = load_conanfile_class(conanfile_path)
name, version = conanfile.name, conanfile.version
if not name or not version:
raise ConanException("conanfile.py doesn't declare package name or version")
reference = ConanFileReference(name, version, user, channel)
scoped_output = ScopedOutput(str(reference), self._user_io.out)
# Forcing an export!
if not not_export:
scoped_output.highlight("Exporting package recipe")
self._manager.export(conanfile_path, name, version, user, channel, keep_source)
if build_modes is None: # Not specified, force build the tested library
build_modes = [name]
manifests = _parse_manifests_arguments(verify, manifests, manifests_interactive, cwd)
manifest_folder, manifest_interactive, manifest_verify = manifests
profile = profile_from_args(profile_name, settings, options, env,
cwd, self._client_cache)
def get_test_conanfile_path(tf):
"""Searchs in the declared test_folder or in the standard locations"""
test_folders = [tf] if tf else ["test_package", "test"]
base_folder = os.path.dirname(conanfile_path)
for test_folder_name in test_folders:
test_folder = os.path.join(base_folder, test_folder_name)
test_conanfile_path = os.path.join(test_folder, "conanfile.py")
if os.path.exists(test_conanfile_path):
return test_conanfile_path
else:
if tf:
raise ConanException("test folder '%s' not available, "
"or it doesn't have a conanfile.py" % tf)
test_conanfile_path = get_test_conanfile_path(test_folder)
if test_conanfile_path:
pt = PackageTester(self._manager, self._user_io)
pt.install_build_and_test(test_conanfile_path, reference, profile,
remote, update, build_modes=build_modes,
manifest_folder=manifest_folder,
manifest_verify=manifest_verify,
manifest_interactive=manifest_interactive)
else:
self._manager.install(reference=reference,
install_folder=None, # Not output anything
manifest_folder=manifest_folder,
manifest_verify=manifest_verify,
manifest_interactive=manifest_interactive,
remote=remote,
profile=profile,
build_modes=build_modes,
update=update)
def install(self,
reference,
install_folder,
profile,
remote=None,
build_modes=None,
filename=None,
update=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False,
inject_require=None,
cwd=None,
deploy=False):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param install_folder: where the output files will be saved
@param remote: install only from that remote
@param profile: Profile object with both the -s introduced options and profile read values
@param build_modes: List of build_modes specified
@param filename: Optional filename of the conanfile
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user
for confirmation
@param generators: List of generators from command line. If False, no generator will be
written
@param no_imports: Install specified packages but avoid running imports
@param inject_require: Reference to add as a requirement to the conanfile
@param cwd: Only used in case of reference, to get a conanfile_directory to a virtual SMELL
"""
if generators is not False:
generators = set(generators) if generators else set()
generators.add("txt") # Add txt generator by default
manifest_manager = ManifestManager(
manifest_folder,
user_io=self._user_io,
client_cache=self._client_cache,
verify=manifest_verify,
interactive=manifest_interactive) if manifest_folder else None
remote_proxy = ConanProxy(self._client_cache,
self._user_io,
self._remote_manager,
remote,
update=update,
manifest_manager=manifest_manager)
loader = self.get_loader(profile)
conanfile = self._get_conanfile_object(loader,
reference,
filename,
cwd=cwd)
if inject_require:
self._inject_require(conanfile, inject_require)
graph_builder = self._get_graph_builder(loader, update, remote_proxy)
deps_graph = graph_builder.load(conanfile)
# This line is so the conaninfo stores the correct complete info
conanfile.info.scope = profile.scopes
registry = RemoteRegistry(self._client_cache.registry,
self._user_io.out)
if inject_require:
output = ScopedOutput("%s test package" % str(inject_require),
self._user_io.out)
output.info("Installing dependencies")
else:
if not isinstance(reference, ConanFileReference):
output = ScopedOutput("PROJECT", self._user_io.out)
output.highlight("Installing %s" % reference)
else:
output = ScopedOutput(str(reference), self._user_io.out)
output.highlight("Installing package")
Printer(self._user_io.out).print_graph(deps_graph, registry)
try:
if loader._settings.os and detected_os() != loader._settings.os:
message = "Cross-platform from '%s' to '%s'" % (
detected_os(), loader._settings.os)
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
build_mode = BuildMode(build_modes, self._user_io.out)
build_requires = BuildRequires(loader, graph_builder, registry, output,
profile.build_requires)
installer = ConanInstaller(self._client_cache, output, remote_proxy,
build_mode, build_requires)
# Apply build_requires to consumer conanfile
if not isinstance(reference, ConanFileReference):
build_requires.install("", conanfile, installer)
installer.install(deps_graph)
build_mode.report_matches()
if install_folder:
# Write generators
if generators is not False:
tmp = list(conanfile.generators
) # Add the command line specified generators
tmp.extend([g for g in generators if g not in tmp])
conanfile.generators = tmp
write_generators(conanfile, install_folder, output)
# Write conaninfo
content = normalize(conanfile.info.dumps())
save(os.path.join(install_folder, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, install_folder, output)
call_system_requirements(conanfile, output)
if deploy:
# The conanfile loaded is really a virtual one. The one with the deploy is the first level one
deploy_conanfile = deps_graph.inverse_levels()[1][0].conanfile
if hasattr(deploy_conanfile, "deploy") and callable(
deploy_conanfile.deploy):
run_deploy(deploy_conanfile, install_folder, output)
if manifest_manager:
manifest_manager.print_log()
def _build(self, nodes_by_level, skip_private_nodes, build_mode):
""" The build assumes an input of conans ordered by degree, first level
should be independent from each other, the next-second level should have
dependencies only to first level conans.
param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...]
build_mode => ["*"] if user wrote "--build"
=> ["hello*", "bye*"] if user wrote "--build hello --build bye"
=> False if user wrote "never"
=> True if user wrote "missing"
=> "outdated" if user wrote "--build outdated"
"""
inverse = self._deps_graph.inverse_levels()
flat = []
for level in inverse:
level = sorted(level, key=lambda x: x.conan_ref)
flat.extend(level)
# Get the nodes in order and if we have to build them
nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes,
build_mode)
for conan_ref, package_id, conan_file, build_needed in nodes_to_process:
if build_needed:
build_allowed = self._build_allowed(conan_ref, build_mode,
conan_file)
if not build_allowed:
self._raise_package_not_found_error(conan_ref, conan_file)
output = ScopedOutput(str(conan_ref), self._out)
package_ref = PackageReference(conan_ref, package_id)
package_folder = self._client_cache.package(
package_ref, conan_file.short_paths)
if build_mode is True:
output.info(
"Building package from source as defined by build_policy='missing'"
)
elif self._build_forced(conan_ref, build_mode, conan_file):
output.warn('Forced build from source')
t1 = time.time()
# Assign to node the propagated info
self._propagate_info(conan_ref, conan_file, flat)
self._remote_proxy.get_recipe_sources(conan_ref)
# Call the conanfile's build method
build_folder = self._build_conanfile(conan_ref, conan_file,
package_ref,
package_folder, output)
# Call the conanfile's package method
self._package_conanfile(conan_ref, conan_file, package_ref,
build_folder, package_folder, output)
# Call the info method
self._package_info_conanfile(conan_ref, conan_file)
duration = time.time() - t1
log_file = os.path.join(build_folder, RUN_LOG_NAME)
log_file = log_file if os.path.exists(log_file) else None
log_package_built(package_ref, duration, log_file)
else:
# Get the package, we have a not outdated remote package
if conan_ref:
self._get_package(conan_ref, conan_file)
# Assign to the node the propagated info
# (conan_ref could be None if user project, but of course assign the info
self._propagate_info(conan_ref, conan_file, flat)
# Call the info method
self._package_info_conanfile(conan_ref, conan_file)
def build(self,
conanfile_path,
source_folder,
build_folder,
package_folder,
install_folder,
test=False,
should_configure=True,
should_build=True,
should_install=True,
should_test=True):
""" Call to build() method saved on the conanfile.py
param conanfile_path: path to a conanfile.py
"""
logger.debug("Building in %s" % build_folder)
logger.debug("Conanfile in %s" % conanfile_path)
try:
# Append env_vars to execution environment and clear when block code ends
output = ScopedOutput(
("%s (test package)" % test) if test else "Project",
self._user_io.out)
conan_file = self._graph_manager.load_consumer_conanfile(
conanfile_path,
install_folder,
output,
deps_info_required=True)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file" %
(CONANFILE, CONANFILE, CONANFILE_TXT))
if test:
try:
conan_file.requires.add(test)
except ConanException:
pass
conan_file.should_configure = should_configure
conan_file.should_build = should_build
conan_file.should_install = should_install
conan_file.should_test = should_test
try:
mkdir(build_folder)
os.chdir(build_folder)
conan_file.build_folder = build_folder
conan_file.source_folder = source_folder
conan_file.package_folder = package_folder
conan_file.install_folder = install_folder
self._plugin_manager.execute("pre_build",
conanfile=conan_file,
conanfile_path=conanfile_path)
with get_env_context_manager(conan_file):
output.highlight("Running build()")
with conanfile_exception_formatter(str(conan_file), "build"):
conan_file.build()
self._plugin_manager.execute("post_build",
conanfile=conan_file,
conanfile_path=conanfile_path)
if test:
output.highlight("Running test()")
with conanfile_exception_formatter(str(conan_file),
"test"):
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" %
'\n'.join(trace[3:]))
def _build(self, nodes_by_level, skip_private_nodes, deps_graph):
""" The build assumes an input of conans ordered by degree, first level
should be independent from each other, the next-second level should have
dependencies only to first level conans.
param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...]
build_mode => ["*"] if user wrote "--build"
=> ["hello*", "bye*"] if user wrote "--build hello --build bye"
=> False if user wrote "never"
=> True if user wrote "missing"
=> "outdated" if user wrote "--build outdated"
"""
inverse = deps_graph.inverse_levels()
flat = []
for level in inverse:
level = sorted(level, key=lambda x: x.conan_ref)
flat.extend(level)
# Get the nodes in order and if we have to build them
nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes)
for conan_ref, package_id, conan_file, build_needed in nodes_to_process:
output = ScopedOutput(str(conan_ref), self._out)
package_ref = PackageReference(conan_ref, package_id)
if build_needed and (conan_ref,
package_id) not in self._built_packages:
build_allowed = self._build_mode.allowed(conan_file, conan_ref)
if not build_allowed:
_raise_package_not_found_error(conan_file, conan_ref,
output)
if conan_file.build_policy_missing:
output.info(
"Building package from source as defined by build_policy='missing'"
)
elif self._build_mode.forced(conan_file, conan_ref):
output.warn('Forced build from source')
self._build_requires.install(conan_ref, conan_file, self)
t1 = time.time()
# Assign to node the propagated info
self._propagate_info(conan_file, conan_ref, flat, deps_graph)
self._remote_proxy.get_recipe_sources(conan_ref,
conan_file.short_paths)
builder = _ConanPackageBuilder(conan_file, package_ref,
self._client_cache, output)
builder.build()
builder.package()
self._remote_proxy.handle_package_manifest(package_ref,
installed=True)
# Call the info method
call_package_info(conan_file)
# Log build
self._log_built_package(conan_file, package_ref,
time.time() - t1)
self._built_packages.add((conan_ref, package_id))
else:
# Get the package, we have a not outdated remote package
if conan_ref:
self.get_remote_package(conan_file, package_ref, output)
# Assign to the node the propagated info
# (conan_ref could be None if user project, but of course assign the info
self._propagate_info(conan_file, conan_ref, flat, deps_graph)
# Call the info method
call_package_info(conan_file)
def _evaluate_node(self, node, build_mode, update, evaluated_references,
remote_name):
assert node.binary is None
conan_ref, conanfile = node.conan_ref, node.conanfile
package_id = conanfile.info.package_id()
package_ref = PackageReference(conan_ref, package_id)
# Check that this same reference hasn't already been checked
previous_node = evaluated_references.get(package_ref)
if previous_node:
node.binary = previous_node.binary
node.binary_remote = previous_node.binary_remote
return
evaluated_references[package_ref] = node
output = ScopedOutput(str(conan_ref), self._out)
if build_mode.forced(conanfile, conan_ref):
output.warn('Forced build from source')
node.binary = BINARY_BUILD
return
package_folder = self._client_cache.package(
package_ref, short_paths=conanfile.short_paths)
# Check if dirty, to remove it
local_project = self._workspace[conan_ref] if self._workspace else None
if local_project:
node.binary = BINARY_WORKSPACE
return
with self._client_cache.package_lock(package_ref):
if is_dirty(package_folder):
output.warn("Package is corrupted, removing folder: %s" %
package_folder)
rmdir(package_folder)
if remote_name:
remote = self._registry.remote(remote_name)
else:
remote = self._registry.get_ref(conan_ref)
remotes = self._registry.remotes
if os.path.exists(package_folder):
if update:
if remote:
if self._check_update(package_folder, package_ref, remote,
output):
node.binary = BINARY_UPDATE
if build_mode.outdated:
package_hash = self._get_package_info(
package_ref, remote).recipe_hash
elif remotes:
pass
else:
output.warn("Can't update, no remote defined")
if not node.binary:
node.binary = BINARY_CACHE
package_hash = ConanInfo.load_file(
os.path.join(package_folder, CONANINFO)).recipe_hash
else: # Binary does NOT exist locally
remote_info = None
if remote:
remote_info = self._get_package_info(package_ref, remote)
elif remotes: # Iterate all remotes to get this binary
for r in remotes:
remote_info = self._get_package_info(package_ref, r)
if remote_info:
remote = r
break
if remote_info:
node.binary = BINARY_DOWNLOAD
package_hash = remote_info.recipe_hash
else:
if build_mode.allowed(conanfile, conan_ref):
node.binary = BINARY_BUILD
else:
node.binary = BINARY_MISSING
if build_mode.outdated:
if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE):
local_recipe_hash = self._client_cache.load_manifest(
package_ref.conan).summary_hash
if local_recipe_hash != package_hash:
output.info("Outdated package!")
node.binary = BINARY_BUILD
else:
output.info("Package is up to date")
node.binary_remote = remote
def get_recipe(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rm_conandir(self._client_cache.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path,
current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._client_cache.store)
if path_exist:
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it." %
remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
DiskRemover(
self._client_cache).remove_packages(
conan_reference)
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error(
"Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." %
(remote.name, conan_reference))
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def load_graph(self, reference, create_reference, graph_info, build_mode,
check_updates, update, remote_name, recorder, workspace):
def _inject_require(conanfile, reference):
""" test_package functionality requires injecting the tested package as requirement
before running the install
"""
require = conanfile.requires.get(reference.name)
if require:
require.conan_reference = require.range_reference = reference
else:
conanfile.requires(str(reference))
conanfile._conan_user = reference.user
conanfile._conan_channel = reference.channel
# Computing the full dependency graph
profile = graph_info.profile
cache_settings = profile.processed_settings
processed_profile = ProcessedProfile(profile, create_reference)
if isinstance(reference,
list): # Install workspace with multiple root nodes
conanfile = self._loader.load_virtual(reference, processed_profile)
elif isinstance(reference, ConanFileReference):
# create without test_package and install <ref>
conanfile = self._loader.load_virtual([reference],
processed_profile)
else:
output = ScopedOutput("PROJECT", self._output)
if reference.endswith(".py"):
conanfile = self._loader.load_conanfile(reference,
output,
processed_profile,
consumer=True)
if create_reference: # create with test_package
_inject_require(conanfile, create_reference)
else:
conanfile = self._loader.load_conanfile_txt(
reference, output, processed_profile)
build_mode = BuildMode(build_mode, self._output)
root_node = Node(None, conanfile)
deps_graph = self._load_graph(
root_node,
check_updates,
update,
build_mode=build_mode,
remote_name=remote_name,
profile_build_requires=profile.build_requires,
recorder=recorder,
workspace=workspace,
processed_profile=processed_profile)
# THIS IS NECESSARY to store dependencies options in profile, for consumer
# FIXME: This is a hack. Might dissapear if the graph for local commands is always recomputed
graph_info.options = root_node.conanfile.options.values
version_ranges_output = self._resolver.output
if version_ranges_output:
self._output.success("Version ranges solved")
for msg in version_ranges_output:
self._output.info(" %s" % msg)
self._output.writeln("")
build_mode.report_matches()
return deps_graph, conanfile, cache_settings
def package(self, reference, package_id):
# Package paths
conan_file_path = self._client_cache.conanfile(reference)
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" %
str(reference))
conanfile = load_conanfile_class(conan_file_path)
if hasattr(conanfile, "build_id"):
raise ConanException(
"package command does not support recipes with 'build_id'\n"
"To repackage them use 'conan install'")
if not package_id:
packages = [
PackageReference(reference, packid)
for packid in self._client_cache.conan_builds(reference)
]
if not packages:
raise NotFoundException(
"%s: Package has not been built in local cache\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" % str(reference))
else:
packages = [PackageReference(reference, package_id)]
package_source_folder = self._client_cache.source(
reference, conanfile.short_paths)
for package_reference in packages:
build_folder = self._client_cache.build(package_reference,
short_paths=None)
if not os.path.exists(build_folder):
raise NotFoundException(
"%s: Package binary '%s' folder doesn't exist\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" %
(str(reference), package_reference.package_id))
# The package already exist, we can use short_paths if they were defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Will read current conaninfo with specified options and load conanfile with them
output = ScopedOutput(str(reference), self._user_io.out)
output.info("Re-packaging %s" % package_reference.package_id)
conanfile = self.load_consumer_conanfile(conan_file_path,
build_folder,
output,
reference=reference)
rmdir(package_folder)
if getattr(conanfile, 'no_copy_source', False):
source_folder = package_source_folder
else:
source_folder = build_folder
with environment_append(conanfile.env):
packager.create_package(conanfile,
source_folder,
build_folder,
package_folder,
output,
copy_info=True)
def _create_new_node(self,
current_node,
dep_graph,
requirement,
public_deps,
name_req,
aliased,
check_updates,
update,
remote_name,
processed_profile,
alias_ref=None):
""" creates and adds a new node to the dependency graph
"""
output = ScopedOutput(str(requirement.conan_reference), self._output)
workspace_package = self._workspace[
requirement.conan_reference] if self._workspace else None
if workspace_package:
conanfile_path = workspace_package.conanfile_path
recipe_status = RECIPE_WORKSPACE
remote = WORKSPACE_FILE
new_ref = requirement.conan_reference
else:
try:
result = self._proxy.get_recipe(requirement.conan_reference,
check_updates, update,
remote_name, self._recorder)
except ConanException as e:
base_ref = str(current_node.conan_ref or "PROJECT")
self._output.error("Failed requirement '%s' from '%s'" %
(requirement.conan_reference, base_ref))
raise e
conanfile_path, recipe_status, remote, new_ref = result
dep_conanfile = self._loader.load_conanfile(
conanfile_path,
output,
processed_profile,
reference=requirement.conan_reference)
if workspace_package:
workspace_package.conanfile = dep_conanfile
if getattr(dep_conanfile, "alias", None):
alias_reference = alias_ref or new_ref.copy_clear_rev()
requirement.conan_reference = ConanFileReference.loads(
dep_conanfile.alias)
aliased[alias_reference] = requirement.conan_reference
return self._create_new_node(current_node,
dep_graph,
requirement,
public_deps,
name_req,
aliased,
check_updates,
update,
remote_name,
processed_profile,
alias_ref=alias_reference)
logger.debug("GRAPH: new_node: %s" % str(new_ref))
new_node = Node(new_ref, dep_conanfile)
new_node.revision_pinned = requirement.conan_reference.revision is not None
new_node.recipe = recipe_status
new_node.remote = remote
dep_graph.add_node(new_node)
dep_graph.add_edge(current_node, new_node, requirement.private)
if not requirement.private:
public_deps[name_req] = new_node, None
return new_node
def _get_recipe(self, reference, check_updates, update, remote_name, recorder):
output = ScopedOutput(str(reference), self._out)
# check if it is in disk
conanfile_path = self._client_cache.conanfile(reference)
# NOT in disk, must be retrieved from remotes
if not os.path.exists(conanfile_path):
remote, new_ref = self._download_recipe(reference, output, remote_name, recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
metadata = self._client_cache.load_metadata(reference)
cur_revision = metadata.recipe.revision
remote = self._registry.refs.get(reference)
named_remote = self._registry.remotes.get(remote_name) if remote_name else None
update_remote = named_remote or remote
# Check if we have a revision different from the requested one
revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False)
if revisions_enabled and reference.revision and cur_revision != reference.revision:
output.info("Different revision requested, removing current local recipe...")
DiskRemover(self._client_cache).remove_recipe(reference)
output.info("Retrieving from remote '%s'..." % update_remote.name)
new_ref = self._remote_manager.get_recipe(reference, update_remote)
self._registry.refs.set(new_ref, update_remote.name)
status = RECIPE_UPDATED
return conanfile_path, status, update_remote, new_ref
check_updates = check_updates or update
# Recipe exists in disk, but no need to check updates
if not check_updates:
status = RECIPE_INCACHE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, remote, ref
if not update_remote:
status = RECIPE_NO_REMOTE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, None, ref
try: # get_conan_manifest can fail, not in server
upstream_manifest = self._remote_manager.get_conan_manifest(reference, update_remote)
except NotFoundException:
status = RECIPE_NOT_IN_REMOTE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, update_remote, ref
export = self._client_cache.export(reference)
read_manifest = FileTreeManifest.load(export)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
if update:
DiskRemover(self._client_cache).remove_recipe(reference)
output.info("Retrieving from remote '%s'..." % update_remote.name)
new_ref = self._remote_manager.get_recipe(reference, update_remote)
self._registry.refs.set(new_ref, update_remote.name)
status = RECIPE_UPDATED
return conanfile_path, status, update_remote, new_ref
else:
status = RECIPE_UPDATEABLE
else:
status = RECIPE_NEWER
else:
status = RECIPE_INCACHE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, update_remote, ref
def install(self,
reference,
current_path,
profile,
remote=None,
build_mode=None,
filename=None,
update=False,
check_updates=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param current_path: where the output files will be saved
@param remote: install only from that remote
@param options: list of tuples: [(optionname, optionvalue), (optionname, optionvalue)...]
@param settings: list of tuples: [(settingname, settingvalue), (settingname, value)...]
@param package_settings: dict name=> settings: {"zlib": [(settingname, settingvalue), ...]}
@param profile: name of the profile to use
@param env: list of tuples for environment vars: [(var, value), (var2, value2)...]
@param package_env: package dict of list of tuples: {"package_name": [(var, value), (var2, value2)...]}
"""
generators = generators or []
if manifest_folder:
manifest_manager = ManifestManager(
manifest_folder,
user_io=self._user_io,
client_cache=self._client_cache,
verify=manifest_verify,
interactive=manifest_interactive)
else:
manifest_manager = None
objects = self._get_graph(reference, current_path, profile, remote,
filename, update, check_updates,
manifest_manager)
(_, deps_graph, _, registry, conanfile, remote_proxy, loader) = objects
Printer(self._user_io.out).print_graph(deps_graph, registry)
try:
if detected_os() != loader._settings.os:
message = "Cross-platform from '%s' to '%s'" % (
detected_os(), loader._settings.os)
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
installer = ConanInstaller(self._client_cache, self._user_io,
remote_proxy)
installer.install(deps_graph, build_mode)
prefix = "PROJECT" if not isinstance(
reference, ConanFileReference) else str(reference)
output = ScopedOutput(prefix, self._user_io.out)
# Write generators
tmp = list(
conanfile.generators) # Add the command line specified generators
tmp.extend(generators)
conanfile.generators = tmp
write_generators(conanfile, current_path, output)
if not isinstance(reference, ConanFileReference):
content = normalize(conanfile.info.dumps())
save(os.path.join(current_path, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, current_path, output)
installer.call_system_requirements(conanfile, output)
if manifest_manager:
manifest_manager.print_log()
def create_package(conanfile,
package_id,
source_folder,
build_folder,
package_folder,
install_folder,
hook_manager,
conanfile_path,
ref,
local=False,
copy_info=False):
""" copies built artifacts, libs, headers, data, etc. from build_folder to
package folder
"""
mkdir(package_folder)
output = conanfile.output
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % package_folder)
try:
conanfile.package_folder = package_folder
conanfile.source_folder = source_folder
conanfile.install_folder = install_folder
conanfile.build_folder = build_folder
hook_manager.execute("pre_package",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=ref,
package_id=package_id)
package_output = ScopedOutput("%s package()" % output.scope, output)
output.highlight("Calling package()")
folders = [source_folder, build_folder
] if source_folder != build_folder else [build_folder]
conanfile.copy = FileCopier(folders, package_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
with chdir(build_folder):
conanfile.package()
except Exception as e:
if not local:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s" %
(package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise
raise ConanException(e)
hook_manager.execute("post_package",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=ref,
package_id=package_id)
manifest = _create_aux_files(install_folder, package_folder, conanfile,
copy_info)
_report_files_from_manifest(package_output, manifest)
package_id = package_id or os.path.basename(package_folder)
output.success("Package '%s' created" % package_id)
prev = manifest.summary_hash
output.info("Created package revision %s" % prev)
return prev
def create(self,
conanfile_path,
name=None,
version=None,
user=None,
channel=None,
profile_name=None,
settings=None,
options=None,
env=None,
test_folder=None,
not_export=False,
build_modes=None,
keep_source=False,
keep_build=False,
verify=None,
manifests=None,
manifests_interactive=None,
remote=None,
update=False,
cwd=None,
test_build_folder=None):
"""
API method to create a conan package
:param test_folder: default None - looks for default 'test' or 'test_package' folder),
string - test_folder path
False - disabling tests
"""
settings = settings or []
options = options or []
env = env or []
try:
cwd = cwd or os.getcwd()
recorder = ActionRecorder()
conanfile_path = _get_conanfile_path(conanfile_path, cwd, py=True)
if not name or not version:
conanfile = load_conanfile_class(conanfile_path)
name, version = conanfile.name, conanfile.version
if not name or not version:
raise ConanException(
"conanfile.py doesn't declare package name or version")
reference = ConanFileReference(name, version, user, channel)
scoped_output = ScopedOutput(str(reference), self._user_io.out)
# Make sure keep_source is set for keep_build
if keep_build:
keep_source = True
# Forcing an export!
if not not_export:
scoped_output.highlight("Exporting package recipe")
cmd_export(conanfile_path, name, version, user, channel,
keep_source, self._user_io.out, self._client_cache)
if build_modes is None: # Not specified, force build the tested library
build_modes = [name]
manifests = _parse_manifests_arguments(verify, manifests,
manifests_interactive, cwd)
manifest_folder, manifest_interactive, manifest_verify = manifests
profile = profile_from_args(profile_name, settings, options, env,
cwd, self._client_cache)
manager = self._init_manager(recorder)
recorder.add_recipe_being_developed(reference)
create(reference, manager, self._user_io, profile, remote, update,
build_modes, manifest_folder, manifest_verify,
manifest_interactive, keep_build, test_build_folder,
test_folder, conanfile_path)
return recorder.get_info()
except ConanException as exc:
recorder.error = True
exc.info = recorder.get_info()
raise
def _build_package(self, export_folder, src_folder, build_folder,
conan_file, output):
""" builds the package, creating the corresponding build folder if necessary
and copying there the contents from the src folder. The code is duplicated
in every build, as some configure processes actually change the source
code
"""
output.info('Building your package in %s' % build_folder)
config_source(export_folder, src_folder, conan_file, output)
output.info('Copying sources to build folder')
def check_max_path_len(src, files):
if platform.system() != "Windows":
return []
filtered_files = []
for the_file in files:
source_path = os.path.join(src, the_file)
# Without storage path, just relative
rel_path = os.path.relpath(source_path, src_folder)
dest_path = os.path.normpath(
os.path.join(build_folder, rel_path))
# it is NOT that "/" is counted as "\\" so it counts double
# seems a bug in python, overflows paths near the limit of 260,
if len(dest_path) >= 249:
filtered_files.append(the_file)
output.warn("Filename too long, file excluded: %s" %
dest_path)
return filtered_files
shutil.copytree(src_folder,
build_folder,
symlinks=True,
ignore=check_max_path_len)
logger.debug("Copied to %s" % build_folder)
logger.debug("Files copied %s" % os.listdir(build_folder))
os.chdir(build_folder)
conan_file._conanfile_directory = build_folder
# Read generators from conanfile and generate the needed files
logger.debug("Writing generators")
write_generators(conan_file, build_folder, output)
logger.debug("Files copied after generators %s" %
os.listdir(build_folder))
# Build step might need DLLs, binaries as protoc to generate source files
# So execute imports() before build, storing the list of copied_files
from conans.client.importer import FileImporter
local_installer = FileImporter(self._deps_graph, self._client_cache,
build_folder)
conan_file.copy = local_installer
conan_file.imports()
copied_files = local_installer.execute()
import_output = ScopedOutput("%s imports()" % output.scope, output)
report_copied_files(copied_files, import_output)
try:
# This is necessary because it is different for user projects
# than for packages
conan_file._conanfile_directory = build_folder
logger.debug(
"Call conanfile.build() with files in build folder: %s" %
os.listdir(build_folder))
conan_file.build()
self._out.writeln("")
output.success("Package '%s' built" % conan_file.info.package_id())
output.info("Build folder %s" % build_folder)
except Exception as e:
os.chdir(src_folder)
self._out.writeln("")
output.error("Package '%s' build failed" %
conan_file.info.package_id())
output.warn("Build folder %s" % build_folder)
raise ConanException("%s: %s" % (conan_file.name, str(e)))
finally:
conan_file._conanfile_directory = export_folder
# Now remove all files that were imported with imports()
for f in copied_files:
try:
if (f.startswith(build_folder)):
os.remove(f)
except Exception:
self._out.warn(
"Unable to remove imported file from build: %s" % f)
def install(self, reference, current_path, remote=None, options=None, settings=None,
build_mode=False, filename=None, update=False, check_updates=False,
manifest_folder=None, manifest_verify=False, manifest_interactive=False,
scopes=None, generators=None, profile_name=None, package_settings=None,
env=None, package_env=None, no_imports=False):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param current_path: where the output files will be saved
@param remote: install only from that remote
@param options: list of tuples: [(optionname, optionvalue), (optionname, optionvalue)...]
@param settings: list of tuples: [(settingname, settingvalue), (settingname, value)...]
@param package_settings: dict name=> settings: {"zlib": [(settingname, settingvalue), ...]}
@param profile: name of the profile to use
@param env: list of tuples for environment vars: [(var, value), (var2, value2)...]
@param package_env: package dict of list of tuples: {"package_name": [(var, value), (var2, value2)...]}
"""
generators = generators or []
if manifest_folder:
manifest_manager = ManifestManager(manifest_folder, user_io=self._user_io,
client_cache=self._client_cache,
verify=manifest_verify,
interactive=manifest_interactive)
else:
manifest_manager = None
profile = self.read_profile(profile_name, current_path)
# Mix Settings, Env vars and scopes between profile and command line
if profile:
profile.update_settings(settings)
profile.update_package_settings(package_settings)
settings = profile.settings
package_settings = profile.package_settings
profile.update_env(env)
profile.update_packages_env(package_env)
env = profile.env
package_env = profile.package_env
profile.update_scopes(scopes)
scopes = profile.scopes
objects = self._get_graph(reference, current_path, remote, options, settings, filename,
update, check_updates, manifest_manager, scopes, package_settings,
env, package_env)
(_, deps_graph, _, registry, conanfile, remote_proxy, loader) = objects
Printer(self._user_io.out).print_graph(deps_graph, registry)
# Warn if os doesn't match
try:
if detected_os() != loader._settings.os:
message = '''You are building this package with settings.os='%s' on a '%s' system.
If this is your intention, you can ignore this message.
If not:
- Check the passed settings (-s)
- Check your global settings in ~/.conan/conan.conf
- Remove conaninfo.txt to avoid bad cached settings
''' % (loader._settings.os, detected_os())
self._user_io.out.warn(message)
except ConanException: # Setting os doesn't exist
pass
installer = ConanInstaller(self._client_cache, self._user_io, remote_proxy)
installer.install(deps_graph, build_mode)
prefix = "PROJECT" if not isinstance(reference, ConanFileReference) else str(reference)
output = ScopedOutput(prefix, self._user_io.out)
# Write generators
tmp = list(conanfile.generators) # Add the command line specified generators
tmp.extend(generators)
conanfile.generators = tmp
write_generators(conanfile, current_path, output)
if not isinstance(reference, ConanFileReference):
content = normalize(conanfile.info.dumps())
save(os.path.join(current_path, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, current_path, output)
installer.call_system_requirements(conanfile, output)
if manifest_manager:
manifest_manager.print_log()
def _get_recipe(self, reference, check_updates, update, remote_name,
recorder):
output = ScopedOutput(str(reference), self._out)
# check if it is in disk
conanfile_path = self._client_cache.conanfile(reference)
# NOT in disk, must be retrieved from remotes
if not os.path.exists(conanfile_path):
remote, new_ref = self._download_recipe(reference, output,
remote_name, recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
remote = self._registry.get_recipe_remote(reference)
check_updates = check_updates or update
# Recipe exists in disk, but no need to check updates
if not check_updates:
status = RECIPE_INCACHE
log_recipe_got_from_local_cache(reference)
recorder.recipe_fetched_from_cache(reference)
return conanfile_path, status, remote, reference
named_remote = self._registry.remote(
remote_name) if remote_name else None
update_remote = named_remote or remote
if not update_remote:
status = RECIPE_NO_REMOTE
log_recipe_got_from_local_cache(reference)
recorder.recipe_fetched_from_cache(reference)
return conanfile_path, status, None, reference
try: # get_conan_manifest can fail, not in server
upstream_manifest = self._remote_manager.get_conan_manifest(
reference, update_remote)
except NotFoundException:
status = RECIPE_NOT_IN_REMOTE
log_recipe_got_from_local_cache(reference)
recorder.recipe_fetched_from_cache(reference)
return conanfile_path, status, update_remote, reference
export = self._client_cache.export(reference)
read_manifest = FileTreeManifest.load(export)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
if update:
DiskRemover(self._client_cache).remove_recipe(reference)
output.info("Retrieving from remote '%s'..." %
update_remote.name)
reference = self._remote_manager.get_recipe(
reference, update_remote)
self._registry.set_ref(reference, update_remote.name)
status = RECIPE_UPDATED
else:
status = RECIPE_UPDATEABLE
else:
status = RECIPE_NEWER
else:
status = RECIPE_INCACHE
log_recipe_got_from_local_cache(reference)
recorder.recipe_fetched_from_cache(reference)
return conanfile_path, status, update_remote, reference
def complete_test(self):
""" basic installation of a new conans
"""
client = TestClient()
client.init_dynamic_vars()
files = hello_source_files()
conan_ref = ConanFileReference.loads("Hello/1.2.1/frodo/stable")
reg_folder = client.paths.export(conan_ref)
client.save(files, path=reg_folder)
client.save(
{
CONANFILE: myconan1,
"infos/%s" % CONANINFO: "//empty",
"include/no_copy/lib0.h": "NO copy",
"include/math/lib1.h": "copy",
"include/math/lib2.h": "copy",
"include/physics/lib.hpp": "copy",
"my_lib/debug/libd.a": "copy",
"my_data/readme.txt": "copy",
"my_data/readme.md": "NO copy",
"contrib/math/math.h": "copy",
"contrib/physics/gravity.h": "copy",
"contrib/contrib.h": "copy",
"include/opencv/opencv.hpp": "copy",
"include/opencv2/opencv2.hpp": "copy",
"modules/simu/src/simu.cpp": "NO copy",
"modules/simu/include/opencv2/simu/simu.hpp": "copy",
"modules/3D/doc/readme.md": "NO copy",
"modules/3D/include/opencv2/3D/3D.hpp": "copy",
"modules/dev/src/dev.cpp": "NO copy",
"modules/dev/include/opencv2/dev/dev.hpp": "copy",
"modules/opencv_mod.hpp": "copy"
},
path=reg_folder)
conanfile_path = os.path.join(reg_folder, CONANFILE)
package_ref = PackageReference(conan_ref, "myfakeid")
build_folder = client.paths.build(package_ref)
package_folder = client.paths.package(package_ref)
install_folder = os.path.join(build_folder, "infos")
shutil.copytree(reg_folder, build_folder)
output = ScopedOutput("", TestBufferConanOutput())
loader = ConanFileLoader(None, None, ConanPythonRequire(None, None))
conanfile = loader.load_conanfile(conanfile_path, None,
ProcessedProfile())
create_package(conanfile,
None,
build_folder,
build_folder,
package_folder,
install_folder,
output,
client.hook_manager,
conanfile_path,
conan_ref,
copy_info=True)
# test build folder
self.assertTrue(os.path.exists(build_folder))
self.assertTrue(os.path.exists(os.path.join(package_folder,
CONANINFO)))
# test pack folder
self.assertTrue(os.path.exists(package_folder))
def exist(rel_path):
return os.path.exists(os.path.join(package_folder, rel_path))
# Expected files
self.assertTrue(exist("include/lib1.h"))
self.assertTrue(exist("include/lib2.h"))
self.assertTrue(exist("include/physics/lib.hpp"))
self.assertTrue(exist("include/contrib/math/math.h"))
self.assertTrue(exist("include/contrib/physics/gravity.h"))
self.assertTrue(exist("include/contrib/contrib.h"))
self.assertTrue(exist("include/opencv/opencv.hpp"))
self.assertTrue(exist("include/opencv2/opencv2.hpp"))
self.assertTrue(exist("include/opencv2/simu/simu.hpp"))
self.assertTrue(exist("include/opencv2/3D/3D.hpp"))
self.assertTrue(exist("include/opencv2/dev/dev.hpp"))
self.assertTrue(exist("lib/my_lib/libd.a"))
self.assertTrue(exist("res/shares/readme.txt"))
# Not expected files
self.assertFalse(exist("include/opencv2/opencv_mod.hpp"))
self.assertFalse(exist("include/opencv2/simu.hpp"))
self.assertFalse(exist("include/opencv2/3D.hpp"))
self.assertFalse(exist("include/opencv2/dev.hpp"))
self.assertFalse(exist("include/modules/simu/src/simu.cpp"))
self.assertFalse(exist("include/modules/3D/doc/readme.md"))
self.assertFalse(exist("include/modules/dev/src/dev.cpp"))
self.assertFalse(exist("include/opencv2/opencv_mod.hpp"))
self.assertFalse(exist("include/include/no_copy/lib0.h"))
self.assertFalse(exist("res/my_data/readme.md"))
def get_conanfile(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
conan_dir_path = self._paths.export(conan_reference)
rmdir(conan_dir_path)
rmdir(self._paths.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_conanfile(conan_reference, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._paths.conanfile(conan_reference)
if path_exists(conanfile_path, self._paths.store):
# Check manifest integrity
read_manifest, expected_manifest = self._paths.conan_manifests(
conan_reference)
if read_manifest.file_sums != expected_manifest.file_sums:
output.warn(
"Bad conanfile detected! Removing export directory... ")
_refresh()
else: # Check for updates
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it." %
remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(
self._paths.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error(
"Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." %
(remote.name, conan_reference))
else:
self._retrieve_conanfile(conan_reference, output)
return conanfile_path
def _build_node(self, conan_ref, conan_file, build_mode):
# Compute conan_file package from local (already compiled) or from remote
output = ScopedOutput(str(conan_ref), self._out)
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
check_outdated = build_mode == "outdated"
conan_ref = package_reference.conan
package_folder = self._client_cache.package(package_reference,
conan_file.short_paths)
build_folder = self._client_cache.build(package_reference,
conan_file.short_paths)
src_folder = self._client_cache.source(conan_ref,
conan_file.short_paths)
export_folder = self._client_cache.export(conan_ref)
# If already exists do not dirt the output, the common situation
# is that package is already installed and OK. If don't, the proxy
# will print some other message about it
if not os.path.exists(package_folder):
output.info("Installing package %s" % package_id)
self._handle_system_requirements(conan_ref, package_reference,
conan_file, output)
force_build = self._build_forced(conan_ref, build_mode, conan_file)
if self._remote_proxy.get_package(package_reference,
force_build,
short_paths=conan_file.short_paths,
check_outdated=check_outdated):
return
# we need and can build? Only if we are forced or build_mode missing and package not exists
# Option "--build outdated" means: missing or outdated, so don't care if it's really oudated
# just build it.
build = force_build or build_mode is True or check_outdated or conan_file.build_policy_missing
if build:
if not force_build and not build_mode:
output.info(
"Building package from source as defined by build_policy='missing'"
)
try:
rmdir(build_folder)
rmdir(package_folder)
except Exception as e:
raise ConanException(
"%s\n\nCouldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % str(e))
if force_build:
output.warn('Forced build from source')
with environment_append(conan_file.env):
self._build_package(export_folder, src_folder, build_folder,
conan_file, output)
# FIXME: Is weak to assign here the recipe_hash
conan_file.info.recipe_hash = self._client_cache.load_manifest(
conan_ref).summary_hash
# Creating ***info.txt files
save(os.path.join(build_folder, CONANINFO),
conan_file.info.dumps())
output.info("Generated %s" % CONANINFO)
save(os.path.join(build_folder, BUILD_INFO),
TXTGenerator(conan_file).content)
output.info("Generated %s" % BUILD_INFO)
save(os.path.join(build_folder, CONANENV),
ConanEnvGenerator(conan_file).content)
output.info("Generated %s" % CONANENV)
os.chdir(build_folder)
with environment_append(conan_file.env):
create_package(conan_file, build_folder, package_folder,
output)
self._remote_proxy.handle_package_manifest(package_reference,
installed=True)
else:
self._raise_package_not_found_error(conan_ref, conan_file)
def _get_recipe(self, conan_reference, check_updates, update):
output = ScopedOutput(str(conan_reference), self._out)
check_updates = check_updates or update
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
if os.path.exists(conanfile_path):
if check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not update:
if remote != ref_remote: # Forced new remote
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it." %
remote.name)
output.warn("Refused to install!")
else:
DiskRemover(
self._client_cache).remove(conan_reference)
output.info("Retrieving from remote '%s'..." %
remote.name)
self._remote_manager.get_recipe(
conan_reference, remote)
self._registry.set_ref(conan_reference, remote)
output.info("Updated!")
elif ret == -1:
if not update:
output.info(
"Current conanfile is newer than %s's one" %
remote.name)
else:
output.error(
"Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." %
(remote.name, conan_reference))
log_recipe_got_from_local_cache(conan_reference)
self._recorder.recipe_fetched_from_cache(conan_reference)
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
# Just make sure that the recipe sources are there to check
conanfile = load_conanfile_class(conanfile_path)
complete_recipe_sources(self._remote_manager, self._client_cache,
self._registry, conanfile, conan_reference)
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def install(self,
reference,
install_folder,
profile,
remote_name=None,
build_modes=None,
update=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False,
create_reference=None,
keep_build=False):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param install_folder: where the output files will be saved
@param remote: install only from that remote
@param profile: Profile object with both the -s introduced options and profile read values
@param build_modes: List of build_modes specified
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user
for confirmation
@param generators: List of generators from command line. If False, no generator will be
written
@param no_imports: Install specified packages but avoid running imports
@param inject_require: Reference to add as a requirement to the conanfile
"""
if generators is not False:
generators = set(generators) if generators else set()
generators.add("txt") # Add txt generator by default
self._user_io.out.info("Configuration:")
self._user_io.out.writeln(profile.dumps())
result = self._graph_manager.load_graph(reference, create_reference,
profile, build_modes, False,
update, remote_name,
self._recorder, None)
deps_graph, conanfile, cache_settings = result
if not isinstance(reference, ConanFileReference):
output = ScopedOutput(
("%s (test package)" %
str(create_reference)) if create_reference else "PROJECT",
self._user_io.out)
output.highlight("Installing %s" % reference)
else:
output = ScopedOutput(str(reference), self._user_io.out)
output.highlight("Installing package")
print_graph(deps_graph, self._user_io.out)
try:
if cross_building(cache_settings):
b_os, b_arch, h_os, h_arch = get_cross_building_settings(
cache_settings)
message = "Cross-build from '%s:%s' to '%s:%s'" % (
b_os, b_arch, h_os, h_arch)
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
installer = ConanInstaller(self._client_cache,
output,
self._remote_manager,
self._registry,
recorder=self._recorder,
workspace=None,
plugin_manager=self._plugin_manager)
installer.install(deps_graph, keep_build)
if manifest_folder:
manifest_manager = ManifestManager(manifest_folder,
user_io=self._user_io,
client_cache=self._client_cache)
for node in deps_graph.nodes:
if not node.conan_ref:
continue
complete_recipe_sources(self._remote_manager,
self._client_cache, self._registry,
node.conanfile, node.conan_ref)
manifest_manager.check_graph(deps_graph,
verify=manifest_verify,
interactive=manifest_interactive)
manifest_manager.print_log()
if install_folder:
# Write generators
if generators is not False:
tmp = list(conanfile.generators
) # Add the command line specified generators
tmp.extend([g for g in generators if g not in tmp])
conanfile.generators = tmp
write_generators(conanfile, install_folder, output)
if not isinstance(reference, ConanFileReference):
# Write conaninfo
content = normalize(conanfile.info.dumps())
save(os.path.join(install_folder, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, install_folder, output)
call_system_requirements(conanfile, output)
if not create_reference and isinstance(reference,
ConanFileReference):
# The conanfile loaded is really a virtual one. The one with the deploy is the first level one
neighbours = deps_graph.root.neighbors()
deploy_conanfile = neighbours[0].conanfile
if hasattr(deploy_conanfile, "deploy") and callable(
deploy_conanfile.deploy):
run_deploy(deploy_conanfile, install_folder, output)
def create_package(conanfile,
source_folder,
build_folder,
package_folder,
install_folder,
output,
local=False,
copy_info=False):
""" copies built artifacts, libs, headers, data, etc from build_folder to
package folder
"""
mkdir(package_folder)
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % (package_folder))
def wrap(dst_folder):
def new_method(pattern, src=""):
conanfile.copy(pattern, dst_folder, src)
return new_method
# FIXME: Deprecate these methods. Not documented. Confusing. Rely on LINTER
conanfile.copy_headers = wrap(DEFAULT_INCLUDE)
conanfile.copy_libs = wrap(DEFAULT_LIB)
conanfile.copy_bins = wrap(DEFAULT_BIN)
conanfile.copy_res = wrap(DEFAULT_RES)
try:
package_output = ScopedOutput("%s package()" % output.scope, output)
output.highlight("Calling package()")
conanfile.package_folder = package_folder
conanfile.source_folder = source_folder
conanfile.build_folder = build_folder
if source_folder != build_folder:
conanfile.copy = FileCopier(source_folder, package_folder,
build_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
with tools.chdir(build_folder):
conanfile.package()
conanfile.copy.report(package_output, warn=True)
conanfile.copy = FileCopier(build_folder, package_folder)
with tools.chdir(build_folder):
with conanfile_exception_formatter(str(conanfile), "package"):
conanfile.package()
conanfile.copy.report(package_output, warn=True)
except Exception as e:
if not local:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s" %
(package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise
raise ConanException(e)
_create_aux_files(install_folder, package_folder, conanfile, copy_info)
output.success("Package '%s' created" % os.path.basename(package_folder))
def _config_node(self, node, down_reqs, down_ref, down_options, aliased):
""" update settings and option in the current ConanFile, computing actual
requirement values, cause they can be overridden by downstream requires
param settings: dict of settings values => {"os": "windows"}
"""
try:
conanfile, conanref = node.conanfile, node.conan_ref
# Avoid extra time manipulating the sys.path for python
with get_env_context_manager(conanfile, without_python=True):
if hasattr(conanfile, "config"):
if not conanref:
output = ScopedOutput(str("PROJECT"), self._output)
output.warn("config() has been deprecated."
" Use config_options and configure")
with conanfile_exception_formatter(str(conanfile), "config"):
conanfile.config()
with conanfile_exception_formatter(str(conanfile), "config_options"):
conanfile.config_options()
conanfile.options.propagate_upstream(down_options, down_ref, conanref)
if hasattr(conanfile, "config"):
with conanfile_exception_formatter(str(conanfile), "config"):
conanfile.config()
with conanfile_exception_formatter(str(conanfile), "configure"):
conanfile.configure()
conanfile.settings.validate() # All has to be ok!
conanfile.options.validate()
# Update requirements (overwrites), computing new upstream
if hasattr(conanfile, "requirements"):
# If re-evaluating the recipe, in a diamond graph, with different options,
# it could happen that one execution path of requirements() defines a package
# and another one a different package raising Duplicate dependency error
# Or the two consecutive calls, adding 2 different dependencies for the two paths
# So it is necessary to save the "requires" state and restore it before a second
# execution of requirements(). It is a shallow copy, if first iteration is
# RequireResolve'd or overridden, the inner requirements are modified
if not hasattr(conanfile, "_conan_original_requires"):
conanfile._conan_original_requires = conanfile.requires.copy()
else:
conanfile.requires = conanfile._conan_original_requires.copy()
with conanfile_exception_formatter(str(conanfile), "requirements"):
conanfile.requirements()
new_options = conanfile.options.deps_package_values
if aliased:
for req in conanfile.requires.values():
req.conan_reference = aliased.get(req.conan_reference,
req.conan_reference)
new_down_reqs = conanfile.requires.update(down_reqs, self._output,
conanref, down_ref)
except ConanExceptionInUserConanfileMethod:
raise
except ConanException as e:
raise ConanException("%s: %s" % (conanref or "Conanfile", str(e)))
except Exception as e:
raise ConanException(e)
return new_down_reqs, new_options
def create(self,
profile_name=None,
settings=None,
options=None,
env=None,
scope=None,
test_folder=None,
not_export=False,
build=None,
keep_source=False,
verify=default_manifest_folder,
manifests=default_manifest_folder,
manifests_interactive=default_manifest_folder,
remote=None,
update=False,
cwd=None,
user=None,
channel=None,
name=None,
version=None):
settings = settings or []
options = options or []
env = env or []
cwd = prepare_cwd(cwd)
if not name or not version:
conanfile_path = os.path.join(cwd, "conanfile.py")
conanfile = load_conanfile_class(conanfile_path)
try:
name, version = conanfile.name, conanfile.version
except:
raise ConanException(
"conanfile.py doesn't declare package name or version")
reference = ConanFileReference(name, version, user, channel)
scoped_output = ScopedOutput(str(reference), self._user_io.out)
# Forcing an export!
if not not_export:
scoped_output.highlight("Exporting package recipe")
self._manager.export(user,
channel,
cwd,
keep_source=keep_source,
name=name,
version=version)
if build is None: # Not specified, force build the tested library
build = [name]
manifests = _parse_manifests_arguments(verify, manifests,
manifests_interactive, cwd)
manifest_folder, manifest_interactive, manifest_verify = manifests
profile = profile_from_args(profile_name, settings, options, env,
scope, cwd,
self._client_cache.profiles_path)
self._manager.install(reference=reference,
current_path=cwd,
manifest_folder=manifest_folder,
manifest_verify=manifest_verify,
manifest_interactive=manifest_interactive,
remote=remote,
profile=profile,
build_modes=build,
update=update)
test_folders = [test_folder
] if test_folder else ["test_package", "test"]
for test_folder_name in test_folders:
test_folder = os.path.join(cwd, test_folder_name)
test_conanfile_path = os.path.join(test_folder, "conanfile.py")
if os.path.exists(test_conanfile_path):
break
else:
self._user_io.out.warn(
"test package folder not available, or it doesn't have "
"a conanfile.py\nIt is recommended to set a 'test_package' "
"while creating packages")
return
scoped_output.highlight("Testing with 'test_package'")
sha = hashlib.sha1("".join(options + settings).encode()).hexdigest()
build_folder = os.path.join(test_folder, "build", sha)
rmdir(build_folder)
test_conanfile = os.path.join(test_folder, CONANFILE)
self._manager.install(inject_require=reference,
reference=test_folder,
current_path=build_folder,
manifest_folder=manifest_folder,
manifest_verify=manifest_verify,
manifest_interactive=manifest_interactive,
remote=remote,
profile=profile,
update=update,
generators=["txt"])
self._manager.build(test_conanfile,
test_folder,
build_folder,
package_folder=None,
test=str(reference))
def create_package(conanfile,
pkg_id,
source_folder,
build_folder,
package_folder,
install_folder,
output,
hook_manager,
conanfile_path,
reference,
local=False,
copy_info=False):
""" copies built artifacts, libs, headers, data, etc. from build_folder to
package folder
"""
mkdir(package_folder)
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % package_folder)
try:
conanfile.package_folder = package_folder
conanfile.source_folder = source_folder
conanfile.install_folder = install_folder
conanfile.build_folder = build_folder
hook_manager.execute("pre_package",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=reference,
package_id=pkg_id)
package_output = ScopedOutput("%s package()" % output.scope, output)
output.highlight("Calling package()")
def recipe_has(attribute):
return attribute in conanfile.__class__.__dict__
if source_folder != build_folder:
conanfile.copy = FileCopier(source_folder, package_folder,
build_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
with tools.chdir(source_folder):
conanfile.package()
copy_done = conanfile.copy.report(package_output)
if not copy_done and recipe_has("package"):
output.warn("No files copied from source folder!")
conanfile.copy = FileCopier(build_folder, package_folder)
with tools.chdir(build_folder):
with conanfile_exception_formatter(str(conanfile), "package"):
conanfile.package()
copy_done = conanfile.copy.report(package_output)
if not copy_done and recipe_has("build") and recipe_has("package"):
output.warn("No files copied from build folder!")
except Exception as e:
if not local:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s" %
(package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise
raise ConanException(e)
_create_aux_files(install_folder, package_folder, conanfile, copy_info)
pkg_id = pkg_id or os.path.basename(package_folder)
output.success("Package '%s' created" % pkg_id)
hook_manager.execute("post_package",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=reference,
package_id=pkg_id)
def build(self,
conanfile_path,
current_path,
test=False,
filename=None,
profile_name=None,
env=None,
package_env=None):
""" Call to build() method saved on the conanfile.py
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
logger.debug("Building in %s" % current_path)
logger.debug("Conanfile in %s" % conanfile_path)
if filename and filename.endswith(".txt"):
raise ConanException(
"A conanfile.py is needed to call 'conan build'")
conanfile_file = os.path.join(conanfile_path, filename or CONANFILE)
try:
# Append env_vars to execution environment and clear when block code ends
profile = self.read_profile(profile_name, current_path)
output = ScopedOutput("Project", self._user_io.out)
if profile:
profile.update_env(env)
profile.update_packages_env(package_env)
env = profile.env
package_env = profile.package_env
package_env = profile.package_env if profile else None
loader = self._loader(current_path,
env=env,
package_env=package_env)
conan_file = loader.load_conan(conanfile_file,
output,
consumer=True)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file" %
(CONANFILE, CONANFILE, CONANFILE_TXT))
try:
self._load_deps_info(current_path, conan_file, output)
os.chdir(current_path)
conan_file._conanfile_directory = conanfile_path
with environment_append(conan_file.env):
conan_file.build()
if test:
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" %
'\n'.join(trace[3:]))
def _evaluate_node(self, node, build_mode, update, evaluated_references,
remote_name):
assert node.binary is None
conan_ref, conanfile = node.conan_ref, node.conanfile
revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False)
package_id = conanfile.info.package_id()
package_ref = PackageReference(conan_ref, package_id)
# Check that this same reference hasn't already been checked
previous_node = evaluated_references.get(package_ref)
if previous_node:
node.binary = previous_node.binary
node.binary_remote = previous_node.binary_remote
return
evaluated_references[package_ref] = node
output = ScopedOutput(str(conan_ref), self._out)
if build_mode.forced(conanfile, conan_ref):
output.warn('Forced build from source')
node.binary = BINARY_BUILD
return
package_folder = self._client_cache.package(
package_ref, short_paths=conanfile.short_paths)
# Check if dirty, to remove it
local_project = self._workspace[conan_ref] if self._workspace else None
if local_project:
node.binary = BINARY_WORKSPACE
return
with self._client_cache.package_lock(package_ref):
if is_dirty(package_folder):
output.warn("Package is corrupted, removing folder: %s" %
package_folder)
rmdir(package_folder)
if remote_name:
remote = self._registry.remotes.get(remote_name)
else:
# If the remote_name is not given, follow the binary remote, or
# the recipe remote
# If it is defined it won't iterate (might change in conan2.0)
remote = self._registry.prefs.get(
package_ref) or self._registry.refs.get(conan_ref)
remotes = self._registry.remotes.list
if os.path.exists(package_folder):
if update:
if remote:
if self._check_update(package_folder, package_ref, remote,
output, node):
node.binary = BINARY_UPDATE
if build_mode.outdated:
package_hash = self._get_package_info(
package_ref, remote).recipe_hash
elif remotes:
pass
else:
output.warn("Can't update, no remote defined")
if not node.binary:
node.binary = BINARY_CACHE
package_hash = ConanInfo.load_from_package(
package_folder).recipe_hash
else: # Binary does NOT exist locally
if not revisions_enabled and not node.revision_pinned:
# Do not search for packages for the specific resolved recipe revision but all
package_ref = package_ref.copy_clear_rev()
remote_info = None
if remote:
remote_info = self._get_package_info(package_ref, remote)
# If the "remote" came from the registry but the user didn't specified the -r, with
# revisions iterate all remotes
if not remote or (not remote_info and revisions_enabled
and not remote_name):
for r in remotes:
remote_info = self._get_package_info(package_ref, r)
if remote_info:
remote = r
break
if remote_info:
node.binary = BINARY_DOWNLOAD
package_hash = remote_info.recipe_hash
else:
if build_mode.allowed(conanfile, conan_ref):
node.binary = BINARY_BUILD
else:
node.binary = BINARY_MISSING
if build_mode.outdated:
if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE):
local_recipe_hash = self._client_cache.load_manifest(
package_ref.conan).summary_hash
if local_recipe_hash != package_hash:
output.info("Outdated package!")
node.binary = BINARY_BUILD
else:
output.info("Package is up to date")
node.binary_remote = remote
def install(self, reference, current_path, profile, remote=None,
build_modes=None, filename=None, update=False,
manifest_folder=None, manifest_verify=False, manifest_interactive=False,
generators=None, no_imports=False):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param current_path: where the output files will be saved
@param remote: install only from that remote
@param profile: Profile object with both the -s introduced options and profile readed values
@param build_modes: List of build_modes specified
@param filename: Optional filename of the conanfile
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user for confirmation
@param generators: List of generators from command line
@param no_imports: Install specified packages but avoid running imports
"""
generators = generators or []
manifest_manager = ManifestManager(manifest_folder, user_io=self._user_io,
client_cache=self._client_cache,
verify=manifest_verify,
interactive=manifest_interactive) if manifest_folder else None
remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote,
update=update, check_updates=False, manifest_manager=manifest_manager)
loader = ConanFileLoader(self._runner, self._client_cache.settings, profile)
conanfile = self._get_conanfile_object(loader, reference, filename, current_path)
graph_builder = self._get_graph_builder(loader, update, remote_proxy)
deps_graph = graph_builder.load(conanfile)
# This line is so the conaninfo stores the correct complete info
conanfile.info.scope = profile.scopes
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
Printer(self._user_io.out).print_graph(deps_graph, registry)
try:
if detected_os() != loader._settings.os:
message = "Cross-platform from '%s' to '%s'" % (detected_os(), loader._settings.os)
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
build_mode = BuildMode(build_modes, self._user_io.out)
build_requires = BuildRequires(loader, remote_proxy, self._user_io.out, self._client_cache,
self._search_manager, profile.build_requires, current_path,
build_mode)
# Apply build_requires to consumer conanfile
build_requires.install("", conanfile)
installer = ConanInstaller(self._client_cache, self._user_io.out, remote_proxy,
build_requires)
installer.install(deps_graph, build_mode, current_path)
build_mode.report_matches()
prefix = "PROJECT" if not isinstance(reference, ConanFileReference) else str(reference)
output = ScopedOutput(prefix, self._user_io.out)
# Write generators
tmp = list(conanfile.generators) # Add the command line specified generators
tmp.extend(generators)
conanfile.generators = tmp
write_generators(conanfile, current_path, output)
if not isinstance(reference, ConanFileReference):
content = normalize(conanfile.info.dumps())
save(os.path.join(current_path, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, current_path, output)
installer.call_system_requirements(conanfile, output)
if manifest_manager:
manifest_manager.print_log()
