def download(app, ref, package_ids, remote, recipe, recorder, remotes): out, remote_manager, cache, loader = app.out, app.remote_manager, app.cache, app.loader hook_manager = app.hook_manager assert (isinstance(ref, ConanFileReference)) output = ScopedOutput(str(ref), out) hook_manager.execute("pre_download", reference=ref, remote=remote) try: ref = remote_manager.get_recipe(ref, remote) except NotFoundException: raise RecipeNotFoundException(ref) conan_file_path = cache.package_layout(ref).conanfile() conanfile = loader.load_basic(conan_file_path) # Download the sources too, don't be lazy retrieve_exports_sources(remote_manager, cache, conanfile, ref, remotes) if not recipe: # Not only the recipe if not package_ids: # User didn't specify a specific package binary output.info("Getting the complete package list from '%s'..." % ref.full_str()) packages_props = remote_manager.search_packages(remote, ref, None) package_ids = list(packages_props.keys()) if not package_ids: output.warn("No remote binary packages found in remote") parallel = cache.config.parallel_download _download_binaries(conanfile, ref, package_ids, cache, remote_manager, remote, output, recorder, parallel) hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=ref, remote=remote)
def download(self, reference, package_ids, remote=None): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote """ assert (isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) package = remote_proxy.search(reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) if package_ids: remote_proxy.download_packages(reference, package_ids) else: self._user_io.out.info("Getting the complete package list " "from '%s'..." % str(reference)) packages_props = remote_proxy.search_packages(reference, None) if not packages_props: output = ScopedOutput(str(reference), self._user_io.out) output.warn("No remote binary packages found in remote") else: remote_proxy.download_packages(reference, list(packages_props.keys()))
def download(reference, package_ids, remote_name, recipe, remote_manager, client_cache, out, recorder, loader, hook_manager): assert(isinstance(reference, ConanFileReference)) output = ScopedOutput(str(reference), out) registry = client_cache.registry remote = registry.remotes.get(remote_name) if remote_name else registry.remotes.default hook_manager.execute("pre_download", reference=reference, remote=remote) # First of all download package recipe try: remote_manager.get_recipe(reference, remote) except NotFoundException: raise NotFoundException("'%s' not found in remote '%s'" % (str(reference), remote.name)) registry.refs.set(reference, remote.name) conan_file_path = client_cache.conanfile(reference) conanfile = loader.load_class(conan_file_path) if not recipe: # Not only the recipe # Download the sources too, don't be lazy complete_recipe_sources(remote_manager, client_cache, conanfile, reference) if not package_ids: # User didnt specify a specific package binary output.info("Getting the complete package list from '%s'..." % str(reference)) packages_props = remote_manager.search_packages(remote, reference, None) package_ids = list(packages_props.keys()) if not package_ids: output.warn("No remote binary packages found in remote") _download_binaries(conanfile, reference, package_ids, client_cache, remote_manager, remote, output, recorder) hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=reference, remote=remote)
def download(ref, package_ids, remote, recipe, remote_manager, cache, out, recorder, loader, hook_manager, remotes): assert (isinstance(ref, ConanFileReference)) output = ScopedOutput(str(ref), out) hook_manager.execute("pre_download", reference=ref, remote=remote) ref = remote_manager.get_recipe(ref, remote) with cache.package_layout(ref).update_metadata() as metadata: metadata.recipe.remote = remote.name conan_file_path = cache.package_layout(ref).conanfile() conanfile = loader.load_class(conan_file_path) # Download the sources too, don't be lazy complete_recipe_sources(remote_manager, cache, conanfile, ref, remotes) if not recipe: # Not only the recipe if not package_ids: # User didn't specify a specific package binary output.info("Getting the complete package list from '%s'..." % ref.full_repr()) packages_props = remote_manager.search_packages(remote, ref, None) package_ids = list(packages_props.keys()) if not package_ids: output.warn("No remote binary packages found in remote") _download_binaries(conanfile, ref, package_ids, cache, remote_manager, remote, output, recorder) hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=ref, remote=remote)
def _build_node(self, conan_ref, conan_file, build_mode): # Compute conan_file package from local (already compiled) or from remote output = ScopedOutput(str(conan_ref), self._out) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) conan_ref = package_reference.conan package_folder = self._paths.package(package_reference) build_folder = self._paths.build(package_reference, conan_file.short_paths) src_folder = self._paths.source(conan_ref, conan_file.short_paths) export_folder = self._paths.export(conan_ref) # If already exists do not dirt the output, the common situation # is that package is already installed and OK. If don't, the proxy # will print some other message about it if not os.path.exists(package_folder): output.info("Installing package %s" % package_id) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) force_build = self._build_forced(conan_ref, build_mode, conan_file) if self._remote_proxy.get_package(package_reference, force_build): return # we need and can build? Only if we are forced or build_mode missing and package not exists build = force_build or build_mode is True or conan_file.build_policy_missing if build: if not force_build and not build_mode: output.info( "Building package from source as defined by build_policy='missing'" ) try: rmdir(build_folder, conan_file.short_paths) rmdir(package_folder) except Exception as e: raise ConanException( "%s\n\nCouldn't remove folder, might be busy or open\n" "Close any app using it, and retry" % str(e)) if force_build: output.warn('Forced build from source') self._build_package(export_folder, src_folder, build_folder, package_folder, conan_file, output) # Creating ***info.txt files save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) output.info("Generated %s" % CONANINFO) save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content) output.info("Generated %s" % BUILD_INFO) os.chdir(build_folder) create_package(conan_file, build_folder, package_folder, output) else: self._raise_package_not_found_error(conan_ref, conan_file)
def _config_node(self, node, down_reqs, down_ref, down_options): """ update settings and option in the current ConanFile, computing actual requirement values, cause they can be overriden by downstream requires param settings: dict of settings values => {"os": "windows"} """ try: conanfile, conanref = node.conanfile, node.conan_ref # Avoid extra time manipulating the sys.path for python with get_env_context_manager(conanfile, without_python=True): if hasattr(conanfile, "config"): if not conanref: output = ScopedOutput(str("PROJECT"), self._output) output.warn("config() has been deprecated." " Use config_options and configure") with conanfile_exception_formatter(str(conanfile), "config"): conanfile.config() with conanfile_exception_formatter(str(conanfile), "config_options"): conanfile.config_options() conanfile.options.propagate_upstream(down_options, down_ref, conanref) if hasattr(conanfile, "config"): with conanfile_exception_formatter(str(conanfile), "config"): conanfile.config() with conanfile_exception_formatter(str(conanfile), "configure"): conanfile.configure() conanfile.settings.validate() # All has to be ok! conanfile.options.validate() # Update requirements (overwrites), computing new upstream if hasattr(conanfile, "requirements"): # If re-evaluating the recipe, in a diamond graph, with different options, # it could happen that one execution path of requirements() defines a package # and another one a different package raising Duplicate dependency error # Or the two consecutive calls, adding 2 different dependencies for the two paths # So it is necessary to save the "requires" state and restore it before a second # execution of requirements(). It is a shallow copy, if first iteration is # RequireResolve'd or overridden, the inner requirements are modified if not hasattr(conanfile, "_original_requires"): conanfile._original_requires = conanfile.requires.copy() else: conanfile.requires = conanfile._original_requires.copy() with conanfile_exception_formatter(str(conanfile), "requirements"): conanfile.requirements() new_options = conanfile.options.deps_package_values new_down_reqs = conanfile.requires.update(down_reqs, self._output, conanref, down_ref) except ConanExceptionInUserConanfileMethod: raise except ConanException as e: raise ConanException("%s: %s" % (conanref or "Conanfile", str(e))) except Exception as e: raise ConanException(e) return new_down_reqs, new_options
def _build_node(self, conan_ref, conan_file, build_mode): # Compute conan_file package from local (already compiled) or from remote output = ScopedOutput(str(conan_ref), self._out) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) conan_ref = package_reference.conan package_folder = self._paths.package(package_reference, conan_file.short_paths) build_folder = self._paths.build(package_reference, conan_file.short_paths) src_folder = self._paths.source(conan_ref, conan_file.short_paths) export_folder = self._paths.export(conan_ref) # If already exists do not dirt the output, the common situation # is that package is already installed and OK. If don't, the proxy # will print some other message about it if not package_exists(package_folder): output.info("Installing package %s" % package_id) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) force_build = self._build_forced(conan_ref, build_mode, conan_file) if self._remote_proxy.get_package(package_reference, force_build, short_paths=conan_file.short_paths): return # we need and can build? Only if we are forced or build_mode missing and package not exists build = force_build or build_mode is True or conan_file.build_policy_missing if build: if not force_build and not build_mode: output.info("Building package from source as defined by build_policy='missing'") try: rmdir(build_folder, conan_file.short_paths) rmdir(package_folder) except Exception as e: raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n" "Close any app using it, and retry" % str(e)) if force_build: output.warn('Forced build from source') self._build_package(export_folder, src_folder, build_folder, package_folder, conan_file, output) # Creating ***info.txt files save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) output.info("Generated %s" % CONANINFO) save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content) output.info("Generated %s" % BUILD_INFO) os.chdir(build_folder) create_package(conan_file, build_folder, package_folder, output) self._remote_proxy.handle_package_manifest(package_reference, installed=True) else: self._raise_package_not_found_error(conan_ref, conan_file)
def _get_nodes(self, nodes_by_level, skip_nodes): """Compute a list of (conan_ref, package_id, conan_file, build_node) defining what to do with each node """ nodes_to_build = [] # Now build each level, starting from the most independent one package_references = set() for level in nodes_by_level: for node in level: if node in skip_nodes: continue conan_ref, conan_file = node.conan_ref, node.conanfile build_node = False logger.debug("Processing node %s", repr(conan_ref)) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) # Avoid processing twice the same package reference if package_reference not in package_references: package_references.add(package_reference) package_folder = self._client_cache.package( package_reference, short_paths=conan_file.short_paths) local_project = self._workspace[ conan_ref] if self._workspace else None if not local_project: with self._client_cache.package_lock( package_reference): if is_dirty(package_folder): output = ScopedOutput(str(conan_ref), self._out) output.warn( "Package is corrupted, removing folder: %s" % package_folder) rmdir(package_folder) check_outdated = self._build_mode.outdated if self._build_mode.forced(conan_file, conan_ref): build_node = True else: if local_project: # Maybe the folder is not there, check it! build_node = False else: available = self._remote_proxy.package_available( package_reference, package_folder, check_outdated) build_node = not available nodes_to_build.append((node, package_id, build_node)) return nodes_to_build
def _build_node(self, conan_ref, conan_file, build_mode): # Compute conan_file package from local (already compiled) or from remote output = ScopedOutput(str(conan_ref), self._out) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) conan_ref = package_reference.conan package_folder = self._paths.package(package_reference) build_folder = self._paths.build(package_reference) src_folder = self._paths.source(conan_ref) export_folder = self._paths.export(conan_ref) # If already exists do not dirt the output, the common situation # is that package is already installed and OK. If don't, the proxy # will print some other message about it if not os.path.exists(package_folder): output.info("Installing package %s" % package_id) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) force_build = self._force_build(conan_ref, build_mode) if self._remote_proxy.get_package(package_reference, force_build): return # Can we build? Only if we are forced or build_mode missing and package not exists build_allowed = force_build or build_mode is True if build_allowed: rmdir(build_folder) rmdir(package_folder) if force_build: output.warn('Forced build from source') self._build_package(export_folder, src_folder, build_folder, conan_file, output) # Creating ***info.txt files save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) output.info("Generated %s" % CONANINFO) save( os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file.deps_cpp_info, conan_file.cpp_info).content) output.info("Generated %s" % BUILD_INFO) os.chdir(build_folder) create_package(conan_file, build_folder, package_folder, output) else: self._raise_package_not_found_error(conan_ref, conan_file)
def get_conanfile(self, conan_reference): output = ScopedOutput(str(conan_reference), self._out) def _refresh(): conan_dir_path = self._paths.export(conan_reference) rmdir(conan_dir_path) rmdir(self._paths.source(conan_reference), True) # It might need to remove shortpath current_remote, _ = self._get_remote(conan_reference) output.info("Retrieving from remote '%s'..." % current_remote.name) self._remote_manager.get_conanfile(conan_reference, current_remote) if self._update: output.info("Updated!") else: output.info("Installed!") # check if it is in disk conanfile_path = self._paths.conanfile(conan_reference) path_exist = path_exists(conanfile_path, self._paths.store) if path_exist: if self._check_integrity: # Check if package is corrupted read_manifest, expected_manifest = self._paths.conan_manifests(conan_reference) if read_manifest.file_sums != expected_manifest.file_sums: output.warn("Bad conanfile detected! Removing export directory... ") _refresh() else: # Check for updates if self._check_updates: ret = self.update_available(conan_reference) if ret != 0: # Found and not equal remote, ref_remote = self._get_remote(conan_reference) if ret == 1: if not self._update: if remote != ref_remote: # Forced new remote output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u -r %s' to update it." % (remote.name, remote.name)) else: output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u' to update it." % remote.name) output.warn("Refused to install!") else: if remote != ref_remote: # Delete packages, could be non coherent with new remote rmdir(self._paths.packages(conan_reference)) _refresh() elif ret == -1: if not self._update: output.info("Current conanfile is newer " "than %s's one" % remote.name) else: output.error("Current conanfile is newer than %s's one. " "Run 'conan remove %s' and run install again " "to replace it." % (remote.name, conan_reference)) else: self._retrieve_conanfile(conan_reference, output) return conanfile_path
def _build_node(self, conan_ref, conan_file, build_mode): # Compute conan_file package from local (already compiled) or from remote output = ScopedOutput(str(conan_ref), self._out) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) conan_ref = package_reference.conan package_folder = self._paths.package(package_reference) build_folder = self._paths.build(package_reference) src_folder = self._paths.source(conan_ref) export_folder = self._paths.export(conan_ref) # If already exists do not dirt the output, the common situation # is that package is already installed and OK. If don't, the proxy # will print some other message about it if not os.path.exists(package_folder): output.info("Installing package %s" % package_id) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) force_build = self._force_build(conan_ref, build_mode) if self._remote_proxy.get_package(package_reference, force_build): return # Can we build? Only if we are forced or build_mode missing and package not exists build_allowed = force_build or build_mode is True if build_allowed: rmdir(build_folder) rmdir(package_folder) if force_build: output.warn('Forced build from source') self._build_package(export_folder, src_folder, build_folder, conan_file, output) # Creating ***info.txt files save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) output.info("Generated %s" % CONANINFO) save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file.deps_cpp_info, conan_file.cpp_info).content) output.info("Generated %s" % BUILD_INFO) os.chdir(build_folder) create_package(conan_file, build_folder, package_folder, output) else: self._raise_package_not_found_error(conan_ref, conan_file)
def retrieve_conanfile(self, conan_reference, consumer=False): """ returns the requested conanfile object, retrieving it from remotes if necessary. Can raise NotFoundException """ output = ScopedOutput(str(conan_reference), self._out) conanfile_path = self._paths.conanfile(conan_reference) if not self._paths.valid_conan_digest(conan_reference): conan_dir_path = self._paths.export(conan_reference) if path_exists(conan_dir_path, self._paths.store): # If not valid conanfile, ensure empty folder output.warn("Bad conanfile detected! Removing export directory... ") rmdir(conan_dir_path) output.info("Conanfile not found, retrieving from server") # If not in localhost, download it. Will raise if not found self._remote_manager.get_conanfile(conan_reference, self._remote) conanfile = self._loader.load_conan(conanfile_path, output, consumer) return conanfile
def _get_nodes(self, nodes_by_level, skip_nodes): """Compute a list of (conan_ref, package_id, conan_file, build_node) defining what to do with each node """ nodes_to_build = [] # Now build each level, starting from the most independent one package_references = set() for level in nodes_by_level: for node in level: if node in skip_nodes: continue conan_ref, conan_file = node.conan_ref, node.conanfile build_node = False logger.debug("Processing node %s", repr(conan_ref)) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) # Avoid processing twice the same package reference if package_reference not in package_references: package_references.add(package_reference) package_folder = self._client_cache.package(package_reference, short_paths=conan_file.short_paths) with self._client_cache.package_lock(package_reference): if is_dirty(package_folder): output = ScopedOutput(str(conan_ref), self._out) output.warn("Package is corrupted, removing folder: %s" % package_folder) rmdir(package_folder) check_outdated = self._build_mode.outdated if self._build_mode.forced(conan_file, conan_ref): build_node = True else: available = self._remote_proxy.package_available(package_reference, package_folder, check_outdated) build_node = not available nodes_to_build.append((node, package_id, build_node)) # A check to be sure that if introduced a pattern, something is going to be built if self._build_mode.patterns: to_build = [str(n[0].conan_ref.name) for n in nodes_to_build if n[2]] self._build_mode.check_matches(to_build) return nodes_to_build
def download(self, reference, package_ids, remote, recipe): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote @param only_recipe: download only the recipe """ assert (isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, recorder=self._recorder) remote, _ = remote_proxy._get_remote() package = self._remote_manager.search_recipes(remote, reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) # First of all download package recipe remote_proxy.get_recipe(reference) if recipe: return # Download the sources too, don't be lazy conan_file_path = self._client_cache.conanfile(reference) conanfile = load_conanfile_class(conan_file_path) remote_proxy.complete_recipe_sources(conanfile, reference, short_paths=conanfile.short_paths) if package_ids: remote_proxy.download_packages(reference, package_ids) else: self._user_io.out.info("Getting the complete package list " "from '%s'..." % str(reference)) packages_props = self._remote_manager.search_packages( remote, reference, None) if not packages_props: output = ScopedOutput(str(reference), self._user_io.out) output.warn("No remote binary packages found in remote") else: remote_proxy.download_packages(reference, list(packages_props.keys()))
def download(reference, package_ids, remote_name, recipe, registry, remote_manager, client_cache, out, recorder, loader, hook_manager): assert (isinstance(reference, ConanFileReference)) output = ScopedOutput(str(reference), out) remote = registry.remotes.get( remote_name) if remote_name else registry.remotes.default package = remote_manager.search_recipes(remote, reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) hook_manager.execute("pre_download", reference=reference, remote=remote) # First of all download package recipe remote_manager.get_recipe(reference, remote) registry.refs.set(reference, remote.name) conan_file_path = client_cache.conanfile(reference) conanfile = loader.load_class(conan_file_path) if not recipe: # Download the sources too, don't be lazy complete_recipe_sources(remote_manager, client_cache, registry, conanfile, reference) if package_ids: _download_binaries(reference, package_ids, client_cache, remote_manager, remote, output, recorder, loader) else: output.info("Getting the complete package list " "from '%s'..." % str(reference)) packages_props = remote_manager.search_packages( remote, reference, None) if not packages_props: output = ScopedOutput(str(reference), out) output.warn("No remote binary packages found in remote") else: _download_binaries(reference, list(packages_props.keys()), client_cache, remote_manager, remote, output, recorder, loader) hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=reference, remote=remote)
def get_package(self, package_reference, force_build): """ obtain a package, either from disk or retrieve from remotes if necessary and not necessary to build """ output = ScopedOutput(str(package_reference.conan), self._out) package_folder = self._paths.package(package_reference) # Check current package status if path_exists(package_folder, self._paths.store): if self._check_integrity or self._check_updates: read_manifest, expected_manifest = self._paths.package_manifests( package_reference) if self._check_integrity: # Check if package is corrupted if read_manifest.file_sums != expected_manifest.file_sums: # If not valid package, ensure empty folder output.warn("Bad package '%s' detected! Removing " "package directory... " % str(package_reference.package_id)) rmdir(package_folder) if self._check_updates: try: # get_conan_digest can fail, not in server upstream_manifest = self.get_package_digest( package_reference) if upstream_manifest.file_sums != read_manifest.file_sums: if upstream_manifest.time > read_manifest.time: output.warn( "Current package is older than remote upstream one" ) if self._update: output.warn( "Removing it to retrieve or build an updated one" ) rmdir(package_folder) else: output.warn( "Current package is newer than remote upstream one" ) except ConanException: pass if not force_build: local_package = os.path.exists(package_folder) if local_package: output = ScopedOutput(str(package_reference.conan), self._out) output.info('Already installed!') return True return self._retrieve_remote_package(package_reference, output) return False
def get_package(self, package_ref, short_paths): """ obtain a package, either from disk or retrieve from remotes if necessary and not necessary to build """ output = ScopedOutput(str(package_ref.conan), self._out) package_folder = self._client_cache.package(package_ref, short_paths=short_paths) # Check current package status if os.path.exists(package_folder): if self._check_updates: read_manifest = self._client_cache.load_package_manifest(package_ref) try: # get_conan_digest can fail, not in server upstream_manifest = self.get_package_digest(package_ref) if upstream_manifest.file_sums != read_manifest.file_sums: if upstream_manifest.time > read_manifest.time: output.warn("Current package is older than remote upstream one") if self._update: output.warn("Removing it to retrieve or build an updated one") rmdir(package_folder) else: output.warn("Current package is newer than remote upstream one") except ConanException: pass installed = False local_package = os.path.exists(package_folder) if local_package: output.info('Already installed!') installed = True log_package_got_from_local_cache(package_ref) else: installed = self._retrieve_remote_package(package_ref, package_folder, output) self.handle_package_manifest(package_ref, installed) return installed
def get_recipe(self, conan_reference): output = ScopedOutput(str(conan_reference), self._out) def _refresh(): export_path = self._client_cache.export(conan_reference) rmdir(export_path) # It might need to remove shortpath rm_conandir(self._client_cache.source(conan_reference)) current_remote, _ = self._get_remote(conan_reference) output.info("Retrieving from remote '%s'..." % current_remote.name) self._remote_manager.get_recipe(conan_reference, export_path, current_remote) if self._update: output.info("Updated!") else: output.info("Installed!") # check if it is in disk conanfile_path = self._client_cache.conanfile(conan_reference) if os.path.exists(conanfile_path): log_recipe_got_from_local_cache(conan_reference) if self._check_updates: ret = self.update_available(conan_reference) if ret != 0: # Found and not equal remote, ref_remote = self._get_remote(conan_reference) if ret == 1: if not self._update: if remote != ref_remote: # Forced new remote output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u -r %s' to update it." % (remote.name, remote.name)) else: output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u' to update it." % remote.name) output.warn("Refused to install!") else: if remote != ref_remote: # Delete packages, could be non coherent with new remote DiskRemover(self._client_cache).remove_packages(conan_reference) _refresh() elif ret == -1: if not self._update: output.info("Current conanfile is newer " "than %s's one" % remote.name) else: output.error("Current conanfile is newer than %s's one. " "Run 'conan remove %s' and run install again " "to replace it." % (remote.name, conan_reference)) else: self._retrieve_recipe(conan_reference, output) if self._manifest_manager: # Just make sure that the recipe sources are there to check self.get_recipe_sources(conan_reference) remote = self._registry.get_ref(conan_reference) self._manifest_manager.check_recipe(conan_reference, remote) return conanfile_path
def get_package(self, package_ref, short_paths): """ obtain a package, either from disk or retrieve from remotes if necessary and not necessary to build """ output = ScopedOutput(str(package_ref.conan), self._out) package_folder = self._client_cache.package(package_ref, short_paths=short_paths) # Check current package status if os.path.exists(package_folder): if self._check_updates: read_manifest = self._client_cache.load_package_manifest(package_ref) try: # get_conan_digest can fail, not in server upstream_manifest = self.get_package_digest(package_ref) if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: output.warn("Current package is older than remote upstream one") if self._update: output.warn("Removing it to retrieve or build an updated one") rmdir(package_folder) else: output.warn("Current package is newer than remote upstream one") except ConanException: pass installed = False local_package = os.path.exists(package_folder) if local_package: output.info('Already installed!') installed = True log_package_got_from_local_cache(package_ref) else: installed = self._retrieve_remote_package(package_ref, package_folder, output) self.handle_package_manifest(package_ref, installed) return installed
def download(self, reference, package_ids, remote=None): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote """ assert(isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) package = remote_proxy.search(reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) if package_ids: remote_proxy.download_packages(reference, package_ids) else: packages_props = remote_proxy.search_packages(reference, None) if not packages_props: output = ScopedOutput(str(reference), self._user_io.out) output.warn("No remote binary packages found in remote") else: remote_proxy.download_packages(reference, list(packages_props.keys()))
def download(self, reference, package_ids, remote_name, recipe): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote @param only_recipe: download only the recipe """ assert(isinstance(reference, ConanFileReference)) remote_proxy = self.get_proxy(remote_name=remote_name) remote, _ = remote_proxy._get_remote() package = self._remote_manager.search_recipes(remote, reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) # First of all download package recipe remote_proxy.get_recipe(reference, check_updates=True, update=True) if recipe: return # Download the sources too, don't be lazy conan_file_path = self._client_cache.conanfile(reference) conanfile = load_conanfile_class(conan_file_path) complete_recipe_sources(self._remote_manager, self._client_cache, self._registry, conanfile, reference) if package_ids: remote_proxy.download_packages(reference, package_ids) else: self._user_io.out.info("Getting the complete package list " "from '%s'..." % str(reference)) packages_props = self._remote_manager.search_packages(remote, reference, None) if not packages_props: output = ScopedOutput(str(reference), self._user_io.out) output.warn("No remote binary packages found in remote") else: remote_proxy.download_packages(reference, list(packages_props.keys()))
def get_package(self, package_reference, force_build): """ obtain a package, either from disk or retrieve from remotes if necessary and not necessary to build """ output = ScopedOutput(str(package_reference.conan), self._out) package_folder = self._paths.package(package_reference) # Check current package status if path_exists(package_folder, self._paths.store): if self._check_integrity or self._check_updates: read_manifest, expected_manifest = self._paths.package_manifests(package_reference) if self._check_integrity: # Check if package is corrupted if read_manifest.file_sums != expected_manifest.file_sums: # If not valid package, ensure empty folder output.warn("Bad package '%s' detected! Removing " "package directory... " % str(package_reference.package_id)) rmdir(package_folder) if self._check_updates: try: # get_conan_digest can fail, not in server upstream_manifest = self.get_package_digest(package_reference) if upstream_manifest.file_sums != read_manifest.file_sums: if upstream_manifest.time > read_manifest.time: output.warn("Current package is older than remote upstream one") if self._update: output.warn("Removing it to retrieve or build an updated one") rmdir(package_folder) else: output.warn("Current package is newer than remote upstream one") except ConanException: pass if not force_build: local_package = os.path.exists(package_folder) if local_package: output = ScopedOutput(str(package_reference.conan), self._out) output.info('Already installed!') return True return self._retrieve_remote_package(package_reference, output) return False
def _get_recipe(self, conan_reference, check_updates, update): output = ScopedOutput(str(conan_reference), self._out) check_updates = check_updates or update # check if it is in disk conanfile_path = self._client_cache.conanfile(conan_reference) if os.path.exists(conanfile_path): if check_updates: ret = self.update_available(conan_reference) if ret != 0: # Found and not equal remote, ref_remote = self._get_remote(conan_reference) if ret == 1: if not update: if remote != ref_remote: # Forced new remote output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u -r %s' to update it." % (remote.name, remote.name)) else: output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u' to update it." % remote.name) output.warn("Refused to install!") else: DiskRemover(self._client_cache).remove(conan_reference) output.info("Retrieving from remote '%s'..." % remote.name) self._remote_manager.get_recipe(conan_reference, remote) output.info("Updated!") elif ret == -1: if not update: output.info("Current conanfile is newer than %s's one" % remote.name) else: output.error("Current conanfile is newer than %s's one. " "Run 'conan remove %s' and run install again " "to replace it." % (remote.name, conan_reference)) log_recipe_got_from_local_cache(conan_reference) self._recorder.recipe_fetched_from_cache(conan_reference) else: self._retrieve_recipe(conan_reference, output) if self._manifest_manager: # Just make sure that the recipe sources are there to check conanfile = load_conanfile_class(conanfile_path) complete_recipe_sources(self._remote_manager, self._client_cache, self._registry, conanfile, conan_reference) remote = self._registry.get_ref(conan_reference) self._manifest_manager.check_recipe(conan_reference, remote) return conanfile_path
def _get_recipe(self, conan_reference, check_updates, update): output = ScopedOutput(str(conan_reference), self._out) check_updates = check_updates or update # check if it is in disk conanfile_path = self._client_cache.conanfile(conan_reference) remote = None if os.path.exists(conanfile_path): if check_updates: ret = self.update_available(conan_reference) if ret != 0: # Found and not equal remote, ref_remote = self._get_remote(conan_reference) if ret == 1: if not update: if remote != ref_remote: # Forced new remote output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u -r %s' to update it." % (remote.name, remote.name)) else: output.warn("There is a new conanfile in '%s' remote. " "Execute 'install -u' to update it." % remote.name) output.warn("Refused to install!") else: DiskRemover(self._client_cache).remove(conan_reference) output.info("Retrieving from remote '%s'..." % remote.name) self._remote_manager.get_recipe(conan_reference, remote) self._registry.set_ref(conan_reference, remote) output.info("Updated!") elif ret == -1: if not update: output.info("Current conanfile is newer than %s's one" % remote.name) else: output.error("Current conanfile is newer than %s's one. " "Run 'conan remove %s' and run install again " "to replace it." % (remote.name, conan_reference)) log_recipe_got_from_local_cache(conan_reference) self._recorder.recipe_fetched_from_cache(conan_reference) else: remote = self._retrieve_recipe(conan_reference, output) if not remote: remote = self._registry.get_ref(conan_reference) return conanfile_path, remote
def get_package(self, package_reference, force_build, short_paths, check_outdated): """ obtain a package, either from disk or retrieve from remotes if necessary and not necessary to build """ output = ScopedOutput(str(package_reference.conan), self._out) package_folder = self._client_cache.package(package_reference, short_paths=short_paths) # Check current package status if os.path.exists(package_folder): if self._check_updates: read_manifest = self._client_cache.load_package_manifest(package_reference) try: # get_conan_digest can fail, not in server upstream_manifest = self.get_package_digest(package_reference) if upstream_manifest.file_sums != read_manifest.file_sums: if upstream_manifest.time > read_manifest.time: output.warn("Current package is older than remote upstream one") if self._update: output.warn("Removing it to retrieve or build an updated one") rmdir(package_folder) else: output.warn("Current package is newer than remote upstream one") except ConanException: pass installed = False if not force_build: local_package = os.path.exists(package_folder) if local_package: output.info('Already installed!') installed = True else: installed = self._retrieve_remote_package(package_reference, package_folder, output) # Check if the package is outdated if check_outdated and os.path.exists(package_folder): if self._package_outdated(package_reference, package_folder): output.info("Outdated package!") installed = False else: output.info("Package is up to date") self.handle_package_manifest(package_reference, installed) return installed
class ConanFile(object): """ The base class for all package recipes """ name = None version = None # Any str, can be "1.1" or whatever url = None # The URL where this File is located, as github, to collaborate in package # The license of the PACKAGE, just a shortcut, does not replace or # change the actual license of the source code license = None author = None # Main maintainer/responsible for the package, any format description = None topics = None homepage = None build_policy = None short_paths = False apply_env = True # Apply environment variables from requires deps_env_info and profiles exports = None exports_sources = None generators = ["txt"] revision_mode = "hash" # Vars to control the build steps (build(), package()) should_configure = True should_build = True should_install = True should_test = True in_local_cache = True develop = False # Defaulting the reference fields default_channel = None default_user = None # Settings and Options settings = None options = None default_options = None def __init__(self, output, runner, display_name="", user=None, channel=None): # an output stream (writeln, info, warn error) self.output = ScopedOutput(display_name, output) self.display_name = display_name # something that can run commands, as os.sytem self._conan_runner = runner self._conan_user = user self._conan_channel = channel self.compatible_packages = [] def initialize(self, settings, env): if isinstance(self.generators, str): self.generators = [self.generators] # User defined options self.options = create_options(self) self.requires = create_requirements(self) self.settings = create_settings(self, settings) if 'cppstd' in self.settings.fields: self.output.warn( "Setting 'cppstd' is deprecated in favor of 'compiler.cppstd'," " please update your recipe.") # needed variables to pack the project self.cpp_info = None # Will be initialized at processing time self.deps_cpp_info = DepsCppInfo() # environment variables declared in the package_info self.env_info = None # Will be initialized at processing time self.deps_env_info = DepsEnvInfo() # user declared variables self.user_info = None # Keys are the package names, and the values a dict with the vars self.deps_user_info = DepsUserInfo() # user specified env variables self._conan_env_values = env.copy() # user specified -e @property def env(self): """Apply the self.deps_env_info into a copy of self._conan_env_values (will prioritize the self._conan_env_values, user specified from profiles or -e first, then inherited)""" # Cannot be lazy cached, because it's called in configure node, and we still don't have # the deps_env_info objects available tmp_env_values = self._conan_env_values.copy() tmp_env_values.update(self.deps_env_info) ret, multiple = tmp_env_values.env_dicts(self.name) ret.update(multiple) return ret @property def channel(self): if not self._conan_channel: self._conan_channel = os.getenv( "CONAN_CHANNEL") or self.default_channel if not self._conan_channel: raise ConanException( "channel not defined, but self.channel is used in conanfile" ) return self._conan_channel @property def user(self): if not self._conan_user: self._conan_user = os.getenv("CONAN_USERNAME") or self.default_user if not self._conan_user: raise ConanException( "user not defined, but self.user is used in conanfile") return self._conan_user def collect_libs(self, folder=None): self.output.warn("'self.collect_libs' is deprecated, " "use 'tools.collect_libs(self)' instead") return tools.collect_libs(self, folder=folder) @property def build_policy_missing(self): return self.build_policy == "missing" @property def build_policy_always(self): return self.build_policy == "always" def source(self): pass def system_requirements(self): """ this method can be overwritten to implement logic for system package managers, as apt-get You can define self.global_system_requirements = True, if you want the installation to be for all packages (not depending on settings/options/requirements) """ def config_options(self): """ modify options, probably conditioned to some settings. This call is executed before config_settings. E.g. if self.settings.os == "Windows": del self.options.shared # shared/static not supported in win """ def configure(self): """ modify settings, probably conditioned to some options. This call is executed after config_options. E.g. if self.options.header_only: self.settings.clear() This is also the place for conditional requirements """ def build(self): """ build your project calling the desired build tools as done in the command line. E.g. self.run("cmake --build .") Or use the provided build helpers. E.g. cmake.build() """ self.output.warn("This conanfile has no build step") def package(self): """ package the needed files from source and build folders. E.g. self.copy("*.h", src="src/includes", dst="includes") """ self.output.warn("This conanfile has no package step") def package_info(self): """ define cpp_build_info, flags, etc """ def run(self, command, output=True, cwd=None, win_bash=False, subsystem=None, msys_mingw=True, ignore_errors=False, run_environment=False, with_login=True): def _run(): if not win_bash: return self._conan_runner(command, output, os.path.abspath(RUN_LOG_NAME), cwd) # FIXME: run in windows bash is not using output return tools.run_in_windows_bash(self, bashcmd=command, cwd=cwd, subsystem=subsystem, msys_mingw=msys_mingw, with_login=with_login) if run_environment: with tools.run_environment(self): if OSInfo().is_macos: command = 'DYLD_LIBRARY_PATH="%s" DYLD_FRAMEWORK_PATH="%s" %s' % \ (os.environ.get('DYLD_LIBRARY_PATH', ''), os.environ.get("DYLD_FRAMEWORK_PATH", ''), command) retcode = _run() else: retcode = _run() if not ignore_errors and retcode != 0: raise ConanException("Error %d while executing %s" % (retcode, command)) return retcode def package_id(self): """ modify the binary info, typically to narrow values e.g.: self.info.settings.compiler = "Any" => All compilers will generate same ID """ def test(self): """ test the generated executable. E.g. self.run("./example") """ raise ConanException( "You need to create a method 'test' in your test/conanfile.py") def __repr__(self): return self.display_name
def _build_node(self, conan_ref, conan_file, build_mode): # Compute conan_file package from local (already compiled) or from remote output = ScopedOutput(str(conan_ref), self._out) package_id = conan_file.info.package_id() self._out.writeln("") output.info("Installing package %s" % package_id) package_reference = PackageReference(conan_ref, package_id) conan_ref = package_reference.conan package_folder = self._paths.package(package_reference) build_folder = self._paths.build(package_reference) src_folder = self._paths.source(conan_ref) export_folder = self._paths.export(conan_ref) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) # Check if package is corrupted valid_package_digest = self._paths.valid_package_digest(package_reference) if os.path.exists(package_folder) and not valid_package_digest: # If not valid package, ensure empty folder output.warn("Bad package '%s' detected! Removing " "package directory... " % str(package_id)) rmdir(package_folder) # Check if any only_source pattern matches with package force_build = self._force_build(conan_ref, build_mode) if not force_build: local_package = os.path.exists(package_folder) if local_package: output.info("Package installed in %s" % package_folder) return output.info("Package not installed") remote_package = self._retrieve_remote_package(package_reference, output) if remote_package: return # Can we build? Only if we are forced or build_mode missing and package not exists build_allowed = force_build or build_mode is True if build_allowed: rmdir(build_folder) rmdir(package_folder) if force_build: output.warn("Forced build from source") self._build_package(export_folder, src_folder, build_folder, conan_file, output) # Creating ***info.txt files save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) output.info("Generated %s" % CONANINFO) save( os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file.deps_cpp_info, conan_file.cpp_info).content, ) output.info("Generated %s" % BUILD_INFO) os.chdir(build_folder) create_package(conan_file, build_folder, package_folder, output) else: self._raise_package_not_found_error(conan_ref, conan_file)
def _build(self, nodes_by_level, skip_private_nodes, deps_graph, profile_build_requires): """ The build assumes an input of conans ordered by degree, first level should be independent from each other, the next-second level should have dependencies only to first level conans. param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...] build_mode => ["*"] if user wrote "--build" => ["hello*", "bye*"] if user wrote "--build hello --build bye" => False if user wrote "never" => True if user wrote "missing" => "outdated" if user wrote "--build outdated" """ inverse = deps_graph.inverse_levels() flat = [] for level in inverse: level = sorted(level, key=lambda x: x.conan_ref) flat.extend(n for n in level if n not in skip_private_nodes) # Get the nodes in order and if we have to build them nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes) for conan_ref, package_id, conan_file, build_needed in nodes_to_process: output = ScopedOutput(str(conan_ref), self._out) if build_needed and (conan_ref, package_id) not in self._built_packages: package_ref = PackageReference(conan_ref, package_id) build_allowed = self._build_mode.allowed(conan_file, conan_ref) if not build_allowed: _raise_package_not_found_error(conan_file, conan_ref, output) if conan_file.build_policy_missing: output.info("Building package from source as defined by build_policy='missing'") elif self._build_mode.forced(conan_file, conan_ref): output.warn('Forced build from source') self._build_requires.install(conan_ref, conan_file, self, profile_build_requires, output) t1 = time.time() # Assign to node the propagated info self._propagate_info(conan_file, conan_ref, flat, deps_graph) builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output) with self._client_cache.conanfile_write_lock(conan_ref): self._remote_proxy.get_recipe_sources(conan_ref, conan_file.short_paths) builder.prepare_build() with self._client_cache.conanfile_read_lock(conan_ref): with self._client_cache.package_lock(builder.build_reference): builder.build() builder.package() self._remote_proxy.handle_package_manifest(package_ref, installed=True) package_folder = self._client_cache.package(package_ref, conan_file.short_paths) # Call the info method call_package_info(conan_file, package_folder) # Log build self._log_built_package(conan_file, package_ref, time.time() - t1) self._built_packages.add((conan_ref, package_id)) else: # Get the package, we have a not outdated remote package package_ref = None if conan_ref: package_ref = PackageReference(conan_ref, package_id) with self._client_cache.package_lock(package_ref): self._get_remote_package(conan_file, package_ref, output) # Assign to the node the propagated info # (conan_ref could be None if user project, but of course assign the info self._propagate_info(conan_file, conan_ref, flat, deps_graph) if package_ref: # Call the info method package_folder = self._client_cache.package(package_ref, conan_file.short_paths) call_package_info(conan_file, package_folder)
def _build(self, nodes_by_level, skip_private_nodes, build_mode): """ The build assumes an input of conans ordered by degree, first level should be independent from each other, the next-second level should have dependencies only to first level conans. param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...] build_mode => ["*"] if user wrote "--build" => ["hello*", "bye*"] if user wrote "--build hello --build bye" => False if user wrote "never" => True if user wrote "missing" => "outdated" if user wrote "--build outdated" """ inverse = self._deps_graph.inverse_levels() flat = [] for level in inverse: level = sorted(level, key=lambda x: x.conan_ref) flat.extend(level) # Get the nodes in order and if we have to build them nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes, build_mode) for conan_ref, package_id, conan_file, build_needed in nodes_to_process: if build_needed: build_allowed = build_mode.allowed(conan_ref, conan_file) if not build_allowed: self._raise_package_not_found_error(conan_ref, conan_file) output = ScopedOutput(str(conan_ref), self._out) package_ref = PackageReference(conan_ref, package_id) package_folder = self._client_cache.package(package_ref, conan_file.short_paths) if conan_file.build_policy_missing: output.info("Building package from source as defined by build_policy='missing'") elif build_mode.forced(conan_ref, conan_file): output.warn('Forced build from source') self._build_requires.install(conan_ref, conan_file) t1 = time.time() # Assign to node the propagated info self._propagate_info(conan_ref, conan_file, flat) self._remote_proxy.get_recipe_sources(conan_ref) # Call the conanfile's build method build_folder = self._build_conanfile(conan_ref, conan_file, package_ref, package_folder, output) # Call the conanfile's package method self._package_conanfile(conan_ref, conan_file, package_ref, build_folder, package_folder, output) # Call the info method self._package_info_conanfile(conan_ref, conan_file) duration = time.time() - t1 log_file = os.path.join(build_folder, RUN_LOG_NAME) log_file = log_file if os.path.exists(log_file) else None log_package_built(package_ref, duration, log_file) else: # Get the package, we have a not outdated remote package if conan_ref: self._get_package(conan_ref, conan_file) # Assign to the node the propagated info # (conan_ref could be None if user project, but of course assign the info self._propagate_info(conan_ref, conan_file, flat) # Call the info method self._package_info_conanfile(conan_ref, conan_file)
def _build(self, nodes_by_level, skip_private_nodes, build_mode): """ The build assumes an input of conans ordered by degree, first level should be independent from each other, the next-second level should have dependencies only to first level conans. param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...] build_mode => ["*"] if user wrote "--build" => ["hello*", "bye*"] if user wrote "--build hello --build bye" => False if user wrote "never" => True if user wrote "missing" => "outdated" if user wrote "--build outdated" """ inverse = self._deps_graph.inverse_levels() flat = [] for level in inverse: level = sorted(level, key=lambda x: x.conan_ref) flat.extend(level) # Get the nodes in order and if we have to build them nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes, build_mode) for conan_ref, package_id, conan_file, build_needed in nodes_to_process: if build_needed: build_allowed = build_mode.allowed(conan_ref, conan_file) if not build_allowed: self._raise_package_not_found_error(conan_ref, conan_file) output = ScopedOutput(str(conan_ref), self._out) package_ref = PackageReference(conan_ref, package_id) package_folder = self._client_cache.package( package_ref, conan_file.short_paths) if conan_file.build_policy_missing: output.info( "Building package from source as defined by build_policy='missing'" ) elif build_mode.forced(conan_ref, conan_file): output.warn('Forced build from source') self._build_requires.install(conan_ref, conan_file) t1 = time.time() # Assign to node the propagated info self._propagate_info(conan_ref, conan_file, flat) self._remote_proxy.get_recipe_sources(conan_ref) # Call the conanfile's build method build_folder = self._build_conanfile(conan_ref, conan_file, package_ref, package_folder, output) # Call the conanfile's package method self._package_conanfile(conan_ref, conan_file, package_ref, build_folder, package_folder, output) # Call the info method self._package_info_conanfile(conan_ref, conan_file) duration = time.time() - t1 log_file = os.path.join(build_folder, RUN_LOG_NAME) log_file = log_file if os.path.exists(log_file) else None log_package_built(package_ref, duration, log_file) else: # Get the package, we have a not outdated remote package if conan_ref: self._get_package(conan_ref, conan_file) # Assign to the node the propagated info # (conan_ref could be None if user project, but of course assign the info self._propagate_info(conan_ref, conan_file, flat) # Call the info method self._package_info_conanfile(conan_ref, conan_file)
def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_name): assert node.binary is None conan_ref, conanfile = node.conan_ref, node.conanfile revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False) package_id = conanfile.info.package_id() package_ref = PackageReference(conan_ref, package_id) # Check that this same reference hasn't already been checked previous_node = evaluated_references.get(package_ref) if previous_node: node.binary = previous_node.binary node.binary_remote = previous_node.binary_remote return evaluated_references[package_ref] = node output = ScopedOutput(str(conan_ref), self._out) if build_mode.forced(conanfile, conan_ref): output.warn('Forced build from source') node.binary = BINARY_BUILD return package_folder = self._client_cache.package( package_ref, short_paths=conanfile.short_paths) # Check if dirty, to remove it local_project = self._workspace[conan_ref] if self._workspace else None if local_project: node.binary = BINARY_WORKSPACE return with self._client_cache.package_lock(package_ref): if is_dirty(package_folder): output.warn("Package is corrupted, removing folder: %s" % package_folder) rmdir(package_folder) if remote_name: remote = self._registry.remotes.get(remote_name) else: # If the remote_name is not given, follow the binary remote, or # the recipe remote # If it is defined it won't iterate (might change in conan2.0) remote = self._registry.prefs.get( package_ref) or self._registry.refs.get(conan_ref) remotes = self._registry.remotes.list if os.path.exists(package_folder): if update: if remote: if self._check_update(package_folder, package_ref, remote, output, node): node.binary = BINARY_UPDATE if build_mode.outdated: package_hash = self._get_package_info( package_ref, remote).recipe_hash elif remotes: pass else: output.warn("Can't update, no remote defined") if not node.binary: node.binary = BINARY_CACHE package_hash = ConanInfo.load_from_package( package_folder).recipe_hash else: # Binary does NOT exist locally if not revisions_enabled and not node.revision_pinned: # Do not search for packages for the specific resolved recipe revision but all package_ref = package_ref.copy_clear_rev() remote_info = None if remote: remote_info = self._get_package_info(package_ref, remote) # If the "remote" came from the registry but the user didn't specified the -r, with # revisions iterate all remotes if not remote or (not remote_info and revisions_enabled and not remote_name): for r in remotes: remote_info = self._get_package_info(package_ref, r) if remote_info: remote = r break if remote_info: node.binary = BINARY_DOWNLOAD package_hash = remote_info.recipe_hash else: if build_mode.allowed(conanfile, conan_ref): node.binary = BINARY_BUILD else: node.binary = BINARY_MISSING if build_mode.outdated: if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE): local_recipe_hash = self._client_cache.load_manifest( package_ref.conan).summary_hash if local_recipe_hash != package_hash: output.info("Outdated package!") node.binary = BINARY_BUILD else: output.info("Package is up to date") node.binary_remote = remote
def _build_node(self, conan_ref, conan_file, build_mode): # Compute conan_file package from local (already compiled) or from remote output = ScopedOutput(str(conan_ref), self._out) package_id = conan_file.info.package_id() package_reference = PackageReference(conan_ref, package_id) check_outdated = build_mode == "outdated" conan_ref = package_reference.conan package_folder = self._client_cache.package(package_reference, conan_file.short_paths) build_folder = self._client_cache.build(package_reference, conan_file.short_paths) src_folder = self._client_cache.source(conan_ref, conan_file.short_paths) export_folder = self._client_cache.export(conan_ref) # If already exists do not dirt the output, the common situation # is that package is already installed and OK. If don't, the proxy # will print some other message about it if not os.path.exists(package_folder): output.info("Installing package %s" % package_id) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) force_build = self._build_forced(conan_ref, build_mode, conan_file) if self._remote_proxy.get_package(package_reference, force_build, short_paths=conan_file.short_paths, check_outdated=check_outdated): return # we need and can build? Only if we are forced or build_mode missing and package not exists # Option "--build outdated" means: missing or outdated, so don't care if it's really oudated # just build it. build = force_build or build_mode is True or check_outdated or conan_file.build_policy_missing if build: if not force_build and not build_mode: output.info( "Building package from source as defined by build_policy='missing'" ) try: rmdir(build_folder) rmdir(package_folder) except Exception as e: raise ConanException( "%s\n\nCouldn't remove folder, might be busy or open\n" "Close any app using it, and retry" % str(e)) if force_build: output.warn('Forced build from source') with environment_append(conan_file.env): self._build_package(export_folder, src_folder, build_folder, conan_file, output) # FIXME: Is weak to assign here the recipe_hash conan_file.info.recipe_hash = self._client_cache.load_manifest( conan_ref).summary_hash # Creating ***info.txt files save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) output.info("Generated %s" % CONANINFO) save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content) output.info("Generated %s" % BUILD_INFO) save(os.path.join(build_folder, CONANENV), ConanEnvGenerator(conan_file).content) output.info("Generated %s" % CONANENV) os.chdir(build_folder) with environment_append(conan_file.env): create_package(conan_file, build_folder, package_folder, output) self._remote_proxy.handle_package_manifest(package_reference, installed=True) else: self._raise_package_not_found_error(conan_ref, conan_file)
def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_name): assert node.binary is None conan_ref, conanfile = node.conan_ref, node.conanfile package_id = conanfile.info.package_id() package_ref = PackageReference(conan_ref, package_id) # Check that this same reference hasn't already been checked previous_node = evaluated_references.get(package_ref) if previous_node: node.binary = previous_node.binary node.binary_remote = previous_node.binary_remote return evaluated_references[package_ref] = node output = ScopedOutput(str(conan_ref), self._out) if build_mode.forced(conanfile, conan_ref): output.warn('Forced build from source') node.binary = BINARY_BUILD return package_folder = self._client_cache.package( package_ref, short_paths=conanfile.short_paths) # Check if dirty, to remove it local_project = self._workspace[conan_ref] if self._workspace else None if local_project: node.binary = BINARY_WORKSPACE return with self._client_cache.package_lock(package_ref): if is_dirty(package_folder): output.warn("Package is corrupted, removing folder: %s" % package_folder) rmdir(package_folder) if remote_name: remote = self._registry.remote(remote_name) else: remote = self._registry.get_recipe_remote(conan_ref) remotes = self._registry.remotes if os.path.exists(package_folder): if update: if remote: if self._check_update(package_folder, package_ref, remote, output): node.binary = BINARY_UPDATE if build_mode.outdated: package_hash = self._get_package_info( package_ref, remote).recipe_hash elif remotes: pass else: output.warn("Can't update, no remote defined") if not node.binary: node.binary = BINARY_CACHE package_hash = ConanInfo.load_from_package( package_folder).recipe_hash else: # Binary does NOT exist locally remote_info = None if remote: remote_info = self._get_package_info(package_ref, remote) elif remotes: # Iterate all remotes to get this binary for r in remotes: remote_info = self._get_package_info(package_ref, r) if remote_info: remote = r break if remote_info: node.binary = BINARY_DOWNLOAD package_hash = remote_info.recipe_hash else: if build_mode.allowed(conanfile, conan_ref): node.binary = BINARY_BUILD else: node.binary = BINARY_MISSING if build_mode.outdated: if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE): local_recipe_hash = self._client_cache.load_manifest( package_ref.conan).summary_hash if local_recipe_hash != package_hash: output.info("Outdated package!") node.binary = BINARY_BUILD else: output.info("Package is up to date") node.binary_remote = remote
class ConanFile(object): """ The base class for all package recipes """ name = None version = None # Any str, can be "1.1" or whatever url = None # The URL where this File is located, as github, to collaborate in package # The license of the PACKAGE, just a shortcut, does not replace or # change the actual license of the source code license = None author = None # Main maintainer/responsible for the package, any format description = None topics = None homepage = None build_policy = None short_paths = False apply_env = True # Apply environment variables from requires deps_env_info and profiles exports = None exports_sources = None generators = ["txt"] revision_mode = "hash" # Vars to control the build steps (build(), package()) should_configure = True should_build = True should_install = True should_test = True in_local_cache = True develop = False # Defaulting the reference fields default_channel = None default_user = None # Settings and Options settings = None options = None default_options = None provides = None deprecated = None # Folders folders = None patterns = None # Run in windows bash win_bash = None def __init__(self, output, runner, display_name="", user=None, channel=None): # an output stream (writeln, info, warn error) self.output = ScopedOutput(display_name, output) self.display_name = display_name # something that can run commands, as os.sytem self._conan_runner = runner self._conan_user = user self._conan_channel = channel self.compatible_packages = [] self._conan_using_build_profile = False self._conan_requester = None self.buildenv_info = Environment(self) self.runenv_info = Environment(self) # At the moment only for build_requires, others will be ignored self.conf_info = Conf() self._conan_buildenv = None # The profile buildenv, will be assigned initialize() self._conan_node = None # access to container Node object, to access info, context, deps... self._conan_new_cpp_info = None # Will be calculated lazy in the getter self._conan_dependencies = None self.env_scripts = {} # Accumulate the env scripts generated in order # layout() method related variables: self.folders = Folders() self.patterns = Patterns() self.cpp = Infos() self.patterns.source.include = ["*.h", "*.hpp", "*.hxx"] self.patterns.source.lib = [] self.patterns.source.bin = [] self.patterns.build.include = ["*.h", "*.hpp", "*.hxx"] self.patterns.build.lib = ["*.so", "*.so.*", "*.a", "*.lib", "*.dylib"] self.patterns.build.bin = ["*.exe", "*.dll"] self.cpp.package.includedirs = ["include"] self.cpp.package.libdirs = ["lib"] self.cpp.package.bindirs = ["bin"] self.cpp.package.resdirs = ["res"] self.cpp.package.builddirs = [""] self.cpp.package.frameworkdirs = ["Frameworks"] @property def context(self): return self._conan_node.context @property def dependencies(self): # Caching it, this object is requested many times if self._conan_dependencies is None: self._conan_dependencies = ConanFileDependencies.from_node( self._conan_node) return self._conan_dependencies @property def ref(self): return self._conan_node.ref @property def pref(self): return self._conan_node.pref @property def buildenv(self): # Lazy computation of the package buildenv based on the profileone if not isinstance(self._conan_buildenv, Environment): # TODO: missing user/channel ref_str = "{}/{}".format(self.name, self.version) self._conan_buildenv = self._conan_buildenv.get_env(self, ref_str) return self._conan_buildenv def initialize(self, settings, env, buildenv=None): self._conan_buildenv = buildenv if isinstance(self.generators, str): self.generators = [self.generators] # User defined options self.options = create_options(self) self.requires = create_requirements(self) self.settings = create_settings(self, settings) conan_v2_error( "Setting 'cppstd' is deprecated in favor of 'compiler.cppstd'," " please update your recipe.", 'cppstd' in self.settings.fields) # needed variables to pack the project self.cpp_info = None # Will be initialized at processing time self._conan_dep_cpp_info = None # Will be initialized at processing time self.deps_cpp_info = DepsCppInfo() # environment variables declared in the package_info self.env_info = None # Will be initialized at processing time self.deps_env_info = DepsEnvInfo() # user declared variables self.user_info = None # Keys are the package names (only 'host' if different contexts) self.deps_user_info = DepsUserInfo() # user specified env variables self._conan_env_values = env.copy() # user specified -e if self.description is not None and not isinstance( self.description, six.string_types): raise ConanException("Recipe 'description' must be a string.") if not hasattr(self, "virtualbuildenv" ): # Allow the user to override it with True or False self.virtualbuildenv = True if not hasattr(self, "virtualrunenv" ): # Allow the user to override it with True or False self.virtualrunenv = True @property def new_cpp_info(self): if not self._conan_new_cpp_info: self._conan_new_cpp_info = from_old_cppinfo(self.cpp_info) return self._conan_new_cpp_info @property def source_folder(self): return self.folders.source_folder @source_folder.setter def source_folder(self, folder): self.folders.set_base_source(folder) @property def build_folder(self): return self.folders.build_folder @build_folder.setter def build_folder(self, folder): self.folders.set_base_build(folder) @property def package_folder(self): return self.folders.package_folder @package_folder.setter def package_folder(self, folder): self.folders.set_base_package(folder) @property def install_folder(self): # FIXME: Remove in 2.0, no self.install_folder return self.folders.base_install @install_folder.setter def install_folder(self, folder): # FIXME: Remove in 2.0, no self.install_folder self.folders.set_base_install(folder) @property def generators_folder(self): # FIXME: Remove in 2.0, no self.install_folder return self.folders.generators_folder if self.folders.generators else self.install_folder @property def imports_folder(self): return self.folders.imports_folder @imports_folder.setter def imports_folder(self, folder): self.folders.set_base_imports(folder) @property def env(self): """Apply the self.deps_env_info into a copy of self._conan_env_values (will prioritize the self._conan_env_values, user specified from profiles or -e first, then inherited)""" # Cannot be lazy cached, because it's called in configure node, and we still don't have # the deps_env_info objects available tmp_env_values = self._conan_env_values.copy() tmp_env_values.update(self.deps_env_info) ret, multiple = tmp_env_values.env_dicts(self.name, self.version, self._conan_user, self._conan_channel) ret.update(multiple) return ret @property def channel(self): if not self._conan_channel: _env_channel = os.getenv("CONAN_CHANNEL") conan_v2_error( "Environment variable 'CONAN_CHANNEL' is deprecated", _env_channel) self._conan_channel = _env_channel or self.default_channel if not self._conan_channel: raise ConanException( "channel not defined, but self.channel is used in conanfile" ) return self._conan_channel @property def user(self): if not self._conan_user: _env_username = os.getenv("CONAN_USERNAME") conan_v2_error( "Environment variable 'CONAN_USERNAME' is deprecated", _env_username) self._conan_user = _env_username or self.default_user if not self._conan_user: raise ConanException( "user not defined, but self.user is used in conanfile") return self._conan_user def collect_libs(self, folder=None): conan_v2_error( "'self.collect_libs' is deprecated, use 'tools.collect_libs(self)' instead" ) return tools.collect_libs(self, folder=folder) @property def build_policy_missing(self): return self.build_policy == "missing" @property def build_policy_always(self): return self.build_policy == "always" def source(self): pass def system_requirements(self): """ this method can be overwritten to implement logic for system package managers, as apt-get You can define self.global_system_requirements = True, if you want the installation to be for all packages (not depending on settings/options/requirements) """ def config_options(self): """ modify options, probably conditioned to some settings. This call is executed before config_settings. E.g. if self.settings.os == "Windows": del self.options.shared # shared/static not supported in win """ def configure(self): """ modify settings, probably conditioned to some options. This call is executed after config_options. E.g. if self.options.header_only: self.settings.clear() This is also the place for conditional requirements """ def build(self): """ build your project calling the desired build tools as done in the command line. E.g. self.run("cmake --build .") Or use the provided build helpers. E.g. cmake.build() """ self.output.warn("This conanfile has no build step") def package(self): """ package the needed files from source and build folders. E.g. self.copy("*.h", src="src/includes", dst="includes") """ self.output.warn("This conanfile has no package step") def package_info(self): """ define cpp_build_info, flags, etc """ def run(self, command, output=True, cwd=None, win_bash=False, subsystem=None, msys_mingw=True, ignore_errors=False, run_environment=False, with_login=True, env=None): # NOTE: "self.win_bash" is the new parameter "win_bash" for Conan 2.0 def _run(cmd, _env): # FIXME: run in windows bash is not using output if platform.system() == "Windows": if win_bash: return tools.run_in_windows_bash(self, bashcmd=cmd, cwd=cwd, subsystem=subsystem, msys_mingw=msys_mingw, with_login=with_login) elif self.win_bash: # New, Conan 2.0 from conan.tools.microsoft.subsystems import run_in_windows_bash return run_in_windows_bash(self, command=cmd, cwd=cwd, env=_env) if _env is None: _env = "conanbuild" wrapped_cmd = environment_wrap_command(self, _env, cmd, cwd=self.generators_folder) return self._conan_runner(wrapped_cmd, output, os.path.abspath(RUN_LOG_NAME), cwd) if run_environment: # When using_build_profile the required environment is already applied through # 'conanfile.env' in the contextmanager 'get_env_context_manager' with tools.run_environment( self) if not self._conan_using_build_profile else no_op(): if OSInfo().is_macos and isinstance(command, string_types): # Security policy on macOS clears this variable when executing /bin/sh. To # keep its value, set it again inside the shell when running the command. command = 'DYLD_LIBRARY_PATH="%s" DYLD_FRAMEWORK_PATH="%s" %s' % \ (os.environ.get('DYLD_LIBRARY_PATH', ''), os.environ.get("DYLD_FRAMEWORK_PATH", ''), command) retcode = _run(command, env) else: retcode = _run(command, env) if not ignore_errors and retcode != 0: raise ConanException("Error %d while executing %s" % (retcode, command)) return retcode def package_id(self): """ modify the binary info, typically to narrow values e.g.: self.info.settings.compiler = "Any" => All compilers will generate same ID """ def test(self): """ test the generated executable. E.g. self.run("./example") """ raise ConanException( "You need to create a method 'test' in your test/conanfile.py") def __repr__(self): return self.display_name