def download(app, ref, package_ids, remote, recipe, recorder, remotes): out, remote_manager, cache, loader = app.out, app.remote_manager, app.cache, app.loader hook_manager = app.hook_manager assert (isinstance(ref, ConanFileReference)) output = ScopedOutput(str(ref), out) hook_manager.execute("pre_download", reference=ref, remote=remote) try: ref = remote_manager.get_recipe(ref, remote) except NotFoundException: raise RecipeNotFoundException(ref) conan_file_path = cache.package_layout(ref).conanfile() conanfile = loader.load_basic(conan_file_path) # Download the sources too, don't be lazy retrieve_exports_sources(remote_manager, cache, conanfile, ref, remotes) if not recipe: # Not only the recipe if not package_ids: # User didn't specify a specific package binary output.info("Getting the complete package list from '%s'..." % ref.full_str()) packages_props = remote_manager.search_packages(remote, ref, None) package_ids = list(packages_props.keys()) if not package_ids: output.warn("No remote binary packages found in remote") parallel = cache.config.parallel_download _download_binaries(conanfile, ref, package_ids, cache, remote_manager, remote, output, recorder, parallel) hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=ref, remote=remote)
def prepare_recipe(self, ref, conanfile, remote, remotes, policy): """ do a bunch of things that are necessary before actually executing the upload: - retrieve exports_sources to complete the recipe if necessary - compress the artifacts in conan_export.tgz and conan_export_sources.tgz - check if package is ok to be uploaded, if scm info missing, will raise - check if the remote recipe is newer, raise - compare and decide which files need to be uploaded (and deleted from server) """ layout = self._cache.package_layout(ref) current_remote_name = layout.load_metadata().recipe.remote if remote.name != current_remote_name: retrieve_exports_sources(self._remote_manager, self._cache, conanfile, ref, remotes) conanfile_path = layout.conanfile() self._hook_manager.execute("pre_upload_recipe", conanfile_path=conanfile_path, reference=ref, remote=remote) t1 = time.time() cache_files = self._compress_recipe_files(layout, ref) with layout.update_metadata() as metadata: metadata.recipe.checksums = calc_files_checksum(cache_files) local_manifest = FileTreeManifest.loads( load(cache_files["conanmanifest.txt"])) remote_manifest = None if policy != UPLOAD_POLICY_FORCE: # Check SCM data for auto fields if hasattr(conanfile, "scm") and (conanfile.scm.get("url") == "auto" or conanfile.scm.get("revision") == "auto" or conanfile.scm.get("type") is None or conanfile.scm.get("url") is None or conanfile.scm.get("revision") is None): raise ConanException( "The recipe contains invalid data in the 'scm' attribute" " (some 'auto' values or missing fields 'type', 'url' or" " 'revision'). Use '--force' to ignore this error or export" " again the recipe ('conan export' or 'conan create') to" " fix these issues.") remote_manifest = self._check_recipe_date(ref, remote, local_manifest) if policy == UPLOAD_POLICY_SKIP: return files_to_upload, deleted = self._recipe_files_to_upload( ref, policy, cache_files, remote, remote_manifest, local_manifest) return files_to_upload, deleted, cache_files, conanfile_path, t1, current_remote_name, layout
def _prepare_sources(self, conanfile, pref, package_layout, remotes): export_folder = package_layout.export() export_source_folder = package_layout.export_sources() scm_sources_folder = package_layout.scm_sources() conanfile_path = package_layout.conanfile() source_folder = package_layout.source() retrieve_exports_sources(self._remote_manager, self._cache, conanfile, pref.ref, remotes) config_source(export_folder, export_source_folder, scm_sources_folder, source_folder, conanfile, self._output, conanfile_path, pref.ref, self._hook_manager, self._cache)
def _build_package(self, node, output, keep_build, remotes): conanfile = node.conanfile # It is necessary to complete the sources of python requires, which might be used # Only the legacy python_requires allow this python_requires = getattr(conanfile, "python_requires", None) if python_requires and isinstance(python_requires, dict): # Old legacy python_requires for python_require in python_requires.values(): assert python_require.ref.revision is not None, \ "Installer should receive python_require.ref always" retrieve_exports_sources(self._remote_manager, self._cache, python_require.conanfile, python_require.ref, remotes) builder = _PackageBuilder(self._cache, output, self._hook_manager, self._remote_manager, self._generator_manager) pref = builder.build_package(node, keep_build, self._recorder, remotes) if node.graph_lock_node: node.graph_lock_node.prev = pref.revision return pref
def deps_install(app, ref_or_path, install_folder, base_folder, graph_info, remotes=None, build_modes=None, update=False, manifest_folder=None, manifest_verify=False, manifest_interactive=False, generators=None, no_imports=False, create_reference=None, keep_build=False, recorder=None, lockfile_node_id=None, is_build_require=False, add_txt_generator=True, require_overrides=None, conanfile_path=None, test=None, source_folder=None, output_folder=None): """ Fetch and build all dependencies for the given reference @param app: The ConanApp instance with all collaborators @param ref_or_path: ConanFileReference or path to user space conanfile @param install_folder: where the output files will be saved @param build_modes: List of build_modes specified @param update: Check for updated in the upstream remotes (and update) @param manifest_folder: Folder to install the manifests @param manifest_verify: Verify dependencies manifests against stored ones @param manifest_interactive: Install deps manifests in folder for later verify, asking user for confirmation @param generators: List of generators from command line. @param no_imports: Install specified packages but avoid running imports @param add_txt_generator: Add the txt to the list of generators """ out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache remote_manager, hook_manager = app.remote_manager, app.hook_manager profile_host, profile_build = graph_info.profile_host, graph_info.profile_build if profile_build: out.info("Configuration (profile_host):") out.writeln(profile_host.dumps()) out.info("Configuration (profile_build):") out.writeln(profile_build.dumps()) else: out.info("Configuration:") out.writeln(profile_host.dumps()) deps_graph = graph_manager.load_graph(ref_or_path, create_reference, graph_info, build_modes, False, update, remotes, recorder, lockfile_node_id=lockfile_node_id, is_build_require=is_build_require, require_overrides=require_overrides) graph_lock = graph_info.graph_lock # After the graph is loaded it is defined root_node = deps_graph.root conanfile = root_node.conanfile if root_node.recipe == RECIPE_VIRTUAL: out.highlight("Installing package: %s" % str(ref_or_path)) else: conanfile.output.highlight("Installing package") print_graph(deps_graph, out) try: if cross_building(conanfile): settings = get_cross_building_settings(conanfile) message = "Cross-build from '%s:%s' to '%s:%s'" % settings out.writeln(message, Color.BRIGHT_MAGENTA) except ConanException: # Setting os doesn't exist pass installer = BinaryInstaller(app, recorder=recorder) # TODO: Extract this from the GraphManager, reuse same object, check args earlier build_modes = BuildMode(build_modes, out) installer.install(deps_graph, remotes, build_modes, update, profile_host, profile_build, graph_lock, keep_build=keep_build) graph_lock.complete_matching_prevs() if manifest_folder: manifest_manager = ManifestManager(manifest_folder, user_io=user_io, cache=cache) for node in deps_graph.nodes: if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL): continue retrieve_exports_sources(remote_manager, cache, node.conanfile, node.ref, remotes) manifest_manager.check_graph(deps_graph, verify=manifest_verify, interactive=manifest_interactive) manifest_manager.print_log() if hasattr(conanfile, "layout") and not test: conanfile.folders.set_base_source(source_folder or conanfile_path) conanfile.folders.set_base_install(output_folder or conanfile_path) conanfile.folders.set_base_imports(output_folder or conanfile_path) conanfile.folders.set_base_generators(output_folder or conanfile_path) else: conanfile.folders.set_base_install(install_folder) conanfile.folders.set_base_imports(install_folder) conanfile.folders.set_base_generators(base_folder) output = conanfile.output if root_node.recipe != RECIPE_VIRTUAL else out if install_folder: # Write generators tmp = list( conanfile.generators) # Add the command line specified generators generators = set(generators) if generators else set() tmp.extend([g for g in generators if g not in tmp]) if add_txt_generator: tmp.append("txt") conanfile.generators = tmp app.generator_manager.write_generators(conanfile, install_folder, conanfile.generators_folder, output) write_toolchain(conanfile, conanfile.generators_folder, output) if not isinstance(ref_or_path, ConanFileReference): # Write conaninfo content = normalize(conanfile.info.dumps()) save(os.path.join(install_folder, CONANINFO), content) output.info("Generated %s" % CONANINFO) graph_info.save(install_folder) output.info("Generated graphinfo") graph_lock_file = GraphLockFile(profile_host, profile_build, graph_lock) graph_lock_file.save(os.path.join(install_folder, "conan.lock")) if not no_imports: run_imports(conanfile) if type(conanfile ).system_requirements != ConanFile.system_requirements: call_system_requirements(conanfile, conanfile.output) if not create_reference and isinstance(ref_or_path, ConanFileReference): # The conanfile loaded is a virtual one. The one w deploy is the first level one neighbours = deps_graph.root.neighbors() deploy_conanfile = neighbours[0].conanfile if hasattr(deploy_conanfile, "deploy") and callable( deploy_conanfile.deploy): run_deploy(deploy_conanfile, install_folder)
def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote, remotes): layout = self._cache.package_layout(ref) current_remote_name = layout.load_metadata().recipe.remote if remote.name != current_remote_name: retrieve_exports_sources(self._remote_manager, self._cache, conanfile, ref, remotes) conanfile_path = layout.conanfile() self._hook_manager.execute("pre_upload_recipe", conanfile_path=conanfile_path, reference=ref, remote=remote) t1 = time.time() cache_files = self._compress_recipe_files(layout, ref) with layout.update_metadata() as metadata: metadata.recipe.checksums = calc_files_checksum(cache_files) local_manifest = FileTreeManifest.loads( load(cache_files["conanmanifest.txt"])) remote_manifest = None if policy != UPLOAD_POLICY_FORCE: # Check SCM data for auto fields if hasattr(conanfile, "scm") and (conanfile.scm.get("url") == "auto" or conanfile.scm.get("revision") == "auto" or conanfile.scm.get("type") is None or conanfile.scm.get("url") is None or conanfile.scm.get("revision") is None): raise ConanException( "The recipe contains invalid data in the 'scm' attribute" " (some 'auto' values or missing fields 'type', 'url' or" " 'revision'). Use '--force' to ignore this error or export" " again the recipe ('conan export' or 'conan create') to" " fix these issues.") remote_manifest = self._check_recipe_date(ref, remote, local_manifest) if policy == UPLOAD_POLICY_SKIP: return ref files_to_upload, deleted = self._recipe_files_to_upload( ref, policy, cache_files, remote, remote_manifest, local_manifest) if files_to_upload or deleted: self._remote_manager.upload_recipe(ref, files_to_upload, deleted, remote, retry, retry_wait) self._upload_recipe_end_msg(ref, remote) else: self._output.info("Recipe is up to date, upload skipped") duration = time.time() - t1 log_recipe_upload(ref, duration, cache_files, remote.name) self._hook_manager.execute("post_upload_recipe", conanfile_path=conanfile_path, reference=ref, remote=remote) # The recipe wasn't in the registry or it has changed the revision field only if not current_remote_name: with layout.update_metadata() as metadata: metadata.recipe.remote = remote.name return ref
def _prepare_sources(cache, ref, remote_manager, loader, remotes): conan_file_path = cache.package_layout(ref).conanfile() conanfile = loader.load_basic(conan_file_path) retrieve_exports_sources(remote_manager, cache, conanfile, ref, remotes) return conanfile.short_paths