def save(self, folder, filename=None): filename = filename or GRAPH_INFO_FILE p = os.path.join(folder, filename) serialized_graph_str = self._dumps() save(p, serialized_graph_str) # A bit hacky, but to avoid repetition by now graph_lock_file = GraphLockFile(self.profile_host, self.graph_lock) graph_lock_file.save(os.path.join(folder, LOCKFILE))
def _handle_node_editable(self, node, graph_info): # Get source of information package_layout = self._cache.package_layout(node.ref) base_path = package_layout.base_folder() self._call_package_info(node.conanfile, package_folder=base_path, ref=node.ref) node.conanfile.cpp_info.filter_empty = False # Try with package-provided file editable_cpp_info = package_layout.editable_cpp_info() if editable_cpp_info: editable_cpp_info.apply_to(node.ref, node.conanfile.cpp_info, settings=node.conanfile.settings, options=node.conanfile.options) build_folder = editable_cpp_info.folder( node.ref, EditableLayout.BUILD_FOLDER, settings=node.conanfile.settings, options=node.conanfile.options) if build_folder is not None: build_folder = os.path.join(base_path, build_folder) output = node.conanfile.output self._generator_manager.write_generators( node.conanfile, build_folder, output) write_toolchain(node.conanfile, build_folder, output) save(os.path.join(build_folder, CONANINFO), node.conanfile.info.dumps()) output.info("Generated %s" % CONANINFO) graph_info_node = GraphInfo(graph_info.profile_host, root_ref=node.ref) graph_info_node.options = node.conanfile.options.values graph_info_node.graph_lock = graph_info.graph_lock graph_info_node.save(build_folder) output.info("Generated graphinfo") graph_lock_file = GraphLockFile(graph_info.profile_host, graph_info.profile_build, graph_info.graph_lock) graph_lock_file.save(os.path.join(build_folder, "conan.lock")) save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(node.conanfile).content) output.info("Generated %s" % BUILD_INFO) # Build step might need DLLs, binaries as protoc to generate source files # So execute imports() before build, storing the list of copied_files copied_files = run_imports(node.conanfile, build_folder) report_copied_files(copied_files, output)
def load_consumer_conanfile(self, conanfile_path, info_folder, deps_info_required=False, test=False): """loads a conanfile for local flow: source, imports, package, build """ try: graph_info = GraphInfo.load(info_folder) graph_lock_file = GraphLockFile.load( info_folder, self._cache.config.revisions_enabled) graph_lock = graph_lock_file.graph_lock self._output.info( "Using lockfile: '{}/conan.lock'".format(info_folder)) profile_host = graph_lock_file.profile_host self._output.info("Using cached profile from lockfile") except IOError: # Only if file is missing graph_lock = None # This is very dirty, should be removed for Conan 2.0 (source() method only) profile_host = self._cache.default_profile profile_host.process_settings(self._cache) name, version, user, channel = None, None, None, None else: name, version, user, channel, _ = graph_info.root profile_host.process_settings(self._cache, preprocess=False) # This is the hack of recovering the options from the graph_info profile_host.options.update(graph_info.options) if conanfile_path.endswith(".py"): lock_python_requires = None if graph_lock and not test: # Only lock python requires if it is not test_package node_id = graph_lock.get_node(graph_info.root) lock_python_requires = graph_lock.python_requires(node_id) conanfile = self._loader.load_consumer( conanfile_path, profile_host=profile_host, name=name, version=version, user=user, channel=channel, lock_python_requires=lock_python_requires) if test: conanfile.display_name = "%s (test package)" % str(test) conanfile.output.scope = conanfile.display_name with get_env_context_manager(conanfile, without_python=True): with conanfile_exception_formatter(str(conanfile), "config_options"): conanfile.config_options() with conanfile_exception_formatter(str(conanfile), "configure"): conanfile.configure() conanfile.settings.validate() # All has to be ok! conanfile.options.validate() else: conanfile = self._loader.load_conanfile_txt( conanfile_path, profile_host) load_deps_info(info_folder, conanfile, required=deps_info_required) return conanfile
def update_bundle(bundle_path, revisions_enabled): """ Update both the bundle information as well as every individual lockfile, from the information that was modified in the individual lockfile. At the end, all lockfiles will have the same PREV for the binary of same package_id """ bundle = LockBundle() bundle.loads(load(bundle_path)) for node in bundle._nodes.values(): # Each node in bundle belongs to a "ref", and contains lockinfo for every package_id for bundle_package_ids in node["package_id"].values(): # Each package_id contains information of multiple lockfiles # First, compute the modified PREV from all lockfiles prev = modified = prev_lockfile = None for lockfile, nodes_ids in bundle_package_ids[ "lockfiles"].items(): graph_lock_conf = GraphLockFile.load( lockfile, revisions_enabled) graph_lock = graph_lock_conf.graph_lock for node_id in nodes_ids: # Make sure the PREV from lockfiles is consistent, it cannot be different lock_prev = graph_lock.nodes[node_id].prev if prev is None: prev = lock_prev prev_lockfile = lockfile modified = graph_lock.nodes[node_id].modified elif lock_prev is not None and prev != lock_prev: ref = graph_lock.nodes[node_id].ref msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format( ref, prev_lockfile, prev, lockfile, lock_prev) raise ConanException(msg) bundle_package_ids["prev"] = prev bundle_package_ids["modified"] = modified # Then, update all prev of all config lockfiles for lockfile, nodes_ids in bundle_package_ids[ "lockfiles"].items(): graph_lock_conf = GraphLockFile.load( lockfile, revisions_enabled) graph_lock = graph_lock_conf.graph_lock for node_id in nodes_ids: if graph_lock.nodes[node_id].prev is None: graph_lock.nodes[node_id].prev = prev graph_lock_conf.save(lockfile) save(bundle_path, bundle.dumps())
def clean_modified(bundle_path, revisions_enabled): bundle = LockBundle() bundle.loads(load(bundle_path)) for node in bundle._nodes.values(): for pkg in node["packages"]: pkg["modified"] = None for lockfile, nodes_ids in pkg["lockfiles"].items(): graph_lock_conf = GraphLockFile.load( lockfile, revisions_enabled) graph_lock_conf.graph_lock.clean_modified() graph_lock_conf.save(lockfile) save(bundle_path, bundle.dumps())
def create(lockfiles, revisions_enabled, cwd): def ref_convert(r): # Necessary so "build-order" output is usable by install if "@" not in r: if "#" in r: r = r.replace("#", "@#") else: r += "@" return r result = LockBundle() for lockfile_name in lockfiles: lockfile_abs = os.path.normpath(os.path.join(cwd, lockfile_name)) lockfile = GraphLockFile.load(lockfile_abs, revisions_enabled) lock = lockfile.graph_lock for id_, node in lock.nodes.items(): ref_str = node.ref.full_str() ref_str = ref_convert(ref_str) ref_node = result._nodes.setdefault(ref_str, {}) packages_node = ref_node.setdefault("packages", []) # Find existing package_id in the list of packages # This is the equivalent of a setdefault over a dict, but on a list for pkg in packages_node: if pkg["package_id"] == node.package_id: pid_node = pkg break else: pid_node = {"package_id": node.package_id} packages_node.append(pid_node) ids = pid_node.setdefault("lockfiles", {}) # TODO: Add check that this prev is always the same pid_node["prev"] = node.prev pid_node["modified"] = node.modified ids.setdefault(lockfile_name, []).append(id_) total_requires = node.requires + node.build_requires for require in total_requires: require_node = lock.nodes[require] ref = require_node.ref.full_str() ref = ref_convert(ref) requires = ref_node.setdefault("requires", []) if ref not in requires: requires.append(ref) return result
def deps_install(app, ref_or_path, install_folder, base_folder, graph_info, remotes=None, build_modes=None, update=False, manifest_folder=None, manifest_verify=False, manifest_interactive=False, generators=None, no_imports=False, create_reference=None, keep_build=False, recorder=None, lockfile_node_id=None, is_build_require=False, add_txt_generator=True, require_overrides=None, conanfile_path=None, test=None, source_folder=None, output_folder=None): """ Fetch and build all dependencies for the given reference @param app: The ConanApp instance with all collaborators @param ref_or_path: ConanFileReference or path to user space conanfile @param install_folder: where the output files will be saved @param build_modes: List of build_modes specified @param update: Check for updated in the upstream remotes (and update) @param manifest_folder: Folder to install the manifests @param manifest_verify: Verify dependencies manifests against stored ones @param manifest_interactive: Install deps manifests in folder for later verify, asking user for confirmation @param generators: List of generators from command line. @param no_imports: Install specified packages but avoid running imports @param add_txt_generator: Add the txt to the list of generators """ out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache remote_manager, hook_manager = app.remote_manager, app.hook_manager profile_host, profile_build = graph_info.profile_host, graph_info.profile_build if profile_build: out.info("Configuration (profile_host):") out.writeln(profile_host.dumps()) out.info("Configuration (profile_build):") out.writeln(profile_build.dumps()) else: out.info("Configuration:") out.writeln(profile_host.dumps()) deps_graph = graph_manager.load_graph(ref_or_path, create_reference, graph_info, build_modes, False, update, remotes, recorder, lockfile_node_id=lockfile_node_id, is_build_require=is_build_require, require_overrides=require_overrides) graph_lock = graph_info.graph_lock # After the graph is loaded it is defined root_node = deps_graph.root conanfile = root_node.conanfile if root_node.recipe == RECIPE_VIRTUAL: out.highlight("Installing package: %s" % str(ref_or_path)) else: conanfile.output.highlight("Installing package") print_graph(deps_graph, out) try: if cross_building(conanfile): settings = get_cross_building_settings(conanfile) message = "Cross-build from '%s:%s' to '%s:%s'" % settings out.writeln(message, Color.BRIGHT_MAGENTA) except ConanException: # Setting os doesn't exist pass installer = BinaryInstaller(app, recorder=recorder) # TODO: Extract this from the GraphManager, reuse same object, check args earlier build_modes = BuildMode(build_modes, out) installer.install(deps_graph, remotes, build_modes, update, profile_host, profile_build, graph_lock, keep_build=keep_build) graph_lock.complete_matching_prevs() if manifest_folder: manifest_manager = ManifestManager(manifest_folder, user_io=user_io, cache=cache) for node in deps_graph.nodes: if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL): continue retrieve_exports_sources(remote_manager, cache, node.conanfile, node.ref, remotes) manifest_manager.check_graph(deps_graph, verify=manifest_verify, interactive=manifest_interactive) manifest_manager.print_log() if hasattr(conanfile, "layout") and not test: conanfile.folders.set_base_source(source_folder or conanfile_path) conanfile.folders.set_base_install(output_folder or conanfile_path) conanfile.folders.set_base_imports(output_folder or conanfile_path) conanfile.folders.set_base_generators(output_folder or conanfile_path) else: conanfile.folders.set_base_install(install_folder) conanfile.folders.set_base_imports(install_folder) conanfile.folders.set_base_generators(base_folder) output = conanfile.output if root_node.recipe != RECIPE_VIRTUAL else out if install_folder: # Write generators tmp = list( conanfile.generators) # Add the command line specified generators generators = set(generators) if generators else set() tmp.extend([g for g in generators if g not in tmp]) if add_txt_generator: tmp.append("txt") conanfile.generators = tmp app.generator_manager.write_generators(conanfile, install_folder, conanfile.generators_folder, output) write_toolchain(conanfile, conanfile.generators_folder, output) if not isinstance(ref_or_path, ConanFileReference): # Write conaninfo content = normalize(conanfile.info.dumps()) save(os.path.join(install_folder, CONANINFO), content) output.info("Generated %s" % CONANINFO) graph_info.save(install_folder) output.info("Generated graphinfo") graph_lock_file = GraphLockFile(profile_host, profile_build, graph_lock) graph_lock_file.save(os.path.join(install_folder, "conan.lock")) if not no_imports: run_imports(conanfile) if type(conanfile ).system_requirements != ConanFile.system_requirements: call_system_requirements(conanfile, conanfile.output) if not create_reference and isinstance(ref_or_path, ConanFileReference): # The conanfile loaded is a virtual one. The one w deploy is the first level one neighbours = deps_graph.root.neighbors() deploy_conanfile = neighbours[0].conanfile if hasattr(deploy_conanfile, "deploy") and callable( deploy_conanfile.deploy): run_deploy(deploy_conanfile, install_folder)
def _handle_node_editable(self, node, profile_host, profile_build, graph_lock): # Get source of information conanfile = node.conanfile ref = node.ref package_layout = self._cache.package_layout(ref) base_path = package_layout.base_folder() if hasattr(conanfile, "layout"): conanfile.folders.set_base_folders(base_path, package_layout.output_folder) else: conanfile.folders.set_base_package(base_path) conanfile.folders.set_base_source(None) conanfile.folders.set_base_build(None) conanfile.folders.set_base_install(None) self._call_package_info(conanfile, package_folder=base_path, ref=ref, is_editable=True) # New editables mechanism based on Folders if hasattr(conanfile, "layout"): output = conanfile.output output.info("Rewriting files of editable package " "'{}' at '{}'".format(conanfile.name, conanfile.generators_folder)) self._generator_manager.write_generators( conanfile, conanfile.install_folder, conanfile.generators_folder, output) write_toolchain(conanfile, conanfile.generators_folder, output) output.info("Generated toolchain") graph_info_node = GraphInfo(profile_host, root_ref=node.ref) graph_info_node.options = node.conanfile.options.values graph_info_node.graph_lock = graph_lock graph_info_node.save(base_path) output.info("Generated conan.lock") copied_files = run_imports(conanfile) report_copied_files(copied_files, output) return node.conanfile.cpp_info.filter_empty = False # OLD EDITABLE LAYOUTS: # Try with package-provided file editable_cpp_info = package_layout.editable_cpp_info() if editable_cpp_info: editable_cpp_info.apply_to(ref, conanfile.cpp_info, settings=conanfile.settings, options=conanfile.options) build_folder = editable_cpp_info.folder( ref, EditableLayout.BUILD_FOLDER, settings=conanfile.settings, options=conanfile.options) if build_folder is not None: build_folder = os.path.join(base_path, build_folder) output = conanfile.output self._generator_manager.write_generators( conanfile, build_folder, build_folder, output) write_toolchain(conanfile, build_folder, output) save(os.path.join(build_folder, CONANINFO), conanfile.info.dumps()) output.info("Generated %s" % CONANINFO) graph_info_node = GraphInfo(profile_host, root_ref=node.ref) graph_info_node.options = node.conanfile.options.values graph_info_node.graph_lock = graph_lock graph_info_node.save(build_folder) output.info("Generated graphinfo") graph_lock_file = GraphLockFile(profile_host, profile_build, graph_lock) graph_lock_file.save(os.path.join(build_folder, "conan.lock")) save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conanfile).content) output.info("Generated %s" % BUILD_INFO) # Build step might need DLLs, binaries as protoc to generate source files # So execute imports() before build, storing the list of copied_files conanfile.folders.set_base_imports(build_folder) copied_files = run_imports(conanfile) report_copied_files(copied_files, output)
def test_basic(): client = TestClient() # TODO: This is hardcoded client.run("config set general.revisions_enabled=1") client.save({ "pkga/conanfile.py": GenConanfile().with_settings("os"), "pkgb/conanfile.py": GenConanfile().with_requires("pkga/0.1"), "app1/conanfile.py": GenConanfile().with_settings("os").with_requires("pkgb/0.1"), "app2/conanfile.py": GenConanfile().with_settings("os").with_requires("pkgb/0.2") }) client.run("export pkga pkga/0.1@") client.run("export pkgb pkgb/0.1@") client.run("export pkgb pkgb/0.2@") client.run("export app1 app1/0.1@") client.run("export app2 app2/0.1@") client.run( "lock create --ref=app1/0.1 --base --lockfile-out=app1_base.lock") client.run( "lock create --ref=app2/0.1 --base --lockfile-out=app2_base.lock") client.run( "lock create --ref=app1/0.1 -s os=Windows --lockfile=app1_base.lock " "--lockfile-out=app1_windows.lock") assert "app1/0.1:3bcd6800847f779e0883ee91b411aad9ddd8e83c - Missing" in client.out assert "pkga/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Missing" in client.out assert "pkgb/0.1:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run( "lock create --ref=app1/0.1 -s os=Linux --lockfile=app1_base.lock " "--lockfile-out=app1_linux.lock") assert "app1/0.1:60fbb0a22359b4888f7ecad69bcdfcd6e70e2784 - Missing" in client.out assert "pkga/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Missing" in client.out assert "pkgb/0.1:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run( "lock create --ref=app2/0.1 -s os=Windows --lockfile=app2_base.lock " "--lockfile-out=app2_windows.lock") assert "app2/0.1:0f886d82040d47739aa363db84eef5fe4c958c23 - Missing" in client.out assert "pkga/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Missing" in client.out assert "pkgb/0.2:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run( "lock create --ref=app2/0.1 -s os=Linux --lockfile=app2_base.lock " "--lockfile-out=app2_linux.lock") assert "app2/0.1:156f38906bdcdceba1b26a206240cf199619fee1 - Missing" in client.out assert "pkga/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Missing" in client.out assert "pkgb/0.2:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run("lock bundle create app1_windows.lock app1_linux.lock " "app2_windows.lock app2_linux.lock --bundle-out=lock1.bundle") client.run("lock bundle build-order lock1.bundle --json=bo.json") order = client.load("bo.json") order = json.loads(order) assert order == [["pkga/0.1@#f096d7d54098b7ad7012f9435d9c33f3"], [ "pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20", "pkgb/0.2@#cd8f22d6f264f65398d8c534046e8e20" ], [ "app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2", "app2/0.1@#3850895c1eac8223c43c71d525348019" ]] bundle = client.load("lock1.bundle") bundle = json.loads(bundle)["lock_bundle"] for level in order: for ref in level: # Now get the package_id, lockfile packages = bundle[ref]["packages"] for pkg in packages: lockfiles = pkg["lockfiles"] lockfile = next(iter(sorted(lockfiles))) client.run("install {ref} --build={ref} --lockfile={lockfile} " "--lockfile-out={lockfile}".format( ref=ref, lockfile=lockfile)) client.run("lock bundle update lock1.bundle") app1_win = GraphLockFile.load( os.path.join(client.current_folder, "app1_windows.lock"), client.cache.config.revisions_enabled) nodes = app1_win.graph_lock.nodes assert nodes["1"].modified is True assert nodes["1"].ref.full_str( ) == "app1/0.1#584778f98ba1d0eb7c80a5ae1fe12fe2" assert nodes["1"].package_id == "3bcd6800847f779e0883ee91b411aad9ddd8e83c" assert nodes["1"].prev == "c6658a5c66393cf4d210c35b5fbf34f8" assert nodes["2"].modified is True assert nodes["2"].ref.full_str( ) == "pkgb/0.1#cd8f22d6f264f65398d8c534046e8e20" assert nodes["2"].package_id == "cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23" assert nodes["2"].prev == "d61b9f421cada3b4d8e39540b0aea3d0" assert nodes["3"].modified is True assert nodes["3"].ref.full_str( ) == "pkga/0.1#f096d7d54098b7ad7012f9435d9c33f3" assert nodes["3"].package_id == "3475bd55b91ae904ac96fde0f106a136ab951a5e" assert nodes["3"].prev == "d0f0357277b3417d3984b5a9a85bbab6" app2_linux = GraphLockFile.load( os.path.join(client.current_folder, "app2_linux.lock"), client.cache.config.revisions_enabled) nodes = app2_linux.graph_lock.nodes assert nodes["1"].modified is True assert nodes["1"].ref.full_str( ) == "app2/0.1#3850895c1eac8223c43c71d525348019" assert nodes["1"].package_id == "156f38906bdcdceba1b26a206240cf199619fee1" assert nodes["1"].prev == "f1ca88c668b7f573037d09cb04be0e6f" assert nodes["2"].modified is True assert nodes["2"].ref.full_str( ) == "pkgb/0.2#cd8f22d6f264f65398d8c534046e8e20" assert nodes["2"].package_id == "cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23" assert nodes["2"].prev == "d61b9f421cada3b4d8e39540b0aea3d0" assert nodes["3"].modified is True assert nodes["3"].ref.full_str( ) == "pkga/0.1#f096d7d54098b7ad7012f9435d9c33f3" assert nodes["3"].package_id == "cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31" assert nodes["3"].prev == "9e99cfd92d0d7df79d687b01512ce844" client.run("lock bundle clean-modified lock1.bundle") bundle = client.load("lock1.bundle") assert '"modified": true' not in bundle lock1 = client.load("app1_windows.lock") assert '"modified": true' not in lock1 lock2 = client.load("app2_linux.lock") assert '"modified": true' not in lock2
def save_lock(self, lockfile): graph_lock_file = GraphLockFile(self.profile_host, self.profile_build, self.graph_lock) graph_lock_file.save(lockfile)
def load_consumer_conanfile(self, conanfile_path, info_folder, deps_info_required=False, test=False): """loads a conanfile for local flow: source, imports, package, build """ try: graph_info = GraphInfo.load(info_folder) lock_path = os.path.join(info_folder, "conan.lock") graph_lock_file = GraphLockFile.load( lock_path, self._cache.config.revisions_enabled) graph_lock = graph_lock_file.graph_lock self._output.info( "Using lockfile: '{}/conan.lock'".format(info_folder)) profile_host = graph_lock_file.profile_host profile_build = graph_lock_file.profile_build self._output.info("Using cached profile from lockfile") except IOError: # Only if file is missing graph_lock = None # This is very dirty, should be removed for Conan 2.0 (source() method only) profile_host = self._cache.default_profile profile_host.process_settings(self._cache) profile_build = None name, version, user, channel = None, None, None, None else: name, version, user, channel, _ = graph_info.root profile_host.process_settings(self._cache, preprocess=False) # This is the hack of recovering the options from the graph_info profile_host.options.update(graph_info.options) if profile_build: profile_build.process_settings(self._cache, preprocess=False) if conanfile_path.endswith(".py"): lock_python_requires = None if graph_lock and not test: # Only lock python requires if it is not test_package node_id = graph_lock.get_consumer(graph_info.root) lock_python_requires = graph_lock.python_requires(node_id) # The global.conf is necessary for download_cache definition profile_host.conf.rebase_conf_definition(self._cache.new_config) conanfile = self._loader.load_consumer( conanfile_path, profile_host=profile_host, name=name, version=version, user=user, channel=channel, lock_python_requires=lock_python_requires) if profile_build: conanfile.settings_build = profile_build.processed_settings.copy( ) conanfile.settings_target = None if test: conanfile.display_name = "%s (test package)" % str(test) conanfile.output.scope = conanfile.display_name conanfile.tested_reference_str = repr(test) run_configure_method(conanfile, down_options=None, down_ref=None, ref=None) else: conanfile = self._loader.load_conanfile_txt( conanfile_path, profile_host=profile_host) load_deps_info(info_folder, conanfile, required=deps_info_required) return conanfile