def load_consumer_conanfile(self, conanfile_path, info_folder, deps_info_required=False, test=False): """loads a conanfile for local flow: source, imports, package, build """ try: graph_info = GraphInfo.load(info_folder) graph_lock_file = GraphLockFile.load( info_folder, self._cache.config.revisions_enabled) graph_lock = graph_lock_file.graph_lock self._output.info( "Using lockfile: '{}/conan.lock'".format(info_folder)) profile_host = graph_lock_file.profile_host self._output.info("Using cached profile from lockfile") except IOError: # Only if file is missing graph_lock = None # This is very dirty, should be removed for Conan 2.0 (source() method only) profile_host = self._cache.default_profile profile_host.process_settings(self._cache) name, version, user, channel = None, None, None, None else: name, version, user, channel, _ = graph_info.root profile_host.process_settings(self._cache, preprocess=False) # This is the hack of recovering the options from the graph_info profile_host.options.update(graph_info.options) if conanfile_path.endswith(".py"): lock_python_requires = None if graph_lock and not test: # Only lock python requires if it is not test_package node_id = graph_lock.get_node(graph_info.root) lock_python_requires = graph_lock.python_requires(node_id) conanfile = self._loader.load_consumer( conanfile_path, profile_host=profile_host, name=name, version=version, user=user, channel=channel, lock_python_requires=lock_python_requires) if test: conanfile.display_name = "%s (test package)" % str(test) conanfile.output.scope = conanfile.display_name with get_env_context_manager(conanfile, without_python=True): with conanfile_exception_formatter(str(conanfile), "config_options"): conanfile.config_options() with conanfile_exception_formatter(str(conanfile), "configure"): conanfile.configure() conanfile.settings.validate() # All has to be ok! conanfile.options.validate() else: conanfile = self._loader.load_conanfile_txt( conanfile_path, profile_host) load_deps_info(info_folder, conanfile, required=deps_info_required) return conanfile
def update_bundle(bundle_path, revisions_enabled): """ Update both the bundle information as well as every individual lockfile, from the information that was modified in the individual lockfile. At the end, all lockfiles will have the same PREV for the binary of same package_id """ bundle = LockBundle() bundle.loads(load(bundle_path)) for node in bundle._nodes.values(): # Each node in bundle belongs to a "ref", and contains lockinfo for every package_id for bundle_package_ids in node["package_id"].values(): # Each package_id contains information of multiple lockfiles # First, compute the modified PREV from all lockfiles prev = modified = prev_lockfile = None for lockfile, nodes_ids in bundle_package_ids[ "lockfiles"].items(): graph_lock_conf = GraphLockFile.load( lockfile, revisions_enabled) graph_lock = graph_lock_conf.graph_lock for node_id in nodes_ids: # Make sure the PREV from lockfiles is consistent, it cannot be different lock_prev = graph_lock.nodes[node_id].prev if prev is None: prev = lock_prev prev_lockfile = lockfile modified = graph_lock.nodes[node_id].modified elif lock_prev is not None and prev != lock_prev: ref = graph_lock.nodes[node_id].ref msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format( ref, prev_lockfile, prev, lockfile, lock_prev) raise ConanException(msg) bundle_package_ids["prev"] = prev bundle_package_ids["modified"] = modified # Then, update all prev of all config lockfiles for lockfile, nodes_ids in bundle_package_ids[ "lockfiles"].items(): graph_lock_conf = GraphLockFile.load( lockfile, revisions_enabled) graph_lock = graph_lock_conf.graph_lock for node_id in nodes_ids: if graph_lock.nodes[node_id].prev is None: graph_lock.nodes[node_id].prev = prev graph_lock_conf.save(lockfile) save(bundle_path, bundle.dumps())
def clean_modified(bundle_path, revisions_enabled): bundle = LockBundle() bundle.loads(load(bundle_path)) for node in bundle._nodes.values(): for pkg in node["packages"]: pkg["modified"] = None for lockfile, nodes_ids in pkg["lockfiles"].items(): graph_lock_conf = GraphLockFile.load( lockfile, revisions_enabled) graph_lock_conf.graph_lock.clean_modified() graph_lock_conf.save(lockfile) save(bundle_path, bundle.dumps())
def create(lockfiles, revisions_enabled, cwd): def ref_convert(r): # Necessary so "build-order" output is usable by install if "@" not in r: if "#" in r: r = r.replace("#", "@#") else: r += "@" return r result = LockBundle() for lockfile_name in lockfiles: lockfile_abs = os.path.normpath(os.path.join(cwd, lockfile_name)) lockfile = GraphLockFile.load(lockfile_abs, revisions_enabled) lock = lockfile.graph_lock for id_, node in lock.nodes.items(): ref_str = node.ref.full_str() ref_str = ref_convert(ref_str) ref_node = result._nodes.setdefault(ref_str, {}) packages_node = ref_node.setdefault("packages", []) # Find existing package_id in the list of packages # This is the equivalent of a setdefault over a dict, but on a list for pkg in packages_node: if pkg["package_id"] == node.package_id: pid_node = pkg break else: pid_node = {"package_id": node.package_id} packages_node.append(pid_node) ids = pid_node.setdefault("lockfiles", {}) # TODO: Add check that this prev is always the same pid_node["prev"] = node.prev pid_node["modified"] = node.modified ids.setdefault(lockfile_name, []).append(id_) total_requires = node.requires + node.build_requires for require in total_requires: require_node = lock.nodes[require] ref = require_node.ref.full_str() ref = ref_convert(ref) requires = ref_node.setdefault("requires", []) if ref not in requires: requires.append(ref) return result
def test_basic(): client = TestClient() # TODO: This is hardcoded client.run("config set general.revisions_enabled=1") client.save({ "pkga/conanfile.py": GenConanfile().with_settings("os"), "pkgb/conanfile.py": GenConanfile().with_requires("pkga/0.1"), "app1/conanfile.py": GenConanfile().with_settings("os").with_requires("pkgb/0.1"), "app2/conanfile.py": GenConanfile().with_settings("os").with_requires("pkgb/0.2") }) client.run("export pkga pkga/0.1@") client.run("export pkgb pkgb/0.1@") client.run("export pkgb pkgb/0.2@") client.run("export app1 app1/0.1@") client.run("export app2 app2/0.1@") client.run( "lock create --ref=app1/0.1 --base --lockfile-out=app1_base.lock") client.run( "lock create --ref=app2/0.1 --base --lockfile-out=app2_base.lock") client.run( "lock create --ref=app1/0.1 -s os=Windows --lockfile=app1_base.lock " "--lockfile-out=app1_windows.lock") assert "app1/0.1:3bcd6800847f779e0883ee91b411aad9ddd8e83c - Missing" in client.out assert "pkga/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Missing" in client.out assert "pkgb/0.1:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run( "lock create --ref=app1/0.1 -s os=Linux --lockfile=app1_base.lock " "--lockfile-out=app1_linux.lock") assert "app1/0.1:60fbb0a22359b4888f7ecad69bcdfcd6e70e2784 - Missing" in client.out assert "pkga/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Missing" in client.out assert "pkgb/0.1:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run( "lock create --ref=app2/0.1 -s os=Windows --lockfile=app2_base.lock " "--lockfile-out=app2_windows.lock") assert "app2/0.1:0f886d82040d47739aa363db84eef5fe4c958c23 - Missing" in client.out assert "pkga/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Missing" in client.out assert "pkgb/0.2:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run( "lock create --ref=app2/0.1 -s os=Linux --lockfile=app2_base.lock " "--lockfile-out=app2_linux.lock") assert "app2/0.1:156f38906bdcdceba1b26a206240cf199619fee1 - Missing" in client.out assert "pkga/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Missing" in client.out assert "pkgb/0.2:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out client.run("lock bundle create app1_windows.lock app1_linux.lock " "app2_windows.lock app2_linux.lock --bundle-out=lock1.bundle") client.run("lock bundle build-order lock1.bundle --json=bo.json") order = client.load("bo.json") order = json.loads(order) assert order == [["pkga/0.1@#f096d7d54098b7ad7012f9435d9c33f3"], [ "pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20", "pkgb/0.2@#cd8f22d6f264f65398d8c534046e8e20" ], [ "app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2", "app2/0.1@#3850895c1eac8223c43c71d525348019" ]] bundle = client.load("lock1.bundle") bundle = json.loads(bundle)["lock_bundle"] for level in order: for ref in level: # Now get the package_id, lockfile packages = bundle[ref]["packages"] for pkg in packages: lockfiles = pkg["lockfiles"] lockfile = next(iter(sorted(lockfiles))) client.run("install {ref} --build={ref} --lockfile={lockfile} " "--lockfile-out={lockfile}".format( ref=ref, lockfile=lockfile)) client.run("lock bundle update lock1.bundle") app1_win = GraphLockFile.load( os.path.join(client.current_folder, "app1_windows.lock"), client.cache.config.revisions_enabled) nodes = app1_win.graph_lock.nodes assert nodes["1"].modified is True assert nodes["1"].ref.full_str( ) == "app1/0.1#584778f98ba1d0eb7c80a5ae1fe12fe2" assert nodes["1"].package_id == "3bcd6800847f779e0883ee91b411aad9ddd8e83c" assert nodes["1"].prev == "c6658a5c66393cf4d210c35b5fbf34f8" assert nodes["2"].modified is True assert nodes["2"].ref.full_str( ) == "pkgb/0.1#cd8f22d6f264f65398d8c534046e8e20" assert nodes["2"].package_id == "cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23" assert nodes["2"].prev == "d61b9f421cada3b4d8e39540b0aea3d0" assert nodes["3"].modified is True assert nodes["3"].ref.full_str( ) == "pkga/0.1#f096d7d54098b7ad7012f9435d9c33f3" assert nodes["3"].package_id == "3475bd55b91ae904ac96fde0f106a136ab951a5e" assert nodes["3"].prev == "d0f0357277b3417d3984b5a9a85bbab6" app2_linux = GraphLockFile.load( os.path.join(client.current_folder, "app2_linux.lock"), client.cache.config.revisions_enabled) nodes = app2_linux.graph_lock.nodes assert nodes["1"].modified is True assert nodes["1"].ref.full_str( ) == "app2/0.1#3850895c1eac8223c43c71d525348019" assert nodes["1"].package_id == "156f38906bdcdceba1b26a206240cf199619fee1" assert nodes["1"].prev == "f1ca88c668b7f573037d09cb04be0e6f" assert nodes["2"].modified is True assert nodes["2"].ref.full_str( ) == "pkgb/0.2#cd8f22d6f264f65398d8c534046e8e20" assert nodes["2"].package_id == "cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23" assert nodes["2"].prev == "d61b9f421cada3b4d8e39540b0aea3d0" assert nodes["3"].modified is True assert nodes["3"].ref.full_str( ) == "pkga/0.1#f096d7d54098b7ad7012f9435d9c33f3" assert nodes["3"].package_id == "cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31" assert nodes["3"].prev == "9e99cfd92d0d7df79d687b01512ce844" client.run("lock bundle clean-modified lock1.bundle") bundle = client.load("lock1.bundle") assert '"modified": true' not in bundle lock1 = client.load("app1_windows.lock") assert '"modified": true' not in lock1 lock2 = client.load("app2_linux.lock") assert '"modified": true' not in lock2
def load_consumer_conanfile(self, conanfile_path, info_folder, deps_info_required=False, test=False): """loads a conanfile for local flow: source, imports, package, build """ try: graph_info = GraphInfo.load(info_folder) lock_path = os.path.join(info_folder, "conan.lock") graph_lock_file = GraphLockFile.load( lock_path, self._cache.config.revisions_enabled) graph_lock = graph_lock_file.graph_lock self._output.info( "Using lockfile: '{}/conan.lock'".format(info_folder)) profile_host = graph_lock_file.profile_host profile_build = graph_lock_file.profile_build self._output.info("Using cached profile from lockfile") except IOError: # Only if file is missing graph_lock = None # This is very dirty, should be removed for Conan 2.0 (source() method only) profile_host = self._cache.default_profile profile_host.process_settings(self._cache) profile_build = None name, version, user, channel = None, None, None, None else: name, version, user, channel, _ = graph_info.root profile_host.process_settings(self._cache, preprocess=False) # This is the hack of recovering the options from the graph_info profile_host.options.update(graph_info.options) if profile_build: profile_build.process_settings(self._cache, preprocess=False) if conanfile_path.endswith(".py"): lock_python_requires = None if graph_lock and not test: # Only lock python requires if it is not test_package node_id = graph_lock.get_consumer(graph_info.root) lock_python_requires = graph_lock.python_requires(node_id) # The global.conf is necessary for download_cache definition profile_host.conf.rebase_conf_definition(self._cache.new_config) conanfile = self._loader.load_consumer( conanfile_path, profile_host=profile_host, name=name, version=version, user=user, channel=channel, lock_python_requires=lock_python_requires) if profile_build: conanfile.settings_build = profile_build.processed_settings.copy( ) conanfile.settings_target = None if test: conanfile.display_name = "%s (test package)" % str(test) conanfile.output.scope = conanfile.display_name conanfile.tested_reference_str = repr(test) run_configure_method(conanfile, down_options=None, down_ref=None, ref=None) else: conanfile = self._loader.load_conanfile_txt( conanfile_path, profile_host=profile_host) load_deps_info(info_folder, conanfile, required=deps_info_required) return conanfile