def load_consumer_conanfile(self,
conanfile_path,
info_folder,
deps_info_required=False,
test=False):
"""loads a conanfile for local flow: source, imports, package, build
"""
try:
graph_info = GraphInfo.load(info_folder)
graph_lock_file = GraphLockFile.load(
info_folder, self._cache.config.revisions_enabled)
graph_lock = graph_lock_file.graph_lock
self._output.info(
"Using lockfile: '{}/conan.lock'".format(info_folder))
profile_host = graph_lock_file.profile_host
self._output.info("Using cached profile from lockfile")
except IOError: # Only if file is missing
graph_lock = None
# This is very dirty, should be removed for Conan 2.0 (source() method only)
profile_host = self._cache.default_profile
profile_host.process_settings(self._cache)
name, version, user, channel = None, None, None, None
else:
name, version, user, channel, _ = graph_info.root
profile_host.process_settings(self._cache, preprocess=False)
# This is the hack of recovering the options from the graph_info
profile_host.options.update(graph_info.options)
if conanfile_path.endswith(".py"):
lock_python_requires = None
if graph_lock and not test: # Only lock python requires if it is not test_package
node_id = graph_lock.get_node(graph_info.root)
lock_python_requires = graph_lock.python_requires(node_id)
conanfile = self._loader.load_consumer(
conanfile_path,
profile_host=profile_host,
name=name,
version=version,
user=user,
channel=channel,
lock_python_requires=lock_python_requires)
if test:
conanfile.display_name = "%s (test package)" % str(test)
conanfile.output.scope = conanfile.display_name
with get_env_context_manager(conanfile, without_python=True):
with conanfile_exception_formatter(str(conanfile),
"config_options"):
conanfile.config_options()
with conanfile_exception_formatter(str(conanfile),
"configure"):
conanfile.configure()
conanfile.settings.validate() # All has to be ok!
conanfile.options.validate()
else:
conanfile = self._loader.load_conanfile_txt(
conanfile_path, profile_host)
load_deps_info(info_folder, conanfile, required=deps_info_required)
return conanfile
def update_bundle(bundle_path, revisions_enabled):
""" Update both the bundle information as well as every individual lockfile, from the
information that was modified in the individual lockfile. At the end, all lockfiles will
have the same PREV for the binary of same package_id
"""
bundle = LockBundle()
bundle.loads(load(bundle_path))
for node in bundle._nodes.values():
# Each node in bundle belongs to a "ref", and contains lockinfo for every package_id
for bundle_package_ids in node["package_id"].values():
# Each package_id contains information of multiple lockfiles
# First, compute the modified PREV from all lockfiles
prev = modified = prev_lockfile = None
for lockfile, nodes_ids in bundle_package_ids[
"lockfiles"].items():
graph_lock_conf = GraphLockFile.load(
lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock
for node_id in nodes_ids:
# Make sure the PREV from lockfiles is consistent, it cannot be different
lock_prev = graph_lock.nodes[node_id].prev
if prev is None:
prev = lock_prev
prev_lockfile = lockfile
modified = graph_lock.nodes[node_id].modified
elif lock_prev is not None and prev != lock_prev:
ref = graph_lock.nodes[node_id].ref
msg = "Lock mismatch for {} prev: {}:{} != {}:{}".format(
ref, prev_lockfile, prev, lockfile, lock_prev)
raise ConanException(msg)
bundle_package_ids["prev"] = prev
bundle_package_ids["modified"] = modified
# Then, update all prev of all config lockfiles
for lockfile, nodes_ids in bundle_package_ids[
"lockfiles"].items():
graph_lock_conf = GraphLockFile.load(
lockfile, revisions_enabled)
graph_lock = graph_lock_conf.graph_lock
for node_id in nodes_ids:
if graph_lock.nodes[node_id].prev is None:
graph_lock.nodes[node_id].prev = prev
graph_lock_conf.save(lockfile)
save(bundle_path, bundle.dumps())
def clean_modified(bundle_path, revisions_enabled):
bundle = LockBundle()
bundle.loads(load(bundle_path))
for node in bundle._nodes.values():
for pkg in node["packages"]:
pkg["modified"] = None
for lockfile, nodes_ids in pkg["lockfiles"].items():
graph_lock_conf = GraphLockFile.load(
lockfile, revisions_enabled)
graph_lock_conf.graph_lock.clean_modified()
graph_lock_conf.save(lockfile)
save(bundle_path, bundle.dumps())
def create(lockfiles, revisions_enabled, cwd):
def ref_convert(r):
# Necessary so "build-order" output is usable by install
if "@" not in r:
if "#" in r:
r = r.replace("#", "@#")
else:
r += "@"
return r
result = LockBundle()
for lockfile_name in lockfiles:
lockfile_abs = os.path.normpath(os.path.join(cwd, lockfile_name))
lockfile = GraphLockFile.load(lockfile_abs, revisions_enabled)
lock = lockfile.graph_lock
for id_, node in lock.nodes.items():
ref_str = node.ref.full_str()
ref_str = ref_convert(ref_str)
ref_node = result._nodes.setdefault(ref_str, {})
packages_node = ref_node.setdefault("packages", [])
# Find existing package_id in the list of packages
# This is the equivalent of a setdefault over a dict, but on a list
for pkg in packages_node:
if pkg["package_id"] == node.package_id:
pid_node = pkg
break
else:
pid_node = {"package_id": node.package_id}
packages_node.append(pid_node)
ids = pid_node.setdefault("lockfiles", {})
# TODO: Add check that this prev is always the same
pid_node["prev"] = node.prev
pid_node["modified"] = node.modified
ids.setdefault(lockfile_name, []).append(id_)
total_requires = node.requires + node.build_requires
for require in total_requires:
require_node = lock.nodes[require]
ref = require_node.ref.full_str()
ref = ref_convert(ref)
requires = ref_node.setdefault("requires", [])
if ref not in requires:
requires.append(ref)
return result
def test_basic():
client = TestClient()
# TODO: This is hardcoded
client.run("config set general.revisions_enabled=1")
client.save({
"pkga/conanfile.py":
GenConanfile().with_settings("os"),
"pkgb/conanfile.py":
GenConanfile().with_requires("pkga/0.1"),
"app1/conanfile.py":
GenConanfile().with_settings("os").with_requires("pkgb/0.1"),
"app2/conanfile.py":
GenConanfile().with_settings("os").with_requires("pkgb/0.2")
})
client.run("export pkga pkga/0.1@")
client.run("export pkgb pkgb/0.1@")
client.run("export pkgb pkgb/0.2@")
client.run("export app1 app1/0.1@")
client.run("export app2 app2/0.1@")
client.run(
"lock create --ref=app1/0.1 --base --lockfile-out=app1_base.lock")
client.run(
"lock create --ref=app2/0.1 --base --lockfile-out=app2_base.lock")
client.run(
"lock create --ref=app1/0.1 -s os=Windows --lockfile=app1_base.lock "
"--lockfile-out=app1_windows.lock")
assert "app1/0.1:3bcd6800847f779e0883ee91b411aad9ddd8e83c - Missing" in client.out
assert "pkga/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Missing" in client.out
assert "pkgb/0.1:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out
client.run(
"lock create --ref=app1/0.1 -s os=Linux --lockfile=app1_base.lock "
"--lockfile-out=app1_linux.lock")
assert "app1/0.1:60fbb0a22359b4888f7ecad69bcdfcd6e70e2784 - Missing" in client.out
assert "pkga/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Missing" in client.out
assert "pkgb/0.1:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out
client.run(
"lock create --ref=app2/0.1 -s os=Windows --lockfile=app2_base.lock "
"--lockfile-out=app2_windows.lock")
assert "app2/0.1:0f886d82040d47739aa363db84eef5fe4c958c23 - Missing" in client.out
assert "pkga/0.1:3475bd55b91ae904ac96fde0f106a136ab951a5e - Missing" in client.out
assert "pkgb/0.2:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out
client.run(
"lock create --ref=app2/0.1 -s os=Linux --lockfile=app2_base.lock "
"--lockfile-out=app2_linux.lock")
assert "app2/0.1:156f38906bdcdceba1b26a206240cf199619fee1 - Missing" in client.out
assert "pkga/0.1:cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31 - Missing" in client.out
assert "pkgb/0.2:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Missing" in client.out
client.run("lock bundle create app1_windows.lock app1_linux.lock "
"app2_windows.lock app2_linux.lock --bundle-out=lock1.bundle")
client.run("lock bundle build-order lock1.bundle --json=bo.json")
order = client.load("bo.json")
order = json.loads(order)
assert order == [["pkga/0.1@#f096d7d54098b7ad7012f9435d9c33f3"],
[
"pkgb/0.1@#cd8f22d6f264f65398d8c534046e8e20",
"pkgb/0.2@#cd8f22d6f264f65398d8c534046e8e20"
],
[
"app1/0.1@#584778f98ba1d0eb7c80a5ae1fe12fe2",
"app2/0.1@#3850895c1eac8223c43c71d525348019"
]]
bundle = client.load("lock1.bundle")
bundle = json.loads(bundle)["lock_bundle"]
for level in order:
for ref in level:
# Now get the package_id, lockfile
packages = bundle[ref]["packages"]
for pkg in packages:
lockfiles = pkg["lockfiles"]
lockfile = next(iter(sorted(lockfiles)))
client.run("install {ref} --build={ref} --lockfile={lockfile} "
"--lockfile-out={lockfile}".format(
ref=ref, lockfile=lockfile))
client.run("lock bundle update lock1.bundle")
app1_win = GraphLockFile.load(
os.path.join(client.current_folder, "app1_windows.lock"),
client.cache.config.revisions_enabled)
nodes = app1_win.graph_lock.nodes
assert nodes["1"].modified is True
assert nodes["1"].ref.full_str(
) == "app1/0.1#584778f98ba1d0eb7c80a5ae1fe12fe2"
assert nodes["1"].package_id == "3bcd6800847f779e0883ee91b411aad9ddd8e83c"
assert nodes["1"].prev == "c6658a5c66393cf4d210c35b5fbf34f8"
assert nodes["2"].modified is True
assert nodes["2"].ref.full_str(
) == "pkgb/0.1#cd8f22d6f264f65398d8c534046e8e20"
assert nodes["2"].package_id == "cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23"
assert nodes["2"].prev == "d61b9f421cada3b4d8e39540b0aea3d0"
assert nodes["3"].modified is True
assert nodes["3"].ref.full_str(
) == "pkga/0.1#f096d7d54098b7ad7012f9435d9c33f3"
assert nodes["3"].package_id == "3475bd55b91ae904ac96fde0f106a136ab951a5e"
assert nodes["3"].prev == "d0f0357277b3417d3984b5a9a85bbab6"
app2_linux = GraphLockFile.load(
os.path.join(client.current_folder, "app2_linux.lock"),
client.cache.config.revisions_enabled)
nodes = app2_linux.graph_lock.nodes
assert nodes["1"].modified is True
assert nodes["1"].ref.full_str(
) == "app2/0.1#3850895c1eac8223c43c71d525348019"
assert nodes["1"].package_id == "156f38906bdcdceba1b26a206240cf199619fee1"
assert nodes["1"].prev == "f1ca88c668b7f573037d09cb04be0e6f"
assert nodes["2"].modified is True
assert nodes["2"].ref.full_str(
) == "pkgb/0.2#cd8f22d6f264f65398d8c534046e8e20"
assert nodes["2"].package_id == "cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23"
assert nodes["2"].prev == "d61b9f421cada3b4d8e39540b0aea3d0"
assert nodes["3"].modified is True
assert nodes["3"].ref.full_str(
) == "pkga/0.1#f096d7d54098b7ad7012f9435d9c33f3"
assert nodes["3"].package_id == "cb054d0b3e1ca595dc66bc2339d40f1f8f04ab31"
assert nodes["3"].prev == "9e99cfd92d0d7df79d687b01512ce844"
client.run("lock bundle clean-modified lock1.bundle")
bundle = client.load("lock1.bundle")
assert '"modified": true' not in bundle
lock1 = client.load("app1_windows.lock")
assert '"modified": true' not in lock1
lock2 = client.load("app2_linux.lock")
assert '"modified": true' not in lock2
def load_consumer_conanfile(self,
conanfile_path,
info_folder,
deps_info_required=False,
test=False):
"""loads a conanfile for local flow: source, imports, package, build
"""
try:
graph_info = GraphInfo.load(info_folder)
lock_path = os.path.join(info_folder, "conan.lock")
graph_lock_file = GraphLockFile.load(
lock_path, self._cache.config.revisions_enabled)
graph_lock = graph_lock_file.graph_lock
self._output.info(
"Using lockfile: '{}/conan.lock'".format(info_folder))
profile_host = graph_lock_file.profile_host
profile_build = graph_lock_file.profile_build
self._output.info("Using cached profile from lockfile")
except IOError: # Only if file is missing
graph_lock = None
# This is very dirty, should be removed for Conan 2.0 (source() method only)
profile_host = self._cache.default_profile
profile_host.process_settings(self._cache)
profile_build = None
name, version, user, channel = None, None, None, None
else:
name, version, user, channel, _ = graph_info.root
profile_host.process_settings(self._cache, preprocess=False)
# This is the hack of recovering the options from the graph_info
profile_host.options.update(graph_info.options)
if profile_build:
profile_build.process_settings(self._cache, preprocess=False)
if conanfile_path.endswith(".py"):
lock_python_requires = None
if graph_lock and not test: # Only lock python requires if it is not test_package
node_id = graph_lock.get_consumer(graph_info.root)
lock_python_requires = graph_lock.python_requires(node_id)
# The global.conf is necessary for download_cache definition
profile_host.conf.rebase_conf_definition(self._cache.new_config)
conanfile = self._loader.load_consumer(
conanfile_path,
profile_host=profile_host,
name=name,
version=version,
user=user,
channel=channel,
lock_python_requires=lock_python_requires)
if profile_build:
conanfile.settings_build = profile_build.processed_settings.copy(
)
conanfile.settings_target = None
if test:
conanfile.display_name = "%s (test package)" % str(test)
conanfile.output.scope = conanfile.display_name
conanfile.tested_reference_str = repr(test)
run_configure_method(conanfile,
down_options=None,
down_ref=None,
ref=None)
else:
conanfile = self._loader.load_conanfile_txt(
conanfile_path, profile_host=profile_host)
load_deps_info(info_folder, conanfile, required=deps_info_required)
return conanfile