def get_built_node_id(lock_folder)-> NodeInfo:
data = load(os.path.join(lock_folder, "conan.lock"))
data = json.loads(data)
for node_id, doc in data["graph_lock"]["nodes"].items():
if "modified" in doc and doc["modified"] == "Build":
return NodeInfo(node_id, doc["pref"])
return None
def process_lockfile(self, lockfile_path):
contents = load(lockfile_path)
bi_modules = {}
data = json.loads(contents)
# First iteration, create the modules
for node_id, node in data["graph_lock"]["nodes"].items():
if node.get("modified"):
pref = node["pref"]
# This node has been created
module = {"id": get_module_id(pref),
"artifacts": self._get_artifacts(pref),
"dependencies": []}
bi_modules[module["id"]] = module
# Second iteration, build the dependencies
for node_id, node in data["graph_lock"]["nodes"].items():
if not node.get("modified"):
pref = node["pref"]
for require_ref in node.get("requires", []):
module_id = get_module_id(require_ref)
if module_id in bi_modules:
bi_modules[module_id]["dependencies"] = self._get_artifacts(pref)
self._merge_modules(bi_modules)
def _export_and_queue_modified_node(self, project_ref, profile_name):
build_conf = BuildConfiguration(project_ref, profile_name)
with tmp_folder() as tmp_path:
# Generate and upload the lock file for the project
profile_path = self.repos.meta.download_profile(
profile_name, tmp_path)
# To calculate the first lock only, the dev repo, we don't want to get
# stuff from other PRs
run('conan config set general.default_package_id_mode=package_revision_mode'
)
run('conan remote remove conan-center', ignore_failure=True)
run('conan remote add central_remote {}'.format(
self.repos.read.url))
run('conan user -r central_remote -p')
run("conan graph lock {} --profile {}".format(
project_ref, profile_path))
print("LOCK DESPUES DE CONAN GRAPH LOCK")
self.print_lock(tmp_path)
# Get the reference of the node being modified
# The lockfile is modified with the new RREV
name, version = self.inspect_name_and_version(self.checkout_folder)
reference = "{}/{}@conan/stable".format(name, version)
run("conan export {} {} --lockfile {}".format(
self.checkout_folder, reference, tmp_path))
print("LOCK DESPUES DE EXPORT")
self.print_lock(tmp_path)
# Now we can add the other remote, the revisions are freeze already
if self.repos.read.url != self.repos.write.url:
run('conan remote add upload_remote {}'.format(
self.repos.write.url))
run('conan user -r upload_remote -p')
run('conan upload {} -r upload_remote'.format(reference))
else:
# In a dev build, not a PR
run('conan upload {} -r central_remote'.format(reference))
data = load(os.path.join(tmp_path, "conan.lock"))
self.logger.add_graph(self.build, build_conf, json.loads(data))
# Get the nodes corresponding to the ref being modified
# And queue all of them if they have been modified (no modified => FF)
to_launch = self._get_first_group_to_build(tmp_path)
self.repos.meta.store_project_lock(tmp_path, self.build,
build_conf)
for new_node_id, new_pref in to_launch:
new_ref = self._pref_to_ref(new_pref)
print("::::::: Launching {} ({}) at the start"
" because it is missing".format(new_ref, profile_name,
new_ref))
node_info = NodeInfo(new_node_id, new_ref)
self._call_build(build_conf, node_info)
# Clear generated packages
run('conan remove "*" -f')
def get_docker_image_from_lockfile(folder):
contents = load(os.path.join(folder, "conan.lock"))
version_regex = re.compile(r'.*compiler\.version=(\d+\.*\d*)\\n.*', re.DOTALL)
ret = version_regex.match(contents)
if not ret:
return None
version = ret.group(1)
if "compiler=gcc\\n" in contents:
return "conanio/gcc{}".format(version)
if "compiler=clang\\n" in contents:
return "conanio/clang{}".format(version)
return None
def read_file(self, filename):
return load(os.path.join(self.folder, filename))
def run(self):
try:
run_command('conan remote remove conan-center')
except Exception:
pass
_add_remotes_and_login(self.repo_upload.url, self.repo_read.url)
profiles_names = self.repo_meta.get_profile_names()
projects_refs = self.repo_meta.get_projects_refs()
# TODO: We should do here the same than c3i, infos to calculate
# different package id?
# conan info <ref> -if=<path_to_lock> --use-lock --json
for project_ref in projects_refs:
for profile_name in profiles_names:
run_command('conan remove "*" -f')
with tmp_folder() as tmp_path:
# Generate and upload the lock file for the project
profile_path = self.repo_meta.download_profile(
profile_name, tmp_path)
run_command("conan graph lock {} "
"--profile {} --install-folder .".format(
project_ref, profile_path))
remote_path = self.repo_meta.project_lock_path(
self.build_unique_id, project_ref, profile_name)
self.repo_meta.store_node_lock(tmp_path, remote_path)
# Get the reference of the node being modified
name, version = self.inspect_name_and_version(
self.checkout_folder)
reference = "{}/{}@conan/stable".format(name, version)
run_command(
"conan export {} {} --install-folder {} --use-lock".
format(self.checkout_folder, reference, tmp_path))
run_command(
'conan upload {} -r upload_remote'.format(reference))
# Get the nodes corresponding to the ref being modified
# And queue all the builds
lock = json.loads(load("conan.lock"))["graph_lock"]
for node_id, data in lock["nodes"].items():
node_reference = self._pref_to_ref(data["pref"])
if node_reference == reference:
self._call_build(project_ref, profile_name,
node_id, reference)
# While there are jobs working for the project...
while not self.ci_caller.empty_queue():
print("\n\nSleeping 60 secs..."
) # Do not consume api calls limit checking
time.sleep(60)
print("Checking ended jobs...")
ended = self.ci_caller.check_ended()
for node_info in ended:
# Check status
status = self.repo_meta.get_status(node_info.lock_path)
if not status:
try:
log = self.repo_meta.get_log(node_info.lock_path)
except Exception:
log = "No log generated"
raise Exception("The job '{}:{}' failed with "
"error: {}".format(
node_info.ref,
node_info.profile_name, log))
with tmp_folder() as node_lock_path:
# Download the node lockfile
self.repo_meta.download_node_lock(
node_info.lock_path, node_lock_path)
# Update main lock with the node one
with tmp_folder() as project_lock_folder:
# Download the project lock
remote_path = self.repo_meta.project_lock_path(
self.build_unique_id, project_ref,
node_info.profile_name)
self.repo_meta.download_node_lock(
remote_path, project_lock_folder)
# Update the project lock with the node lock
run_command("conan graph update-lock {} {}".format(
project_lock_folder, node_lock_path))
# Store again the project lock
self.repo_meta.store_node_lock(
project_lock_folder, remote_path)
# Get new build order to iterate the new available nodes
run_command(
'conan graph build-order . --json bo.json')
with open("bo.json") as f:
groups = json.load(f)
if groups:
first_group = groups[0]
for new_node_id, new_pref in first_group:
new_ref = self._pref_to_ref(new_pref)
self._call_build(project_ref,
node_info.profile_name,
new_node_id, new_ref)
def print_lock(lock_folder):
print(load(os.path.join(lock_folder, "conan.lock")))