def inspect_name_and_version(folder):
json_path = os.path.join(folder, "nv.json")
run("conan inspect {} -a name -a version --json {}".format(
folder, json_path))
with open(json_path) as f:
c = f.read()
os.unlink(json_path)
data = json.loads(c)
installed = data["name"]
version = data["version"]
return installed, version
def run_conan_commands(self, commands, package_id_mode, folder=None):
output = ""
tmp = folder or tempfile.mkdtemp()
with environment_append({
"CONAN_USER_HOME": tmp,
"CONAN_REVISIONS_ENABLED": "1",
"CONAN_LOGIN_USERNAME": self.arti_user,
"CONAN_PASSWORD": self.arti_password,
"CONAN_NON_INTERACTIVE": "1"
}):
with chdir(tmp):
run("conan config set general.default_package_id_mode={}".
format(package_id_mode))
for c in commands:
output += run(c)
return tmp, output
def _get_first_group_to_build(self, project_lock_folder):
run('conan graph build-order "{}" --json bo.json -b missing'.format(
project_lock_folder))
with open("bo.json") as f:
groups = json.load(f)
ret = []
if groups:
first_group = groups[0]
for new_node_id, new_pref in first_group:
if new_node_id in self._launched_nodes_ids:
print(":::::: Skipping already launched node: {}".format(
new_pref))
else:
ret.append([new_node_id, new_pref])
print("Lock despues de build-order")
self.print_lock(project_lock_folder)
print("First group: {}".format(ret))
return ret
def create_gh_repo(self, tree, ref, upload_recipe=True):
name, version = ref.split("@")[0].split("/")
slug = self.get_slug(name)
if self.github.repos.get(slug):
return
rand = uuid.uuid4()
reqs = tree.get(ref, [])
reqs_str = ",".join('"{}"'.format(r) for r in reqs)
reqs_line = 'requires = {}'.format(reqs_str) if reqs else ""
cf = conanfile.format(name, version, reqs_line, rand)
files = {
"conanfile.py": cf,
"myfile.txt": "Original content: {}".format(ref)
}
# Register the repo on Github
repo = self.github.create_repository(slug, files)
# Register the repo on travis
self.travis.register_env_vars(slug, travis_env)
def main_action():
"""
This simulates the yml script of a repository of a library
:return:
"""
ci_adapter = TravisCIAdapter()
# ci_caller = TravisAPICallerMultiThreadMock(self.travis)
token = os.getenv("TRAVIS_TOKEN")
ci_caller = TravisAPICaller(self.travis, "lasote/build_node",
token)
main_job = CoordinatorJob(ci_adapter, ci_caller)
with environment_append({"CONAN_USER_HOME": os.getcwd()}):
main_job.run()
self.travis.register_repo(slug, repo, main_action)
tmp = tempfile.mkdtemp()
for name, contents in files.items():
path = os.path.join(tmp, name)
with open(path, "w") as f:
f.write(contents)
if upload_recipe:
with environment_append({
"CONAN_USER_HOME":
tmp,
"CONAN_REVISIONS_ENABLED":
"1",
"CONAN_LOGIN_USERNAME":
travis_env["CONAN_LOGIN_USERNAME"],
"CONAN_PASSWORD":
travis_env["CONAN_PASSWORD"],
"CONAN_NON_INTERACTIVE":
"1"
}):
with chdir(tmp):
run("conan remote add develop {}".format(
self.repo_develop.url))
run("conan export . {}".format(ref))
run("conan upload {} -r develop -c --all".format(ref))
for req in reqs:
self.create_gh_repo(tree, req, upload_recipe=upload_recipe)
def process_ended_nodes(self, project_ref):
# print("Checking ended jobs...")
ended: List[BuildCreateInfo] = self.ci_caller.check_ended()
# Update the project lock files by merging the lock from all the ended jobs
for build_create_info in ended:
# Clear generated packages
run('conan remove "*" -f')
print("Processing ended job: {}-{}".format(
build_create_info.node_info.ref,
build_create_info.build_conf.profile_name))
# Check status
status = self.repos.meta.get_status(build_create_info.build,
build_create_info.build_conf,
build_create_info.node_info)
if not status:
try:
log = self.repos.meta.get_log(build_create_info.build,
build_create_info.build_conf,
build_create_info.node_info)
except Exception:
log = "No log generated"
raise Exception("The job '{}:{}' failed with "
"error: {}".format(
build_create_info.node_info.ref,
build_create_info.build_conf.profile_name,
log))
# Download the node lockfile
node_lock_folder = tempfile.mkdtemp()
project_lock_folder = tempfile.mkdtemp()
self.repos.meta.download_project_lock(project_lock_folder,
build_create_info.build,
build_create_info.build_conf)
self.repos.meta.download_node_lock(node_lock_folder,
build_create_info.build,
build_create_info.build_conf,
build_create_info.node_info)
# Update the project lock with the node lock and upload it
run("conan graph update-lock {} {}".format(project_lock_folder,
node_lock_folder))
self.repos.meta.store_node_lock(project_lock_folder,
build_create_info.build,
build_create_info.build_conf,
build_create_info.node_info)
# Get new build order to iterate the new available nodes
# the cascade could be replaced with a RREV default mode for example
to_launch = self._get_first_group_to_build(project_lock_folder)
# The build-order can modify the graph with the resolved nodes, so store it
self.repos.meta.store_project_lock(project_lock_folder, self.build,
build_create_info.build_conf)
for new_node_id, new_pref in to_launch:
new_ref = self._pref_to_ref(new_pref)
print("::::::: Launching {} ({}) "
"after {} ended".format(
new_ref, build_create_info.build_conf.profile_name,
build_create_info.node_info.ref))
build_conf = BuildConfiguration(
project_ref, build_create_info.build_conf.profile_name)
node_info = NodeInfo(new_node_id, new_ref)
self._call_build(build_conf, node_info)
shutil.rmtree(node_lock_folder)
shutil.rmtree(project_lock_folder)
def _export_and_queue_modified_node(self, project_ref, profile_name):
build_conf = BuildConfiguration(project_ref, profile_name)
with tmp_folder() as tmp_path:
# Generate and upload the lock file for the project
profile_path = self.repos.meta.download_profile(
profile_name, tmp_path)
# To calculate the first lock only, the dev repo, we don't want to get
# stuff from other PRs
run('conan config set general.default_package_id_mode=package_revision_mode'
)
run('conan remote remove conan-center', ignore_failure=True)
run('conan remote add central_remote {}'.format(
self.repos.read.url))
run('conan user -r central_remote -p')
run("conan graph lock {} --profile {}".format(
project_ref, profile_path))
print("LOCK DESPUES DE CONAN GRAPH LOCK")
self.print_lock(tmp_path)
# Get the reference of the node being modified
# The lockfile is modified with the new RREV
name, version = self.inspect_name_and_version(self.checkout_folder)
reference = "{}/{}@conan/stable".format(name, version)
run("conan export {} {} --lockfile {}".format(
self.checkout_folder, reference, tmp_path))
print("LOCK DESPUES DE EXPORT")
self.print_lock(tmp_path)
# Now we can add the other remote, the revisions are freeze already
if self.repos.read.url != self.repos.write.url:
run('conan remote add upload_remote {}'.format(
self.repos.write.url))
run('conan user -r upload_remote -p')
run('conan upload {} -r upload_remote'.format(reference))
else:
# In a dev build, not a PR
run('conan upload {} -r central_remote'.format(reference))
data = load(os.path.join(tmp_path, "conan.lock"))
self.logger.add_graph(self.build, build_conf, json.loads(data))
# Get the nodes corresponding to the ref being modified
# And queue all of them if they have been modified (no modified => FF)
to_launch = self._get_first_group_to_build(tmp_path)
self.repos.meta.store_project_lock(tmp_path, self.build,
build_conf)
for new_node_id, new_pref in to_launch:
new_ref = self._pref_to_ref(new_pref)
print("::::::: Launching {} ({}) at the start"
" because it is missing".format(new_ref, profile_name,
new_ref))
node_info = NodeInfo(new_node_id, new_ref)
self._call_build(build_conf, node_info)
# Clear generated packages
run('conan remove "*" -f')