def inspect_name_and_version(folder): json_path = os.path.join(folder, "nv.json") run("conan inspect {} -a name -a version --json {}".format( folder, json_path)) with open(json_path) as f: c = f.read() os.unlink(json_path) data = json.loads(c) installed = data["name"] version = data["version"] return installed, version
def run_conan_commands(self, commands, package_id_mode, folder=None): output = "" tmp = folder or tempfile.mkdtemp() with environment_append({ "CONAN_USER_HOME": tmp, "CONAN_REVISIONS_ENABLED": "1", "CONAN_LOGIN_USERNAME": self.arti_user, "CONAN_PASSWORD": self.arti_password, "CONAN_NON_INTERACTIVE": "1" }): with chdir(tmp): run("conan config set general.default_package_id_mode={}". format(package_id_mode)) for c in commands: output += run(c) return tmp, output
def _get_first_group_to_build(self, project_lock_folder): run('conan graph build-order "{}" --json bo.json -b missing'.format( project_lock_folder)) with open("bo.json") as f: groups = json.load(f) ret = [] if groups: first_group = groups[0] for new_node_id, new_pref in first_group: if new_node_id in self._launched_nodes_ids: print(":::::: Skipping already launched node: {}".format( new_pref)) else: ret.append([new_node_id, new_pref]) print("Lock despues de build-order") self.print_lock(project_lock_folder) print("First group: {}".format(ret)) return ret
def create_gh_repo(self, tree, ref, upload_recipe=True): name, version = ref.split("@")[0].split("/") slug = self.get_slug(name) if self.github.repos.get(slug): return rand = uuid.uuid4() reqs = tree.get(ref, []) reqs_str = ",".join('"{}"'.format(r) for r in reqs) reqs_line = 'requires = {}'.format(reqs_str) if reqs else "" cf = conanfile.format(name, version, reqs_line, rand) files = { "conanfile.py": cf, "myfile.txt": "Original content: {}".format(ref) } # Register the repo on Github repo = self.github.create_repository(slug, files) # Register the repo on travis self.travis.register_env_vars(slug, travis_env) def main_action(): """ This simulates the yml script of a repository of a library :return: """ ci_adapter = TravisCIAdapter() # ci_caller = TravisAPICallerMultiThreadMock(self.travis) token = os.getenv("TRAVIS_TOKEN") ci_caller = TravisAPICaller(self.travis, "lasote/build_node", token) main_job = CoordinatorJob(ci_adapter, ci_caller) with environment_append({"CONAN_USER_HOME": os.getcwd()}): main_job.run() self.travis.register_repo(slug, repo, main_action) tmp = tempfile.mkdtemp() for name, contents in files.items(): path = os.path.join(tmp, name) with open(path, "w") as f: f.write(contents) if upload_recipe: with environment_append({ "CONAN_USER_HOME": tmp, "CONAN_REVISIONS_ENABLED": "1", "CONAN_LOGIN_USERNAME": travis_env["CONAN_LOGIN_USERNAME"], "CONAN_PASSWORD": travis_env["CONAN_PASSWORD"], "CONAN_NON_INTERACTIVE": "1" }): with chdir(tmp): run("conan remote add develop {}".format( self.repo_develop.url)) run("conan export . {}".format(ref)) run("conan upload {} -r develop -c --all".format(ref)) for req in reqs: self.create_gh_repo(tree, req, upload_recipe=upload_recipe)
def process_ended_nodes(self, project_ref): # print("Checking ended jobs...") ended: List[BuildCreateInfo] = self.ci_caller.check_ended() # Update the project lock files by merging the lock from all the ended jobs for build_create_info in ended: # Clear generated packages run('conan remove "*" -f') print("Processing ended job: {}-{}".format( build_create_info.node_info.ref, build_create_info.build_conf.profile_name)) # Check status status = self.repos.meta.get_status(build_create_info.build, build_create_info.build_conf, build_create_info.node_info) if not status: try: log = self.repos.meta.get_log(build_create_info.build, build_create_info.build_conf, build_create_info.node_info) except Exception: log = "No log generated" raise Exception("The job '{}:{}' failed with " "error: {}".format( build_create_info.node_info.ref, build_create_info.build_conf.profile_name, log)) # Download the node lockfile node_lock_folder = tempfile.mkdtemp() project_lock_folder = tempfile.mkdtemp() self.repos.meta.download_project_lock(project_lock_folder, build_create_info.build, build_create_info.build_conf) self.repos.meta.download_node_lock(node_lock_folder, build_create_info.build, build_create_info.build_conf, build_create_info.node_info) # Update the project lock with the node lock and upload it run("conan graph update-lock {} {}".format(project_lock_folder, node_lock_folder)) self.repos.meta.store_node_lock(project_lock_folder, build_create_info.build, build_create_info.build_conf, build_create_info.node_info) # Get new build order to iterate the new available nodes # the cascade could be replaced with a RREV default mode for example to_launch = self._get_first_group_to_build(project_lock_folder) # The build-order can modify the graph with the resolved nodes, so store it self.repos.meta.store_project_lock(project_lock_folder, self.build, build_create_info.build_conf) for new_node_id, new_pref in to_launch: new_ref = self._pref_to_ref(new_pref) print("::::::: Launching {} ({}) " "after {} ended".format( new_ref, build_create_info.build_conf.profile_name, build_create_info.node_info.ref)) build_conf = BuildConfiguration( project_ref, build_create_info.build_conf.profile_name) node_info = NodeInfo(new_node_id, new_ref) self._call_build(build_conf, node_info) shutil.rmtree(node_lock_folder) shutil.rmtree(project_lock_folder)
def _export_and_queue_modified_node(self, project_ref, profile_name): build_conf = BuildConfiguration(project_ref, profile_name) with tmp_folder() as tmp_path: # Generate and upload the lock file for the project profile_path = self.repos.meta.download_profile( profile_name, tmp_path) # To calculate the first lock only, the dev repo, we don't want to get # stuff from other PRs run('conan config set general.default_package_id_mode=package_revision_mode' ) run('conan remote remove conan-center', ignore_failure=True) run('conan remote add central_remote {}'.format( self.repos.read.url)) run('conan user -r central_remote -p') run("conan graph lock {} --profile {}".format( project_ref, profile_path)) print("LOCK DESPUES DE CONAN GRAPH LOCK") self.print_lock(tmp_path) # Get the reference of the node being modified # The lockfile is modified with the new RREV name, version = self.inspect_name_and_version(self.checkout_folder) reference = "{}/{}@conan/stable".format(name, version) run("conan export {} {} --lockfile {}".format( self.checkout_folder, reference, tmp_path)) print("LOCK DESPUES DE EXPORT") self.print_lock(tmp_path) # Now we can add the other remote, the revisions are freeze already if self.repos.read.url != self.repos.write.url: run('conan remote add upload_remote {}'.format( self.repos.write.url)) run('conan user -r upload_remote -p') run('conan upload {} -r upload_remote'.format(reference)) else: # In a dev build, not a PR run('conan upload {} -r central_remote'.format(reference)) data = load(os.path.join(tmp_path, "conan.lock")) self.logger.add_graph(self.build, build_conf, json.loads(data)) # Get the nodes corresponding to the ref being modified # And queue all of them if they have been modified (no modified => FF) to_launch = self._get_first_group_to_build(tmp_path) self.repos.meta.store_project_lock(tmp_path, self.build, build_conf) for new_node_id, new_pref in to_launch: new_ref = self._pref_to_ref(new_pref) print("::::::: Launching {} ({}) at the start" " because it is missing".format(new_ref, profile_name, new_ref)) node_info = NodeInfo(new_node_id, new_ref) self._call_build(build_conf, node_info) # Clear generated packages run('conan remove "*" -f')