def inspect_name_and_version(folder):
json_path = os.path.join(folder, "nv.json")
run_command("conan inspect {} -a name -a version --json {}".format(
folder, json_path))
with open(json_path) as f:
c = f.read()
os.unlink(json_path)
data = json.loads(c)
installed = data["name"]
version = data["version"]
return installed, version
def _add_remotes_and_login(upload_url, central_url):
run_command('conan remote add upload_remote {}'.format(upload_url))
run_command('conan remote add central_remote {}'.format(central_url))
# If the repo is private, we need to login or it will return 404 to everything
run_command('conan user -r upload_remote -p')
run_command('conan user -r central_remote -p')
def run(self):
# Home at the current dir
with environment_append({"CONAN_USER_HOME": cur_folder()}):
print(
"\n\n\n------------------------------------------------------")
print("BUILDING '{}' AT '{}'".format(self.ref, cur_folder()))
build_folder = cur_folder()
try:
run_command('conan remote remove conan-center')
except Exception:
pass
_add_remotes_and_login(self.repo_upload.url, self.repo_read.url)
# Download the lock file to the install folder
self.repo_meta.download_node_lock(self.project_lock_remote_path,
build_folder)
run_command("conan graph clean-modified {}".format(build_folder))
run_command('conan remove "*" -f')
# Build the ref using the lockfile
print(
"\n\n****************** INSTALL OUTPUT {} ************************"
.format(self.ref))
try:
print("CONAN USER HOME: {}".format(
os.getenv("CONAN_USER_HOME")))
output = run_command_output(
"conan install {} --install-folder {} "
"--use-lock --build {}".format(self.ref, build_folder,
self.ref))
except Exception as exc:
self.repo_meta.store_install_log(self.results_remote_path,
str(exc))
self.repo_meta.store_failure(self.results_remote_path)
raise exc
print(output)
print(
"************************************************************\n\n"
)
self.repo_meta.store_install_log(self.results_remote_path, output)
# Upload the packages
run_command('conan upload {} --all -r upload_remote'.format(
self.ref))
# Upload the modified lockfile to the right location
self.repo_meta.store_node_lock(build_folder,
self.results_remote_path)
print(
"\n\n\n------------------------------------------------------")
self.repo_meta.store_success(self.results_remote_path)
def create_gh_repo(self, tree, ref, upload_recipe=True):
name, version = ref.split("@")[0].split("/")
slug = self.get_slug(name)
if self.github.repos.get(slug):
return
rand = uuid.uuid4()
reqs = tree.get(ref, [])
reqs_str = ",".join('"{}"'.format(r) for r in reqs)
reqs_line = 'requires = {}'.format(reqs_str) if reqs else ""
cf = conanfile.format(name, version, reqs_line, rand)
files = {
"conanfile.py": cf,
"myfile.txt": "Original content: {}".format(ref)
}
# Register the repo on Github
repo = self.github.create_repository(slug, files)
# Register the repo on travis
self.travis.register_env_vars(slug, travis_env)
def main_action():
"""
This simulates the yml script of a repository of a library
:return:
"""
ci_adapter = TravisCIAdapter()
# ci_caller = TravisAPICallerMultiThreadMock(self.travis)
token = os.getenv("TRAVIS_TOKEN")
ci_caller = TravisAPICaller(self.travis, "lasote/build_node",
token)
main_job = MainJob(ci_adapter, ci_caller)
with environment_append({"CONAN_USER_HOME": os.getcwd()}):
main_job.run()
self.travis.register_repo(slug, repo, main_action)
tmp = tempfile.mkdtemp()
for name, contents in files.items():
path = os.path.join(tmp, name)
with open(path, "w") as f:
f.write(contents)
if upload_recipe:
with environment_append({
"CONAN_USER_HOME":
tmp,
"CONAN_REVISIONS_ENABLED":
"1",
"CONAN_LOGIN_USERNAME":
travis_env["CONAN_LOGIN_USERNAME"],
"CONAN_PASSWORD":
travis_env["CONAN_PASSWORD"],
"CONAN_NON_INTERACTIVE":
"1"
}):
with chdir(tmp):
run_command("conan remote add develop {}".format(
self.repo_develop.url))
run_command("conan export . {}".format(ref))
run_command(
"conan upload {} -r develop -c --all".format(ref))
for req in reqs:
self.create_gh_repo(tree, req, upload_recipe=upload_recipe)
def run(self):
try:
run_command('conan remote remove conan-center')
except Exception:
pass
_add_remotes_and_login(self.repo_upload.url, self.repo_read.url)
profiles_names = self.repo_meta.get_profile_names()
projects_refs = self.repo_meta.get_projects_refs()
# TODO: We should do here the same than c3i, infos to calculate
# different package id?
# conan info <ref> -if=<path_to_lock> --use-lock --json
for project_ref in projects_refs:
for profile_name in profiles_names:
run_command('conan remove "*" -f')
with tmp_folder() as tmp_path:
# Generate and upload the lock file for the project
profile_path = self.repo_meta.download_profile(
profile_name, tmp_path)
run_command("conan graph lock {} "
"--profile {} --install-folder .".format(
project_ref, profile_path))
remote_path = self.repo_meta.project_lock_path(
self.build_unique_id, project_ref, profile_name)
self.repo_meta.store_node_lock(tmp_path, remote_path)
# Get the reference of the node being modified
name, version = self.inspect_name_and_version(
self.checkout_folder)
reference = "{}/{}@conan/stable".format(name, version)
run_command(
"conan export {} {} --install-folder {} --use-lock".
format(self.checkout_folder, reference, tmp_path))
run_command(
'conan upload {} -r upload_remote'.format(reference))
# Get the nodes corresponding to the ref being modified
# And queue all the builds
lock = json.loads(load("conan.lock"))["graph_lock"]
for node_id, data in lock["nodes"].items():
node_reference = self._pref_to_ref(data["pref"])
if node_reference == reference:
self._call_build(project_ref, profile_name,
node_id, reference)
# While there are jobs working for the project...
while not self.ci_caller.empty_queue():
print("\n\nSleeping 60 secs..."
) # Do not consume api calls limit checking
time.sleep(60)
print("Checking ended jobs...")
ended = self.ci_caller.check_ended()
for node_info in ended:
# Check status
status = self.repo_meta.get_status(node_info.lock_path)
if not status:
try:
log = self.repo_meta.get_log(node_info.lock_path)
except Exception:
log = "No log generated"
raise Exception("The job '{}:{}' failed with "
"error: {}".format(
node_info.ref,
node_info.profile_name, log))
with tmp_folder() as node_lock_path:
# Download the node lockfile
self.repo_meta.download_node_lock(
node_info.lock_path, node_lock_path)
# Update main lock with the node one
with tmp_folder() as project_lock_folder:
# Download the project lock
remote_path = self.repo_meta.project_lock_path(
self.build_unique_id, project_ref,
node_info.profile_name)
self.repo_meta.download_node_lock(
remote_path, project_lock_folder)
# Update the project lock with the node lock
run_command("conan graph update-lock {} {}".format(
project_lock_folder, node_lock_path))
# Store again the project lock
self.repo_meta.store_node_lock(
project_lock_folder, remote_path)
# Get new build order to iterate the new available nodes
run_command(
'conan graph build-order . --json bo.json')
with open("bo.json") as f:
groups = json.load(f)
if groups:
first_group = groups[0]
for new_node_id, new_pref in first_group:
new_ref = self._pref_to_ref(new_pref)
self._call_build(project_ref,
node_info.profile_name,
new_node_id, new_ref)