def __init__(self, profile_text, base_profile_text, base_profile_name, reference, args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=None, sudo_pip_command=True, docker_image_skip_update=False, build_policy=None, docker_image_skip_pull=False, always_update_conan_in_docker=False, upload=False, runner=None, docker_shell=None, docker_conan_home=None): self.printer = Printer() self._args = args self._upload = upload self._reference = reference self._conan_pip_package = conan_pip_package self._build_policy = build_policy self._docker_image = docker_image self._always_update_conan_in_docker = always_update_conan_in_docker self._docker_image_skip_update = docker_image_skip_update self._docker_image_skip_pull = docker_image_skip_pull self._sudo_docker_command = sudo_docker_command or "" self._sudo_pip_command = sudo_pip_command self._profile_text = profile_text self._base_profile_text = base_profile_text self._base_profile_name = base_profile_name self._docker_shell = docker_shell self._docker_conan_home = docker_conan_home self._runner = PrintRunner(runner, self.printer)
def __init__(self, profile_abs_path, reference, conan_api, uploader, exclude_vcvars_precommand=False, build_policy=None, runner=None, cwd=None, printer=None, upload=False, test_folder=None, config_url=None): self.printer = printer or Printer() self._cwd = cwd or os.getcwd() self._uploader = uploader self._upload = upload self._conan_api = conan_api self._profile_abs_path = profile_abs_path self._reference = reference self._exclude_vcvars_precommand = exclude_vcvars_precommand self._build_policy = build_policy self._runner = PrintRunner(runner or os.system, self.printer) self._uploader.remote_manager.add_remotes_to_conan() self._test_folder = test_folder self._config_url = config_url patch_default_base_profile(conan_api, profile_abs_path) if Version(client_version) < Version("1.12.0"): cache = self._conan_api._client_cache else: cache = self._conan_api._cache self._profile = load_profile(profile_abs_path, cache)
def run(): conan_version = get_client_version() if conan_version < Version("1.18.0"): conan_api, client_cache, _ = Conan.factory() else: conan_api, _, _ = Conan.factory() conan_api.create_app() client_cache = conan_api.app.cache printer = Printer() remotes_manager = RemotesManager(conan_api, printer) remotes_manager.add_remotes_to_conan() default_username = os.getenv("CONAN_USERNAME") auth_manager = AuthManager(conan_api, printer, default_username=default_username) upload_retry = os.getenv("CPT_UPLOAD_RETRY") upload_only_recipe = os.getenv("CPT_UPLOAD_ONLY_RECIPE") upload_force = os.getenv("CPT_UPLOAD_FORCE") uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry, upload_force) build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY")) test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER")) reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE")) profile_text = unscape_env(os.getenv("CPT_PROFILE")) abs_profile_path = save_profile_to_tmp(profile_text) base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE")) config_url = unscape_env(os.getenv("CPT_CONFIG_URL")) config_args = unscape_env(os.getenv("CPT_CONFIG_ARGS")) upload_dependencies = unscape_env(os.getenv("CPT_UPLOAD_DEPENDENCIES")) update_dependencies = unscape_env(os.getenv("CPT_UPDATE_DEPENDENCIES")) conanfile = unscape_env(os.getenv("CPT_CONANFILE")) lockfile = unscape_env(os.getenv("CPT_LOCKFILE")) skip_recipe_export = unscape_env(os.getenv("CPT_SKIP_RECIPE_EXPORT")) if base_profile_text: base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME")) tools.save(os.path.join(client_cache.profiles_path, base_profile_name), base_profile_text) upload = os.getenv("CPT_UPLOAD_ENABLED") runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, build_policy=build_policy, printer=printer, upload=upload, upload_only_recipe=upload_only_recipe, test_folder=test_folder, config_url=config_url, config_args=config_args, upload_dependencies=upload_dependencies, conanfile=conanfile, skip_recipe_export=skip_recipe_export, update_dependencies=update_dependencies, lockfile=lockfile) runner.run()
def login(self): if self.variables.docker_upload_only_when_stable: printer = Printer() ci_manager = CIManager(printer) if ci_manager.get_branch() != "master" or ci_manager.is_pull_request(): logging.info("Skipped login, is not stable branch") return if not self.variables.docker_upload: logging.info("Skipped login, DOCKER_UPLOAD is not activated") return if not self.variables.docker_password: logging.warning("Skipped login, DOCKER_PASSWORD is missing!") return if not self.variables.docker_login_username: logging.warning("Skipped login, DOCKER_LOGIN_USERNAME is missing!") return logging.info("Login to Docker hub account") result = subprocess.call([ "docker", "login", "-u", self.variables.docker_login_username, "-p", self.variables.docker_password ]) if result != os.EX_OK: raise RuntimeError("Could not login username %s " "to Docker hub." % self.variables.docker_login_username) logging.info("Logged in Docker hub account with success") self.loggedin = True
def run(): # Get all from environ conan_api, client_cache, _ = Conan.factory() printer = Printer() if os.path.exists(client_cache.default_profile_path): os.remove(client_cache.default_profile_path) remotes_manager = RemotesManager(conan_api, printer) default_username = os.getenv("CONAN_USERNAME", None) auth_manager = AuthManager(conan_api, printer, default_username=default_username) uploader = Uploader(conan_api, remotes_manager, auth_manager, printer) args = os.getenv("CPT_ARGS", "") build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY")) reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE")) profile_text = unscape_env(os.getenv("CPT_PROFILE")) abs_profile_path = save_profile_to_tmp(profile_text) base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE")) if base_profile_text: base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME")) tools.save(os.path.join(client_cache.profiles_path, base_profile_name), base_profile_text) upload = os.getenv("CPT_UPLOAD_ENABLED", None) runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, args=args, build_policy=build_policy, printer=printer, upload=upload) runner.run()
def __init__(self, profile_abs_path, reference, conan_api, uploader, exclude_vcvars_precommand=False, build_policy=None, runner=None, cwd=None, printer=None, upload=False, upload_only_recipe=None, test_folder=None, config_url=None, config_args=None, upload_dependencies=None, conanfile=None, skip_recipe_export=False, update_dependencies=False, lockfile=None): self.printer = printer or Printer() self._cwd = cwd or os.getcwd() self._uploader = uploader self._upload = upload self._conan_api = conan_api self._profile_abs_path = profile_abs_path self._reference = reference self._exclude_vcvars_precommand = exclude_vcvars_precommand self._build_policy = build_policy.split(",") if \ isinstance(build_policy, str) else \ build_policy self._runner = PrintRunner(runner or os.system, self.printer) self._test_folder = test_folder self._config_url = config_url self._config_args = config_args self._upload_only_recipe = upload_only_recipe self._conanfile = conanfile self._lockfile = lockfile self._upload_dependencies = upload_dependencies.split(",") if \ isinstance(upload_dependencies, str) else \ upload_dependencies self._upload_dependencies = self._upload_dependencies or [] self.skip_recipe_export = skip_recipe_export self._update_dependencies = update_dependencies self._results = None patch_default_base_profile(conan_api, profile_abs_path) client_version = get_client_version() if client_version < Version("1.12.0"): cache = self._conan_api._client_cache elif client_version < Version("1.18.0"): cache = self._conan_api._cache else: if not conan_api.app: conan_api.create_app() cache = conan_api.app.cache self._profile = load_profile(profile_abs_path, cache)
def run(): # Get all from environ conan_api, client_cache, _ = Conan.factory() printer = Printer() remotes_manager = RemotesManager(conan_api, printer) remotes_manager.add_remotes_to_conan() default_username = os.getenv("CONAN_USERNAME") auth_manager = AuthManager(conan_api, printer, default_username=default_username) upload_retry = os.getenv("CPT_UPLOAD_RETRY") upload_only_recipe = os.getenv("CPT_UPLOAD_ONLY_RECIPE") uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry) build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY")) test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER")) reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE")) profile_text = unscape_env(os.getenv("CPT_PROFILE")) abs_profile_path = save_profile_to_tmp(profile_text) base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE")) config_url = unscape_env(os.getenv("CPT_CONFIG_URL")) upload_dependencies = unscape_env(os.getenv("CPT_UPLOAD_DEPENDENCIES")) conanfile = unscape_env(os.getenv("CPT_CONANFILE")) if base_profile_text: base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME")) tools.save(os.path.join(client_cache.profiles_path, base_profile_name), base_profile_text) upload = os.getenv("CPT_UPLOAD_ENABLED") runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, build_policy=build_policy, printer=printer, upload=upload, upload_only_recipe=upload_only_recipe, test_folder=test_folder, config_url=config_url, upload_dependencies=upload_dependencies, conanfile=conanfile) runner.run()
def __init__(self, profile_abs_path, reference, conan_api, uploader, args=None, exclude_vcvars_precommand=False, build_policy=None, runner=None, abs_folder=None, printer=None, upload=False): self.printer = printer or Printer() self._abs_folder = abs_folder or os.getcwd() self._uploader = uploader self._upload = upload self._conan_api = conan_api self._profile_abs_path = profile_abs_path self._reference = reference self._args = args self._exclude_vcvars_precommand = exclude_vcvars_precommand self._build_policy = build_policy self._runner = PrintRunner(runner or os.system, self.printer) self._uploader.remote_manager.add_remotes_to_conan() patch_default_base_profile(conan_api, profile_abs_path) self._profile = load_profile(profile_abs_path, self._conan_api._client_cache)
def __init__(self, profile_text, base_profile_text, base_profile_name, reference, conan_pip_package=None, docker_image=None, sudo_docker_command=None, sudo_pip_command=False, docker_image_skip_update=False, build_policy=None, docker_image_skip_pull=False, always_update_conan_in_docker=False, upload=False, upload_retry=None, runner=None, docker_shell="", docker_conan_home="", docker_platform_param="", lcow_user_workaround="", test_folder=None, pip_install=None, config_url=None): self.printer = Printer() self._upload = upload self._upload_retry = upload_retry self._reference = reference self._conan_pip_package = conan_pip_package self._build_policy = build_policy self._docker_image = docker_image self._always_update_conan_in_docker = always_update_conan_in_docker self._docker_image_skip_update = docker_image_skip_update self._docker_image_skip_pull = docker_image_skip_pull self._sudo_docker_command = sudo_docker_command or "" self._sudo_pip_command = sudo_pip_command self._profile_text = profile_text self._base_profile_text = base_profile_text self._base_profile_name = base_profile_name self._docker_shell = docker_shell self._docker_conan_home = docker_conan_home self._docker_platform_param = docker_platform_param self._lcow_user_workaround = lcow_user_workaround self._runner = PrintRunner(runner, self.printer) self._test_folder = test_folder self._pip_install = pip_install self._config_url = config_url
def __init__(self, profile_text, base_profile_text, base_profile_name, reference, conan_pip_package=None, docker_image=None, sudo_docker_command=None, sudo_pip_command=False, docker_image_skip_update=False, build_policy=None, docker_image_skip_pull=False, always_update_conan_in_docker=False, upload=False, upload_retry=None, upload_only_recipe=None, runner=None, docker_shell="", docker_conan_home="", docker_platform_param="", docker_run_options="", lcow_user_workaround="", test_folder=None, pip_install=None, docker_pip_command=None, config_url=None, config_args=None, printer=None, upload_dependencies=None, conanfile=None, force_selinux=None, skip_recipe_export=False, update_dependencies=False, lockfile=None): self.printer = printer or Printer() self._upload = upload self._upload_retry = upload_retry self._upload_only_recipe = upload_only_recipe self._reference = reference self._conan_pip_package = conan_pip_package self._build_policy = build_policy self._docker_image = docker_image self._always_update_conan_in_docker = always_update_conan_in_docker self._docker_image_skip_update = docker_image_skip_update self._docker_image_skip_pull = docker_image_skip_pull self._sudo_docker_command = sudo_docker_command or "" self._sudo_pip_command = sudo_pip_command self._profile_text = profile_text self._base_profile_text = base_profile_text self._base_profile_name = base_profile_name self._docker_shell = docker_shell self._docker_conan_home = docker_conan_home self._docker_platform_param = docker_platform_param self._docker_run_options = docker_run_options or "" self._lcow_user_workaround = lcow_user_workaround self._runner = PrintRunner(runner, self.printer) self._test_folder = test_folder self._pip_install = pip_install self._docker_pip_command = docker_pip_command self._config_url = config_url self._config_args = config_args self._upload_dependencies = upload_dependencies or [] self._conanfile = conanfile self._lockfile = lockfile self._force_selinux = force_selinux self._skip_recipe_export = skip_recipe_export self._update_dependencies = update_dependencies
def test_plain(self): manager = RemotesManager(self.conan_api, Printer(), remotes_input="url1, url", upload_input="url1") expected = { 'CONAN_REMOTES': 'url1@True@remote0,url@True@remote1', 'CONAN_UPLOAD': 'url1@True@upload_repo' } self.assert_serial_deserial(manager, expected)
def test_valid_config(self): manager = ConfigManager(self.api, Printer()) profiles = self.api.profile_list() self.assertEquals(len(profiles), 0) manager.install("https://github.com/bincrafters/bincrafters-config.git", "-b main") profiles = self.api.profile_list() self.assertGreater(len(profiles), 3)
def test_invalid_config(self): manager = ConfigManager(self.api, Printer()) profiles = self.api.profile_list() self.assertEquals(len(profiles), 0) try: manager.install("https://github.com/") self.fail("Could not accept wrong URL") except ConanException: pass
def test_list(self): remotes_input = [("url1", True, "remote1"), ("url2", False, "remote2")] upload_input = ("url3", True, "remote1") manager = RemotesManager(self.conan_api, Printer(), remotes_input=remotes_input, upload_input=upload_input) expected = { 'CONAN_REMOTES': 'url1@True@remote1,url2@False@remote2', 'CONAN_UPLOAD': 'url3@True@remote1' } self.assert_serial_deserial(manager, expected)
def setUp(self): # Clean env first new_env = {} for var, value in os.environ.items(): if "travis" not in var.lower() and \ "appveyor" not in var.lower() and \ "bamboo" not in var.lower(): new_env[var] = value os.environ = new_env self.output = TestBufferConanOutput() self.printer = Printer(self.output.write)
def test_duplicated_remotes_with_same_url(self): self.api.remote_remove("conancenter") self.api.remote_add("upload_repo", "url1", True) manager = RemotesManager(self.api, Printer(), remotes_input="url1@True@upload_repo", upload_input="url1@True@upload_repo") remotes = self.api.remote_list() self.assertIsNotNone(manager._get_remote_by_name(remotes, "upload_repo")) manager.add_remotes_to_conan() self.assertEquals(len(self.api.remote_list()), 1)
def get_version_from_ci(): printer = Printer(hidesensitive) ci_man = CIManager(printer) if ci_man.is_tag(): try: version = subprocess.check_output( "git describe --exact-match --tags", shell=True).decode().strip() except Exception: return None else: version = ci_man.get_branch() version = re.sub("^refs/tags/", "", version) version = re.sub("^.*/v?", "", version) version = re.sub("^v?", "", version) return version
def upload(self): """ Set taocpp repository to be used on upload. The upload server address could be customized by env var CONAN_UPLOAD. If not defined, the method will check the branch name. Only master or CONAN_STABLE_BRANCH_PATTERN will be accepted. The master branch will be pushed to testing channel, because it does not match the stable pattern. Otherwise it will upload to stable channel. """ if os.getenv("CONAN_UPLOAD", None) is not None: return os.getenv("CONAN_UPLOAD") printer = Printer(None) ci_manager = CIManager(printer) branch = ci_manager.get_branch() patterns = [r"v?\d+\.\d+\.\d+-.*", self.stable_branch_pattern] for pattern in patterns: prog = re.compile(pattern) if branch and prog.match(branch): return "https://api.bintray.com/conan/taocpp/public-conan"
#!/usr/bin/env python3 import argparse import errno import tempfile from pathlib import Path import yaml from cpt.printer import Printer printer = Printer() from ci_tools_gha.utils import get_builder_default, is_pure_c gha_hack = True import os import shutil def copytree(src, dst, symlinks=False, ignore=None): for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(dst, item) if os.path.isdir(s): shutil.copytree(s, d, symlinks, ignore) else: shutil.copy2(s, d) def gha_hack_copy(src):
def _branch(self): """ Get branch name from CI manager """ printer = Printer(None) ci_manager = CIManager(printer) return ci_manager.get_branch()
class DockerCreateRunner(object): def __init__(self, profile_text, base_profile_text, base_profile_name, reference, args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=None, sudo_pip_command=True, docker_image_skip_update=False, build_policy=None, always_update_conan_in_docker=False, upload=False, runner=None): self.printer = Printer() self._args = args self._upload = upload self._reference = reference self._conan_pip_package = conan_pip_package self._build_policy = build_policy self._docker_image = docker_image self._always_update_conan_in_docker = always_update_conan_in_docker self._docker_image_skip_update = docker_image_skip_update self._sudo_docker_command = sudo_docker_command or "" self._sudo_pip_command = sudo_pip_command self._profile_text = profile_text self._base_profile_text = base_profile_text self._base_profile_name = base_profile_name self._runner = PrintRunner(runner, self.printer) def _pip_update_conan_command(self): commands = [] # Hack for testing when retriving cpt from artifactory repo if "conan-package-tools" not in self._conan_pip_package: commands.append("%s pip install conan_package_tools==%s " "--upgrade --no-cache" % (self._sudo_pip_command, package_tools_version)) if self._conan_pip_package: commands.append("%s pip install %s --no-cache" % (self._sudo_pip_command, self._conan_pip_package)) else: commands.append("%s pip install conan --upgrade --no-cache" % self._sudo_pip_command) command = " && ".join(commands) return command def run(self, pull_image=True, docker_entry_script=None): if pull_image: self.pull_image() if not self._docker_image_skip_update and not self._always_update_conan_in_docker: # Update the downloaded image with self.printer.foldable_output("update conan"): command = '%s docker run --name conan_runner ' \ ' %s /bin/sh -c "%s"' % (self._sudo_docker_command, self._docker_image, self._pip_update_conan_command()) ret = self._runner(command) if ret != 0: raise Exception("Error updating the image: %s" % command) # Save the image with the updated installed # packages and remove the intermediate container command = "%s docker commit conan_runner %s" % (self._sudo_docker_command, self._docker_image) ret = self._runner(command) if ret != 0: raise Exception("Error commiting the image: %s" % command) command = "%s docker rm conan_runner" % self._sudo_docker_command ret = self._runner(command) if ret != 0: raise Exception("Error updating the image: %s" % command) # Run the build envs = self.get_env_vars() env_vars_text = " ".join(['-e %s="%s"' % (key, value) for key, value in envs.items() if value]) if self._always_update_conan_in_docker: update_command = self._pip_update_conan_command() + " && " else: update_command = "" command = ("%s docker run --rm -v%s:/home/conan/project %s %s /bin/sh " "-c \"cd project && " "%s run_create_in_docker \"" % (self._sudo_docker_command, os.getcwd(), env_vars_text, self._docker_image, update_command)) # Push entry command before to build if docker_entry_script: command = command.replace("run_create_in_docker", "%s && run_create_in_docker" % docker_entry_script) self.printer.print_in_docker(self._docker_image) ret = self._runner(command) if ret != 0: raise Exception("Error building: %s" % command) self.printer.print_message("Exiting docker...") def pull_image(self): with self.printer.foldable_output("docker pull"): ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image)) if ret != 0: raise Exception("Error pulling the image: %s" % self._docker_image) def get_env_vars(self): ret = {key: value for key, value in os.environ.items() if key.startswith("CONAN_") and key != "CONAN_USER_HOME"} ret["CPT_ARGS"] = escape_env(self._args) ret["CONAN_REFERENCE"] = self._reference ret["CPT_PROFILE"] = escape_env(self._profile_text) ret["CPT_BASE_PROFILE"] = escape_env(self._base_profile_text) ret["CPT_BASE_PROFILE_NAME"] = escape_env(self._base_profile_name) ret["CONAN_USERNAME"] = escape_env(self._reference.user) ret["CONAN_TEMP_TEST_FOLDER"] = "1" # test package folder to a temp one ret["CPT_UPLOAD_ENABLED"] = self._upload ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy) return ret
def __init__(self, username=None, channel=None, runner=None, gcc_versions=None, visual_versions=None, visual_runtimes=None, visual_toolsets=None, apple_clang_versions=None, archs=None, options=None, use_docker=None, curpage=None, total_pages=None, docker_image=None, reference=None, password=None, remotes=None, upload=None, stable_branch_pattern=None, vs10_x86_64_enabled=False, mingw_configurations=None, stable_channel=None, platform_info=None, upload_retry=None, clang_versions=None, login_username=None, upload_only_when_stable=None, upload_only_when_tag=None, upload_only_recipe=None, build_types=None, cppstds=None, skip_check_credentials=False, allow_gcc_minors=False, exclude_vcvars_precommand=False, docker_run_options=None, docker_image_skip_update=False, docker_image_skip_pull=False, docker_entry_script=None, docker_32_images=None, docker_conan_home=None, docker_shell=None, pip_install=None, build_policy=None, always_update_conan_in_docker=False, conan_api=None, client_cache=None, conanfile=None, ci_manager=None, out=None, test_folder=None, cwd=None, config_url=None, config_args=None, upload_dependencies=None, force_selinux=None, skip_recipe_export=False, update_dependencies=None, lockfile=None): conan_version = get_client_version() self.printer = Printer(out) self.printer.print_rule() self.printer.print_ascci_art() self.cwd = cwd or os.getcwd() if not conan_api: self.conan_api, _, _ = Conan.factory() self.conan_api.create_app() self.client_cache = self.conan_api.app.cache else: self.conan_api = conan_api self.client_cache = client_cache self.ci_manager = ci_manager or CIManager(self.printer) self.remotes_manager = RemotesManager(self.conan_api, self.printer, remotes, upload) self.username = username or os.getenv("CONAN_USERNAME", None) self.skip_check_credentials = skip_check_credentials or get_bool_from_env( "CONAN_SKIP_CHECK_CREDENTIALS") self.auth_manager = AuthManager( self.conan_api, self.printer, login_username, password, default_username=self.username, skip_check_credentials=self.skip_check_credentials) # Upload related variables self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3) if upload_only_when_stable is not None: self.upload_only_when_stable = upload_only_when_stable else: self.upload_only_when_stable = get_bool_from_env( "CONAN_UPLOAD_ONLY_WHEN_STABLE") if upload_only_when_tag is not None: self.upload_only_when_tag = upload_only_when_tag else: self.upload_only_when_tag = get_bool_from_env( "CONAN_UPLOAD_ONLY_WHEN_TAG") self.upload_only_recipe = upload_only_recipe or get_bool_from_env( "CONAN_UPLOAD_ONLY_RECIPE") self.remotes_manager.add_remotes_to_conan() self.uploader = Uploader(self.conan_api, self.remotes_manager, self.auth_manager, self.printer, self.upload_retry) self._builds = [] self._named_builds = {} self._packages_summary = [] self._update_conan_in_docker = always_update_conan_in_docker or get_bool_from_env( "CONAN_ALWAYS_UPDATE_CONAN_DOCKER") self._platform_info = platform_info or PlatformInfo() self.stable_branch_pattern = stable_branch_pattern or \ os.getenv("CONAN_STABLE_BRANCH_PATTERN", None) self.stable_channel = stable_channel or os.getenv( "CONAN_STABLE_CHANNEL", "stable") self.stable_channel = self.stable_channel.rstrip() self.partial_reference = reference or os.getenv( "CONAN_REFERENCE", None) self.channel = self._get_specified_channel(channel, reference) self.conanfile = conanfile or os.getenv("CONAN_CONANFILE", "conanfile.py") if self.partial_reference: if "@" in self.partial_reference: self.reference = ConanFileReference.loads( self.partial_reference) else: name, version = self.partial_reference.split("/") self.reference = ConanFileReference(name, version, self.username, self.channel) else: if not os.path.exists(os.path.join(self.cwd, self.conanfile)): raise Exception("Conanfile not found, specify a 'reference' " "parameter with name and version") conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile), self.conan_api) name, version = conanfile.name, conanfile.version if name and version: self.reference = ConanFileReference(name, version, self.username, self.channel) else: self.reference = None self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE", None) # If CONAN_DOCKER_IMAGE is specified, then use docker is True self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False) or self._docker_image is not None) self.docker_conan_home = docker_conan_home or os.getenv( "CONAN_DOCKER_HOME", None) os_name = self._platform_info.system( ) if not self.use_docker else "Linux" self.build_generator = BuildGenerator( reference, os_name, gcc_versions, apple_clang_versions, clang_versions, visual_versions, visual_runtimes, visual_toolsets, vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors, build_types, options, cppstds) self.build_policy = (build_policy or self.ci_manager.get_commit_build_policy() or split_colon_env("CONAN_BUILD_POLICY")) if isinstance(self.build_policy, list): self.build_policy = ",".join(self.build_policy) self.sudo_docker_command = "" if "CONAN_DOCKER_USE_SUDO" in os.environ: self.sudo_docker_command = "sudo -E" if get_bool_from_env( "CONAN_DOCKER_USE_SUDO") else "" elif platform.system() != "Windows": self.sudo_docker_command = "sudo -E" self.sudo_pip_command = "" if "CONAN_PIP_USE_SUDO" in os.environ: self.sudo_pip_command = "sudo -E" if get_bool_from_env( "CONAN_PIP_USE_SUDO") else "" elif platform.system( ) != "Windows" and self._docker_image and 'conanio/' not in str( self._docker_image): self.sudo_pip_command = "sudo -E" self.pip_command = os.getenv("CONAN_PIP_COMMAND", "pip") pip_found = True if tools.os_info.is_windows else tools.which( self.pip_command) if not pip_found or not "pip" in self.pip_command: raise Exception( "CONAN_PIP_COMMAND: '{}' is not a valid pip command.".format( self.pip_command)) self.docker_pip_command = os.getenv("CONAN_DOCKER_PIP_COMMAND", "pip") self.docker_shell = docker_shell or os.getenv("CONAN_DOCKER_SHELL") if self.is_wcow: if self.docker_conan_home is None: self.docker_conan_home = "C:/Users/ContainerAdministrator" self.docker_shell = docker_shell or "cmd /C" else: if self.docker_conan_home is None: self.docker_conan_home = "/home/conan" self.docker_shell = docker_shell or "/bin/sh -c" self.docker_platform_param = "" self.lcow_user_workaround = "" if self.is_lcow: self.docker_platform_param = "--platform=linux" # With LCOW, Docker doesn't respect USER directive in dockerfile yet self.lcow_user_workaround = "sudo su conan && " self.exclude_vcvars_precommand = exclude_vcvars_precommand or \ get_bool_from_env("CONAN_EXCLUDE_VCVARS_PRECOMMAND") self._docker_image_skip_update = docker_image_skip_update or \ get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_UPDATE") self._docker_image_skip_pull = docker_image_skip_pull or \ get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_PULL") self.runner = runner or os.system self.output_runner = ConanOutputRunner() self.docker_run_options = docker_run_options or split_colon_env( "CONAN_DOCKER_RUN_OPTIONS") if isinstance(self.docker_run_options, list): self.docker_run_options = " ".join(self.docker_run_options) self.docker_entry_script = docker_entry_script or os.getenv( "CONAN_DOCKER_ENTRY_SCRIPT") self.pip_install = pip_install or split_colon_env("CONAN_PIP_INSTALL") self.upload_dependencies = upload_dependencies or split_colon_env( "CONAN_UPLOAD_DEPENDENCIES") or "" if isinstance(self.upload_dependencies, list): self.upload_dependencies = ",".join(self.upload_dependencies) if "all" in self.upload_dependencies and self.upload_dependencies != "all": raise Exception( "Upload dependencies only accepts or 'all' or package references. Do not mix both!" ) self.update_dependencies = update_dependencies or get_bool_from_env( "CONAN_UPDATE_DEPENDENCIES") if self.channel: os.environ["CONAN_CHANNEL"] = self.channel if docker_32_images is not None: self.docker_32_images = docker_32_images else: self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False) self.force_selinux = force_selinux or get_bool_from_env( "CONAN_FORCE_SELINUX") self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1) self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1) self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE", "conan==%s" % conan_version) if self.conan_pip_package in ("0", "False"): self.conan_pip_package = "" self.vs10_x86_64_enabled = vs10_x86_64_enabled self.builds_in_current_page = [] self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER") self.config_url = config_url or os.getenv("CONAN_CONFIG_URL") self.skip_recipe_export = skip_recipe_export or \ get_bool_from_env("CONAN_SKIP_RECIPE_EXPORT") self.config_args = config_args or os.getenv("CONAN_CONFIG_ARGS") self.lockfile = lockfile or os.getenv("CONAN_LOCKFILE") def valid_pair(var, value): return (isinstance(value, six.string_types) or isinstance(value, bool) or isinstance(value, list) ) and not var.startswith("_") and "password" not in var with self.printer.foldable_output("local_vars"): self.printer.print_dict({ var: value for var, value in self.__dict__.items() if valid_pair(var, value) }) self._newest_supported_conan_version = Version( NEWEST_CONAN_SUPPORTED).minor(fill=False) self._client_conan_version = conan_version
class ConanMultiPackager(object): """ Help to generate common builds (setting's combinations), adjust the environment, and run conan create command in docker containers""" def __init__(self, username=None, channel=None, runner=None, gcc_versions=None, visual_versions=None, visual_runtimes=None, visual_toolsets=None, apple_clang_versions=None, archs=None, options=None, use_docker=None, curpage=None, total_pages=None, docker_image=None, reference=None, password=None, remotes=None, upload=None, stable_branch_pattern=None, vs10_x86_64_enabled=False, mingw_configurations=None, stable_channel=None, platform_info=None, upload_retry=None, clang_versions=None, login_username=None, upload_only_when_stable=None, upload_only_when_tag=None, upload_only_recipe=None, build_types=None, cppstds=None, skip_check_credentials=False, allow_gcc_minors=False, exclude_vcvars_precommand=False, docker_run_options=None, docker_image_skip_update=False, docker_image_skip_pull=False, docker_entry_script=None, docker_32_images=None, docker_conan_home=None, docker_shell=None, pip_install=None, build_policy=None, always_update_conan_in_docker=False, conan_api=None, client_cache=None, conanfile=None, ci_manager=None, out=None, test_folder=None, cwd=None, config_url=None, config_args=None, upload_dependencies=None, force_selinux=None, skip_recipe_export=False, update_dependencies=None, lockfile=None): conan_version = get_client_version() self.printer = Printer(out) self.printer.print_rule() self.printer.print_ascci_art() self.cwd = cwd or os.getcwd() if not conan_api: self.conan_api, _, _ = Conan.factory() self.conan_api.create_app() self.client_cache = self.conan_api.app.cache else: self.conan_api = conan_api self.client_cache = client_cache self.ci_manager = ci_manager or CIManager(self.printer) self.remotes_manager = RemotesManager(self.conan_api, self.printer, remotes, upload) self.username = username or os.getenv("CONAN_USERNAME", None) self.skip_check_credentials = skip_check_credentials or get_bool_from_env( "CONAN_SKIP_CHECK_CREDENTIALS") self.auth_manager = AuthManager( self.conan_api, self.printer, login_username, password, default_username=self.username, skip_check_credentials=self.skip_check_credentials) # Upload related variables self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3) if upload_only_when_stable is not None: self.upload_only_when_stable = upload_only_when_stable else: self.upload_only_when_stable = get_bool_from_env( "CONAN_UPLOAD_ONLY_WHEN_STABLE") if upload_only_when_tag is not None: self.upload_only_when_tag = upload_only_when_tag else: self.upload_only_when_tag = get_bool_from_env( "CONAN_UPLOAD_ONLY_WHEN_TAG") self.upload_only_recipe = upload_only_recipe or get_bool_from_env( "CONAN_UPLOAD_ONLY_RECIPE") self.remotes_manager.add_remotes_to_conan() self.uploader = Uploader(self.conan_api, self.remotes_manager, self.auth_manager, self.printer, self.upload_retry) self._builds = [] self._named_builds = {} self._packages_summary = [] self._update_conan_in_docker = always_update_conan_in_docker or get_bool_from_env( "CONAN_ALWAYS_UPDATE_CONAN_DOCKER") self._platform_info = platform_info or PlatformInfo() self.stable_branch_pattern = stable_branch_pattern or \ os.getenv("CONAN_STABLE_BRANCH_PATTERN", None) self.stable_channel = stable_channel or os.getenv( "CONAN_STABLE_CHANNEL", "stable") self.stable_channel = self.stable_channel.rstrip() self.partial_reference = reference or os.getenv( "CONAN_REFERENCE", None) self.channel = self._get_specified_channel(channel, reference) self.conanfile = conanfile or os.getenv("CONAN_CONANFILE", "conanfile.py") if self.partial_reference: if "@" in self.partial_reference: self.reference = ConanFileReference.loads( self.partial_reference) else: name, version = self.partial_reference.split("/") self.reference = ConanFileReference(name, version, self.username, self.channel) else: if not os.path.exists(os.path.join(self.cwd, self.conanfile)): raise Exception("Conanfile not found, specify a 'reference' " "parameter with name and version") conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile), self.conan_api) name, version = conanfile.name, conanfile.version if name and version: self.reference = ConanFileReference(name, version, self.username, self.channel) else: self.reference = None self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE", None) # If CONAN_DOCKER_IMAGE is specified, then use docker is True self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False) or self._docker_image is not None) self.docker_conan_home = docker_conan_home or os.getenv( "CONAN_DOCKER_HOME", None) os_name = self._platform_info.system( ) if not self.use_docker else "Linux" self.build_generator = BuildGenerator( reference, os_name, gcc_versions, apple_clang_versions, clang_versions, visual_versions, visual_runtimes, visual_toolsets, vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors, build_types, options, cppstds) self.build_policy = (build_policy or self.ci_manager.get_commit_build_policy() or split_colon_env("CONAN_BUILD_POLICY")) if isinstance(self.build_policy, list): self.build_policy = ",".join(self.build_policy) self.sudo_docker_command = "" if "CONAN_DOCKER_USE_SUDO" in os.environ: self.sudo_docker_command = "sudo -E" if get_bool_from_env( "CONAN_DOCKER_USE_SUDO") else "" elif platform.system() != "Windows": self.sudo_docker_command = "sudo -E" self.sudo_pip_command = "" if "CONAN_PIP_USE_SUDO" in os.environ: self.sudo_pip_command = "sudo -E" if get_bool_from_env( "CONAN_PIP_USE_SUDO") else "" elif platform.system( ) != "Windows" and self._docker_image and 'conanio/' not in str( self._docker_image): self.sudo_pip_command = "sudo -E" self.pip_command = os.getenv("CONAN_PIP_COMMAND", "pip") pip_found = True if tools.os_info.is_windows else tools.which( self.pip_command) if not pip_found or not "pip" in self.pip_command: raise Exception( "CONAN_PIP_COMMAND: '{}' is not a valid pip command.".format( self.pip_command)) self.docker_pip_command = os.getenv("CONAN_DOCKER_PIP_COMMAND", "pip") self.docker_shell = docker_shell or os.getenv("CONAN_DOCKER_SHELL") if self.is_wcow: if self.docker_conan_home is None: self.docker_conan_home = "C:/Users/ContainerAdministrator" self.docker_shell = docker_shell or "cmd /C" else: if self.docker_conan_home is None: self.docker_conan_home = "/home/conan" self.docker_shell = docker_shell or "/bin/sh -c" self.docker_platform_param = "" self.lcow_user_workaround = "" if self.is_lcow: self.docker_platform_param = "--platform=linux" # With LCOW, Docker doesn't respect USER directive in dockerfile yet self.lcow_user_workaround = "sudo su conan && " self.exclude_vcvars_precommand = exclude_vcvars_precommand or \ get_bool_from_env("CONAN_EXCLUDE_VCVARS_PRECOMMAND") self._docker_image_skip_update = docker_image_skip_update or \ get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_UPDATE") self._docker_image_skip_pull = docker_image_skip_pull or \ get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_PULL") self.runner = runner or os.system self.output_runner = ConanOutputRunner() self.docker_run_options = docker_run_options or split_colon_env( "CONAN_DOCKER_RUN_OPTIONS") if isinstance(self.docker_run_options, list): self.docker_run_options = " ".join(self.docker_run_options) self.docker_entry_script = docker_entry_script or os.getenv( "CONAN_DOCKER_ENTRY_SCRIPT") self.pip_install = pip_install or split_colon_env("CONAN_PIP_INSTALL") self.upload_dependencies = upload_dependencies or split_colon_env( "CONAN_UPLOAD_DEPENDENCIES") or "" if isinstance(self.upload_dependencies, list): self.upload_dependencies = ",".join(self.upload_dependencies) if "all" in self.upload_dependencies and self.upload_dependencies != "all": raise Exception( "Upload dependencies only accepts or 'all' or package references. Do not mix both!" ) self.update_dependencies = update_dependencies or get_bool_from_env( "CONAN_UPDATE_DEPENDENCIES") if self.channel: os.environ["CONAN_CHANNEL"] = self.channel if docker_32_images is not None: self.docker_32_images = docker_32_images else: self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False) self.force_selinux = force_selinux or get_bool_from_env( "CONAN_FORCE_SELINUX") self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1) self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1) self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE", "conan==%s" % conan_version) if self.conan_pip_package in ("0", "False"): self.conan_pip_package = "" self.vs10_x86_64_enabled = vs10_x86_64_enabled self.builds_in_current_page = [] self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER") self.config_url = config_url or os.getenv("CONAN_CONFIG_URL") self.skip_recipe_export = skip_recipe_export or \ get_bool_from_env("CONAN_SKIP_RECIPE_EXPORT") self.config_args = config_args or os.getenv("CONAN_CONFIG_ARGS") self.lockfile = lockfile or os.getenv("CONAN_LOCKFILE") def valid_pair(var, value): return (isinstance(value, six.string_types) or isinstance(value, bool) or isinstance(value, list) ) and not var.startswith("_") and "password" not in var with self.printer.foldable_output("local_vars"): self.printer.print_dict({ var: value for var, value in self.__dict__.items() if valid_pair(var, value) }) self._newest_supported_conan_version = Version( NEWEST_CONAN_SUPPORTED).minor(fill=False) self._client_conan_version = conan_version def _check_conan_version(self): tmp = self._newest_supported_conan_version if Version(self._client_conan_version).minor(fill=False) > tmp: msg = "Conan/CPT version mismatch. Conan version installed: " \ "%s . This version of CPT supports only Conan < %s" \ "" % (self._client_conan_version, str(tmp)) self.printer.print_message(msg) raise Exception(msg) # For Docker on Windows, including Linux containers on Windows @property def is_lcow(self): return self.container_os == "linux" and platform.system() == "Windows" @property def is_wcow(self): return self.container_os == "windows" and platform.system( ) == "Windows" @property def container_os(self): # CONAN_DOCKER_PLATFORM=linux must be specified for LCOW if self.use_docker: if "CONAN_DOCKER_PLATFORM" in os.environ: return os.getenv("CONAN_DOCKER_PLATFORM", "windows").lower() else: return "windows" else: return "" @property def packages_summary(self): return self._packages_summary def save_packages_summary(self, file): self.printer.print_message("Saving packages summary to " + file) import json import datetime def default(o): if isinstance(o, (datetime.date, datetime.datetime)): return o.isoformat() with open(file, 'w') as outfile: json.dump(self.packages_summary, outfile, default=default) @property def items(self): return self._builds @items.setter def items(self, confs): self.builds = confs @property def builds(self): # Retrocompatibility iterating self.printer.print_message( "WARNING", "\n\n\n******* ITERATING THE CONAN_PACKAGE_TOOLS BUILDS WITH " ".builds is deprecated use '.items' instead (unpack 5 elements: " "settings, options, env_vars, build_requires, reference *******" "**\n\n\n") return [elem[0:4] for elem in self._builds] @builds.setter def builds(self, confs): """For retro compatibility directly assigning builds""" self._named_builds = {} self._builds = [] for values in confs: if len(values) == 2: self._builds.append( BuildConf(values[0], values[1], {}, {}, self.reference)) elif len(values) == 4: self._builds.append( BuildConf(values[0], values[1], values[2], values[3], self.reference)) elif len(values) != 5: raise Exception( "Invalid build configuration, has to be a tuple of " "(settings, options, env_vars, build_requires, reference)") else: self._builds.append(BuildConf(*values)) @property def named_builds(self): return self._named_builds @named_builds.setter def named_builds(self, confs): self._builds = [] self._named_builds = {} for key, pages in confs.items(): for values in pages: if len(values) == 2: bc = BuildConf(values[0], values[1], {}, {}, self.reference) self._named_builds.setdefault(key, []).append(bc) elif len(values) == 4: bc = BuildConf(values[0], values[1], values[2], values[3], self.reference) self._named_builds.setdefault(key, []).append(bc) elif len(values) != 5: raise Exception( "Invalid build configuration, has to be a tuple of " "(settings, options, env_vars, build_requires, reference)" ) else: self._named_builds.setdefault(key, []).append( BuildConf(*values)) def login(self, remote_name): self.auth_manager.login(remote_name) def add_common_builds(self, shared_option_name=None, pure_c=True, dll_with_static_runtime=False, reference=None, header_only=True, build_all_options_values=None): if reference: if "@" in reference: reference = ConanFileReference.loads(reference) else: name, version = reference.split("/") reference = ConanFileReference(name, version, self.username, self.channel) else: reference = self.reference if not reference: raise Exception( "Specify a CONAN_REFERENCE or name and version fields in the recipe" ) if shared_option_name is None: env_shared_option_name = os.getenv("CONAN_SHARED_OPTION_NAME", None) shared_option_name = env_shared_option_name if str( env_shared_option_name).lower() != "false" else False build_all_options_values = build_all_options_values or split_colon_env( "CONAN_BUILD_ALL_OPTIONS_VALUES") or [] if not isinstance(build_all_options_values, list): raise Exception( "'build_all_options_values' must be a list. e.g. ['foo:opt', 'foo:bar']" ) conanfile = None if os.path.exists(os.path.join(self.cwd, self.conanfile)): conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile), self.conan_api) header_only_option = None if conanfile: if hasattr( conanfile, "options" ) and conanfile.options and "header_only" in conanfile.options: header_only_option = "%s:header_only" % reference.name if shared_option_name is None: if conanfile: if hasattr( conanfile, "options" ) and conanfile.options and "shared" in conanfile.options: shared_option_name = "%s:shared" % reference.name # filter only valid options raw_options_for_building = [ opt[opt.find(":") + 1:] for opt in build_all_options_values ] for raw_option in reversed(raw_options_for_building): if hasattr(conanfile, "options") and conanfile.options and \ not isinstance(conanfile.options.get(raw_option), list): raw_options_for_building.remove(raw_option) if raw_options_for_building and conanfile: # get option and its values cloned_options = copy.copy(conanfile.options) for key, value in conanfile.options.items(): if key == "shared" and shared_option_name: continue elif key not in raw_options_for_building: del cloned_options[key] cloned_options2 = {} for key, value in cloned_options.items(): # add package reference to the option name if not key.startswith("{}:".format(reference.name)): cloned_options2["{}:{}".format(reference.name, key)] = value # combine all options x values (cartesian product) build_all_options_values = [ dict(zip(cloned_options2, v)) for v in product(*cloned_options2.values()) ] builds = self.build_generator.get_builds(pure_c, shared_option_name, dll_with_static_runtime, reference, build_all_options_values) if header_only_option and header_only: if conanfile.default_options.get("header_only"): cloned_builds = copy.deepcopy(builds) for settings, options, env_vars, build_requires, reference in cloned_builds: options.update({header_only_option: False}) builds.extend(cloned_builds) else: settings, options, env_vars, build_requires, reference = builds[ 0] cloned_options = copy.copy(options) cloned_options.update({header_only_option: True}) builds.append( BuildConf(copy.copy(settings), cloned_options, copy.copy(env_vars), copy.copy(build_requires), reference)) self._builds.extend(builds) def add(self, settings=None, options=None, env_vars=None, build_requires=None, reference=None): settings = settings or {} options = options or {} env_vars = env_vars or {} build_requires = build_requires or {} if reference: reference = ConanFileReference.loads( "%s@%s/%s" % (reference, self.username, self.channel)) reference = reference or self.reference self._builds.append( BuildConf(settings, options, env_vars, build_requires, reference)) def remove_build_if(self, predicate): filtered_builds = [] for build in self.items: if not predicate(build): filtered_builds.append(build) self._builds = filtered_builds def update_build_if(self, predicate, new_settings=None, new_options=None, new_env_vars=None, new_build_requires=None, new_reference=None): updated_builds = [] for build in self.items: if predicate(build): if new_settings: build.settings.update(new_settings) if new_options: build.options.update(new_options) if new_build_requires: build.build_requires.update(new_build_requires) if new_env_vars: build.env_vars.update(new_env_vars) if new_reference: build.reference = new_reference updated_builds.append(build) self._builds = updated_builds def run(self, base_profile_name=None, summary_file=None): self._check_conan_version() env_vars = self.auth_manager.env_vars() env_vars.update(self.remotes_manager.env_vars()) with tools.environment_append(env_vars): self.printer.print_message("Running builds...") if self.ci_manager.skip_builds(): self.printer.print_message( "Skipped builds due [skip ci] commit message") return 99 if not self.skip_check_credentials and self._upload_enabled(): self.auth_manager.login( self.remotes_manager.upload_remote_name) if self.conan_pip_package and not self.use_docker: with self.printer.foldable_output("pip_update"): self.runner('%s %s install -q %s' % (self.sudo_pip_command, self.pip_command, self.conan_pip_package)) if self.pip_install: packages = " ".join(self.pip_install) self.printer.print_message( "Install extra python packages: {}".format( packages)) self.runner('%s %s install -q %s' % (self.sudo_pip_command, self.pip_command, packages)) self.run_builds(base_profile_name=base_profile_name) summary_file = summary_file or os.getenv("CPT_SUMMARY_FILE", None) if summary_file: self.save_packages_summary(summary_file) def _upload_enabled(self): if not self.remotes_manager.upload_remote_name: return False if not self.auth_manager.credentials_ready( self.remotes_manager.upload_remote_name): return False if self.upload_only_when_tag and not self.ci_manager.is_tag(): self.printer.print_message("Skipping upload, not tag branch") return False st_channel = self.stable_channel or "stable" if self.upload_only_when_stable and self.channel != st_channel and not self.upload_only_when_tag: self.printer.print_message("Skipping upload, not stable channel") return False if not os.getenv("CONAN_TEST_SUITE", False): if self.ci_manager.is_pull_request(): # PENDING! can't found info for gitlab/bamboo self.printer.print_message( "Skipping upload, this is a Pull Request") return False def raise_error(field): raise Exception("Upload not possible, '%s' is missing!" % field) if not self.channel and "@" not in self.partial_reference: raise_error("channel") if not self.username and "@" not in self.partial_reference: raise_error("username") return True def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): if len(self.named_builds) > 0 and len(self.items) > 0: raise Exception( "Both bulk and named builds are set. Only one is allowed.") self.builds_in_current_page = [] if len(self.items) > 0: curpage = curpage or int(self.curpage) total_pages = total_pages or int(self.total_pages) for index, build in enumerate(self.items): if curpage is None or total_pages is None or ( index % total_pages) + 1 == curpage: self.builds_in_current_page.append(build) elif len(self.named_builds) > 0: curpage = curpage or self.curpage if curpage not in self.named_builds: raise Exception("No builds set for page %s" % curpage) for build in self.named_builds[curpage]: self.builds_in_current_page.append(build) self.printer.print_current_page(curpage, total_pages) self.printer.print_jobs(self.builds_in_current_page) pulled_docker_images = defaultdict(lambda: False) skip_recipe_export = False # FIXME: Remove in Conan 1.3, https://github.com/conan-io/conan/issues/2787 for index, build in enumerate(self.builds_in_current_page): self.printer.print_message( "Build: %s/%s" % (index + 1, len(self.builds_in_current_page))) base_profile_name = base_profile_name or os.getenv( "CONAN_BASE_PROFILE") if base_profile_name: self.printer.print_message( "**************************************************") self.printer.print_message("Using specified default " "base profile: %s" % base_profile_name) self.printer.print_message( "**************************************************") profile_text, base_profile_text = get_profiles( self.client_cache, build, base_profile_name) if not self.use_docker: profile_abs_path = save_profile_to_tmp(profile_text) r = CreateRunner( profile_abs_path, build.reference, self.conan_api, self.uploader, exclude_vcvars_precommand=self.exclude_vcvars_precommand, build_policy=self.build_policy, runner=self.runner, cwd=self.cwd, printer=self.printer, upload=self._upload_enabled(), upload_only_recipe=self.upload_only_recipe, test_folder=self.test_folder, config_url=self.config_url, config_args=self.config_args, upload_dependencies=self.upload_dependencies, conanfile=self.conanfile, lockfile=self.lockfile, skip_recipe_export=skip_recipe_export, update_dependencies=self.update_dependencies) r.run() self._packages_summary.append({ "configuration": build, "package": r.results }) else: docker_image = self._get_docker_image(build) r = DockerCreateRunner( profile_text, base_profile_text, base_profile_name, build.reference, conan_pip_package=self.conan_pip_package, docker_image=docker_image, sudo_docker_command=self.sudo_docker_command, sudo_pip_command=self.sudo_pip_command, docker_image_skip_update=self._docker_image_skip_update, docker_image_skip_pull=self._docker_image_skip_pull, build_policy=self.build_policy, always_update_conan_in_docker=self._update_conan_in_docker, upload=self._upload_enabled(), upload_retry=self.upload_retry, upload_only_recipe=self.upload_only_recipe, runner=self.runner, docker_shell=self.docker_shell, docker_conan_home=self.docker_conan_home, docker_platform_param=self.docker_platform_param, docker_run_options=self.docker_run_options, lcow_user_workaround=self.lcow_user_workaround, test_folder=self.test_folder, pip_install=self.pip_install, docker_pip_command=self.docker_pip_command, config_url=self.config_url, config_args=self.config_args, printer=self.printer, upload_dependencies=self.upload_dependencies, conanfile=self.conanfile, lockfile=self.lockfile, force_selinux=self.force_selinux, skip_recipe_export=skip_recipe_export, update_dependencies=self.update_dependencies) r.run(pull_image=not pulled_docker_images[docker_image], docker_entry_script=self.docker_entry_script) pulled_docker_images[docker_image] = True skip_recipe_export = self.skip_recipe_export def _get_docker_image(self, build): if self._docker_image: docker_image = self._docker_image else: compiler_name = build.settings.get("compiler", "") compiler_version = build.settings.get("compiler.version", "") docker_image = self._autodetect_docker_base_image( compiler_name, compiler_version) arch = build.settings.get("arch", "") or build.settings.get( "arch_build", "") if self.docker_32_images and arch == "x86": build.settings["arch_build"] = "x86" docker_arch_suffix = "x86" elif arch != "x86" and arch != "x86_64": docker_arch_suffix = arch else: docker_arch_suffix = None if docker_arch_suffix and "-" not in docker_image: docker_image = "%s-%s" % (docker_image, docker_arch_suffix) return docker_image @staticmethod def _autodetect_docker_base_image(compiler_name, compiler_version): if compiler_name not in ["clang", "gcc"]: raise Exception("Docker image cannot be autodetected for " "the compiler %s" % compiler_name) if compiler_name == "gcc" and Version(compiler_version) > Version("5"): compiler_version = Version(compiler_version).major(fill=False) return "conanio/%s%s" % (compiler_name, compiler_version.replace(".", "")) def _get_channel(self, specified_channel, stable_channel, upload_when_tag): if not specified_channel: return if self.stable_branch_pattern: stable_patterns = [self.stable_branch_pattern] else: stable_patterns = ["master$", "release*", "stable*"] branch = self.ci_manager.get_branch() self.printer.print_message("Branch detected", branch) for pattern in stable_patterns: prog = re.compile(pattern) if branch and prog.match(branch): self.printer.print_message( "Info", "Redefined channel by CI branch matching with '%s', " "setting CONAN_CHANNEL to '%s'" % (pattern, stable_channel)) return stable_channel if self.ci_manager.is_tag() and upload_when_tag: self.printer.print_message( "Info", "Redefined channel by branch tag, " "setting CONAN_CHANNEL to '%s'" % stable_channel) return stable_channel return specified_channel def _get_specified_channel(self, channel, reference): partial_reference = reference or os.getenv("CONAN_REFERENCE", None) specified_channel = None # without name/channel e.g. zlib/1.2.11@ if partial_reference: if "@" in partial_reference: specified_channel = channel or os.getenv("CONAN_CHANNEL", None) else: specified_channel = channel or os.getenv( "CONAN_CHANNEL", "testing") specified_channel = specified_channel.rstrip() else: if self.username: specified_channel = channel or os.getenv( "CONAN_CHANNEL", "testing") specified_channel = specified_channel.rstrip() else: specified_channel = channel or os.getenv("CONAN_CHANNEL", None) return self._get_channel(specified_channel, self.stable_channel, self.upload_only_when_tag)
def is_tag(): printer = Printer() ci_manager = CIManager(printer) return ci_manager.is_tag()
def test_valid_config(self): manager = ConfigManager(self.conan_api, Printer()) manager.install('https://github.com/bincrafters/bincrafters-config.git')
def assert_serial_deserial(self, manager, expected): self.assertEquals(manager.env_vars(), expected) with tools.environment_append(expected): manager = RemotesManager(self.conan_api, Printer()) self.assertEquals(manager.env_vars(), expected)
def __init__(self, args=None, username=None, channel=None, runner=None, gcc_versions=None, visual_versions=None, visual_runtimes=None, apple_clang_versions=None, archs=None, use_docker=None, curpage=None, total_pages=None, docker_image=None, reference=None, password=None, remotes=None, upload=None, stable_branch_pattern=None, vs10_x86_64_enabled=False, mingw_configurations=None, stable_channel=None, platform_info=None, upload_retry=None, clang_versions=None, login_username=None, upload_only_when_stable=None, build_types=None, skip_check_credentials=False, allow_gcc_minors=False, exclude_vcvars_precommand=False, docker_image_skip_update=False, docker_image_skip_pull=False, docker_entry_script=None, docker_32_images=None, build_policy=None, always_update_conan_in_docker=False, conan_api=None, client_cache=None, ci_manager=None, out=None, test_folder=None): self.printer = Printer(out) self.printer.print_rule() self.printer.print_ascci_art() if not conan_api: self.conan_api, self.client_cache, _ = Conan.factory() else: self.conan_api = conan_api self.client_cache = client_cache self.ci_manager = ci_manager or CIManager(self.printer) self.remotes_manager = RemotesManager(self.conan_api, self.printer, remotes, upload) self.username = username or os.getenv("CONAN_USERNAME", None) if not self.username: raise Exception( "Instance ConanMultiPackage with 'username' parameter or use " "CONAN_USERNAME env variable") self.skip_check_credentials = skip_check_credentials or \ os.getenv("CONAN_SKIP_CHECK_CREDENTIALS", False) self.auth_manager = AuthManager( self.conan_api, self.printer, login_username, password, default_username=self.username, skip_check_credentials=self.skip_check_credentials) self.uploader = Uploader(self.conan_api, self.remotes_manager, self.auth_manager, self.printer) self._builds = [] self._named_builds = {} self._update_conan_in_docker = ( always_update_conan_in_docker or os.getenv("CONAN_ALWAYS_UPDATE_CONAN_DOCKER", False)) self._platform_info = platform_info or PlatformInfo() self.stable_branch_pattern = stable_branch_pattern or \ os.getenv("CONAN_STABLE_BRANCH_PATTERN", None) self.specified_channel = channel or os.getenv("CONAN_CHANNEL", "testing") self.specified_channel = self.specified_channel.rstrip() self.stable_channel = stable_channel or os.getenv( "CONAN_STABLE_CHANNEL", "stable") self.stable_channel = self.stable_channel.rstrip() self.channel = self._get_channel(self.specified_channel, self.stable_channel) self.partial_reference = reference or os.getenv( "CONAN_REFERENCE", None) if self.partial_reference: if "@" in self.partial_reference: self.reference = ConanFileReference.loads( self.partial_reference) else: name, version = self.partial_reference.split("/") self.reference = ConanFileReference(name, version, self.username, self.channel) else: if not os.path.exists("conanfile.py"): raise Exception( "Conanfile not found, specify a 'reference' parameter with name and version" ) conanfile = load_conanfile_class("./conanfile.py") name, version = conanfile.name, conanfile.version if name and version: self.reference = ConanFileReference(name, version, self.username, self.channel) else: self.reference = None # If CONAN_DOCKER_IMAGE is speified, then use docker is True self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False) or os.getenv("CONAN_DOCKER_IMAGE", None) is not None) os_name = self._platform_info.system( ) if not self.use_docker else "Linux" self.build_generator = BuildGenerator( reference, os_name, gcc_versions, apple_clang_versions, clang_versions, visual_versions, visual_runtimes, vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors, build_types) build_policy = (build_policy or self.ci_manager.get_commit_build_policy() or os.getenv("CONAN_BUILD_POLICY", None)) if build_policy: if build_policy.lower() not in ("never", "outdated", "missing"): raise Exception( "Invalid build policy, valid values: never, outdated, missing" ) self.build_policy = build_policy self.sudo_docker_command = "" if "CONAN_DOCKER_USE_SUDO" in os.environ: self.sudo_docker_command = "sudo -E" if get_bool_from_env( "CONAN_DOCKER_USE_SUDO") else "" elif platform.system() != "Windows": self.sudo_docker_command = "sudo -E" self.sudo_pip_command = "" if "CONAN_PIP_USE_SUDO" in os.environ: self.sudo_pip_command = "sudo -E" if get_bool_from_env( "CONAN_PIP_USE_SUDO") else "" elif platform.system() != "Windows": self.sudo_pip_command = "sudo -E" self.docker_shell = "" self.docker_conan_home = "" if self.is_wcow: self.docker_conan_home = "C:/Users/ContainerAdministrator" self.docker_shell = "cmd /C" else: self.docker_conan_home = "/home/conan" self.docker_shell = "/bin/sh -c" self.docker_platform_param = "" self.lcow_user_workaround = "" if self.is_lcow: self.docker_platform_param = "--platform=linux" # With LCOW, Docker doesn't respect USER directive in dockerfile yet self.lcow_user_workaround = "sudo su conan && " self.exclude_vcvars_precommand = ( exclude_vcvars_precommand or os.getenv("CONAN_EXCLUDE_VCVARS_PRECOMMAND", False)) self._docker_image_skip_update = ( docker_image_skip_update or os.getenv("CONAN_DOCKER_IMAGE_SKIP_UPDATE", False)) self._docker_image_skip_pull = (docker_image_skip_pull or os.getenv( "CONAN_DOCKER_IMAGE_SKIP_PULL", False)) self.runner = runner or os.system self.output_runner = ConanOutputRunner() self.args = " ".join(args) if args else " ".join(sys.argv[1:]) # Upload related variables self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3) if upload_only_when_stable is not None: self.upload_only_when_stable = upload_only_when_stable else: self.upload_only_when_stable = get_bool_from_env( "CONAN_UPLOAD_ONLY_WHEN_STABLE") self.docker_entry_script = docker_entry_script or os.getenv( "CONAN_DOCKER_ENTRY_SCRIPT") os.environ["CONAN_CHANNEL"] = self.channel # If CONAN_DOCKER_IMAGE is speified, then use docker is True self.use_docker = use_docker or os.getenv("CONAN_USE_DOCKER", False) or \ (os.getenv("CONAN_DOCKER_IMAGE", None) is not None) if docker_32_images is not None: self.docker_32_images = docker_32_images else: self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False) self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1) self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1) self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE", None) self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE", "conan==%s" % client_version) if self.conan_pip_package in ("0", "False"): self.conan_pip_package = False self.vs10_x86_64_enabled = vs10_x86_64_enabled self.builds_in_current_page = [] self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER", None) def valid_pair(var, value): return (isinstance(value, six.string_types) or isinstance(value, bool) or isinstance(value, list) ) and not var.startswith("_") and "password" not in var with self.printer.foldable_output("local_vars"): self.printer.print_dict({ var: value for var, value in self.__dict__.items() if valid_pair(var, value) })
class ConanMultiPackager(object): """ Help to generate common builds (setting's combinations), adjust the environment, and run conan create command in docker containers""" def __init__(self, args=None, username=None, channel=None, runner=None, gcc_versions=None, visual_versions=None, visual_runtimes=None, apple_clang_versions=None, archs=None, use_docker=None, curpage=None, total_pages=None, docker_image=None, reference=None, password=None, remotes=None, upload=None, stable_branch_pattern=None, vs10_x86_64_enabled=False, mingw_configurations=None, stable_channel=None, platform_info=None, upload_retry=None, clang_versions=None, login_username=None, upload_only_when_stable=None, build_types=None, skip_check_credentials=False, allow_gcc_minors=False, exclude_vcvars_precommand=False, docker_image_skip_update=False, docker_image_skip_pull=False, docker_entry_script=None, docker_32_images=None, build_policy=None, always_update_conan_in_docker=False, conan_api=None, client_cache=None, ci_manager=None, out=None, test_folder=None): self.printer = Printer(out) self.printer.print_rule() self.printer.print_ascci_art() if not conan_api: self.conan_api, self.client_cache, _ = Conan.factory() else: self.conan_api = conan_api self.client_cache = client_cache self.ci_manager = ci_manager or CIManager(self.printer) self.remotes_manager = RemotesManager(self.conan_api, self.printer, remotes, upload) self.username = username or os.getenv("CONAN_USERNAME", None) if not self.username: raise Exception( "Instance ConanMultiPackage with 'username' parameter or use " "CONAN_USERNAME env variable") self.skip_check_credentials = skip_check_credentials or \ os.getenv("CONAN_SKIP_CHECK_CREDENTIALS", False) self.auth_manager = AuthManager( self.conan_api, self.printer, login_username, password, default_username=self.username, skip_check_credentials=self.skip_check_credentials) self.uploader = Uploader(self.conan_api, self.remotes_manager, self.auth_manager, self.printer) self._builds = [] self._named_builds = {} self._update_conan_in_docker = ( always_update_conan_in_docker or os.getenv("CONAN_ALWAYS_UPDATE_CONAN_DOCKER", False)) self._platform_info = platform_info or PlatformInfo() self.stable_branch_pattern = stable_branch_pattern or \ os.getenv("CONAN_STABLE_BRANCH_PATTERN", None) self.specified_channel = channel or os.getenv("CONAN_CHANNEL", "testing") self.specified_channel = self.specified_channel.rstrip() self.stable_channel = stable_channel or os.getenv( "CONAN_STABLE_CHANNEL", "stable") self.stable_channel = self.stable_channel.rstrip() self.channel = self._get_channel(self.specified_channel, self.stable_channel) self.partial_reference = reference or os.getenv( "CONAN_REFERENCE", None) if self.partial_reference: if "@" in self.partial_reference: self.reference = ConanFileReference.loads( self.partial_reference) else: name, version = self.partial_reference.split("/") self.reference = ConanFileReference(name, version, self.username, self.channel) else: if not os.path.exists("conanfile.py"): raise Exception( "Conanfile not found, specify a 'reference' parameter with name and version" ) conanfile = load_conanfile_class("./conanfile.py") name, version = conanfile.name, conanfile.version if name and version: self.reference = ConanFileReference(name, version, self.username, self.channel) else: self.reference = None # If CONAN_DOCKER_IMAGE is speified, then use docker is True self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False) or os.getenv("CONAN_DOCKER_IMAGE", None) is not None) os_name = self._platform_info.system( ) if not self.use_docker else "Linux" self.build_generator = BuildGenerator( reference, os_name, gcc_versions, apple_clang_versions, clang_versions, visual_versions, visual_runtimes, vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors, build_types) build_policy = (build_policy or self.ci_manager.get_commit_build_policy() or os.getenv("CONAN_BUILD_POLICY", None)) if build_policy: if build_policy.lower() not in ("never", "outdated", "missing"): raise Exception( "Invalid build policy, valid values: never, outdated, missing" ) self.build_policy = build_policy self.sudo_docker_command = "" if "CONAN_DOCKER_USE_SUDO" in os.environ: self.sudo_docker_command = "sudo -E" if get_bool_from_env( "CONAN_DOCKER_USE_SUDO") else "" elif platform.system() != "Windows": self.sudo_docker_command = "sudo -E" self.sudo_pip_command = "" if "CONAN_PIP_USE_SUDO" in os.environ: self.sudo_pip_command = "sudo -E" if get_bool_from_env( "CONAN_PIP_USE_SUDO") else "" elif platform.system() != "Windows": self.sudo_pip_command = "sudo -E" self.docker_shell = "" self.docker_conan_home = "" if self.is_wcow: self.docker_conan_home = "C:/Users/ContainerAdministrator" self.docker_shell = "cmd /C" else: self.docker_conan_home = "/home/conan" self.docker_shell = "/bin/sh -c" self.docker_platform_param = "" self.lcow_user_workaround = "" if self.is_lcow: self.docker_platform_param = "--platform=linux" # With LCOW, Docker doesn't respect USER directive in dockerfile yet self.lcow_user_workaround = "sudo su conan && " self.exclude_vcvars_precommand = ( exclude_vcvars_precommand or os.getenv("CONAN_EXCLUDE_VCVARS_PRECOMMAND", False)) self._docker_image_skip_update = ( docker_image_skip_update or os.getenv("CONAN_DOCKER_IMAGE_SKIP_UPDATE", False)) self._docker_image_skip_pull = (docker_image_skip_pull or os.getenv( "CONAN_DOCKER_IMAGE_SKIP_PULL", False)) self.runner = runner or os.system self.output_runner = ConanOutputRunner() self.args = " ".join(args) if args else " ".join(sys.argv[1:]) # Upload related variables self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3) if upload_only_when_stable is not None: self.upload_only_when_stable = upload_only_when_stable else: self.upload_only_when_stable = get_bool_from_env( "CONAN_UPLOAD_ONLY_WHEN_STABLE") self.docker_entry_script = docker_entry_script or os.getenv( "CONAN_DOCKER_ENTRY_SCRIPT") os.environ["CONAN_CHANNEL"] = self.channel # If CONAN_DOCKER_IMAGE is speified, then use docker is True self.use_docker = use_docker or os.getenv("CONAN_USE_DOCKER", False) or \ (os.getenv("CONAN_DOCKER_IMAGE", None) is not None) if docker_32_images is not None: self.docker_32_images = docker_32_images else: self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False) self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1) self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1) self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE", None) self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE", "conan==%s" % client_version) if self.conan_pip_package in ("0", "False"): self.conan_pip_package = False self.vs10_x86_64_enabled = vs10_x86_64_enabled self.builds_in_current_page = [] self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER", None) def valid_pair(var, value): return (isinstance(value, six.string_types) or isinstance(value, bool) or isinstance(value, list) ) and not var.startswith("_") and "password" not in var with self.printer.foldable_output("local_vars"): self.printer.print_dict({ var: value for var, value in self.__dict__.items() if valid_pair(var, value) }) # For Docker on Windows, including Linux containers on Windows @property def is_lcow(self): return self.container_os == "linux" and platform.system() == "Windows" @property def is_wcow(self): return self.container_os == "windows" and platform.system( ) == "Windows" @property def container_os(self): # CONAN_DOCKER_PLATFORM=linux must be specified for LCOW if self.use_docker: if "CONAN_DOCKER_PLATFORM" in os.environ: return os.getenv("CONAN_DOCKER_PLATFORM", "windows").lower() else: return "windows" else: return "" @property def items(self): return self._builds @items.setter def items(self, confs): self.builds = confs @property def builds(self): # Retrocompatibility iterating self.printer.print_message( "WARNING", "\n\n\n******* ITERATING THE CONAN_PACKAGE_TOOLS BUILDS WITH " ".builds is deprecated use .items() instead (unpack 5 elements: " "settings, options, env_vars, build_requires, reference *******" "**\n\n\n") return [elem[0:4] for elem in self._builds] @builds.setter def builds(self, confs): """For retro compatibility directly assigning builds""" self._named_builds = {} self._builds = [] for values in confs: if len(values) == 2: self._builds.append( BuildConf(values[0], values[1], {}, {}, self.reference)) elif len(values) == 4: self._builds.append( BuildConf(values[0], values[1], values[2], values[3], self.reference)) elif len(values) != 5: raise Exception( "Invalid build configuration, has to be a tuple of " "(settings, options, env_vars, build_requires, reference)") else: self._builds.append(BuildConf(*values)) @property def named_builds(self): return self._named_builds @named_builds.setter def named_builds(self, confs): self._builds = [] self._named_builds = {} for key, pages in confs.items(): for values in pages: if len(values) == 2: bc = BuildConf(values[0], values[1], {}, {}, self.reference) self._named_builds.setdefault(key, []).append(bc) elif len(values) == 4: bc = BuildConf(values[0], values[1], values[2], values[3], self.reference) self._named_builds.setdefault(key, []).append(bc) elif len(values) != 5: raise Exception( "Invalid build configuration, has to be a tuple of " "(settings, options, env_vars, build_requires, reference)" ) else: self._named_builds.setdefault(key, []).append( BuildConf(*values)) def add_common_builds(self, shared_option_name=None, pure_c=True, dll_with_static_runtime=False, reference=None): if not reference and not self.reference: raise Exception( "Specify a CONAN_REFERENCE or name and version fields in the recipe" ) if shared_option_name is None: if os.path.exists("conanfile.py"): conanfile = load_conanfile_class("./conanfile.py") if hasattr(conanfile, "options") and "shared" in conanfile.options: shared_option_name = "%s:shared" % self.reference.name tmp = self.build_generator.get_builds(pure_c, shared_option_name, dll_with_static_runtime, reference or self.reference) self._builds.extend(tmp) def add(self, settings=None, options=None, env_vars=None, build_requires=None, reference=None): settings = settings or {} options = options or {} env_vars = env_vars or {} build_requires = build_requires or {} if reference: reference = ConanFileReference.loads( "%s@%s/%s" % (reference, self.username, self.channel)) reference = reference or self.reference self._builds.append( BuildConf(settings, options, env_vars, build_requires, reference)) def run(self, base_profile_name=None): env_vars = self.auth_manager.env_vars() env_vars.update(self.remotes_manager.env_vars()) with tools.environment_append(env_vars): self.printer.print_message("Running builds...") if self.ci_manager.skip_builds(): print("Skipped builds due [skip ci] commit message") return 99 if not self.skip_check_credentials and self._upload_enabled(): self.remotes_manager.add_remotes_to_conan() self.auth_manager.login( self.remotes_manager.upload_remote_name) if self.conan_pip_package and not self.use_docker: with self.printer.foldable_output("pip_update"): self.runner( '%s pip install %s' % (self.sudo_pip_command, self.conan_pip_package)) self.run_builds(base_profile_name=base_profile_name) def _upload_enabled(self): if not self.remotes_manager.upload_remote_name: return False if not self.auth_manager.credentials_ready( self.remotes_manager.upload_remote_name): return False st_channel = self.stable_channel or "stable" if self.upload_only_when_stable and self.channel != st_channel: self.printer.print_message("Skipping upload, not stable channel") return False if not os.getenv("CONAN_TEST_SUITE", False): if self.ci_manager.is_pull_request(): # PENDING! can't found info for gitlab/bamboo self.printer.print_message( "Skipping upload, this is a Pull Request") return False def raise_error(field): raise Exception("Upload not possible, '%s' is missing!" % field) if not self.channel: raise_error("channel") if not self.username: raise_error("username") return True def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): if len(self.named_builds) > 0 and len(self.items) > 0: raise Exception( "Both bulk and named builds are set. Only one is allowed.") self.builds_in_current_page = [] if len(self.items) > 0: curpage = curpage or int(self.curpage) total_pages = total_pages or int(self.total_pages) for index, build in enumerate(self.items): if curpage is None or total_pages is None or ( index % total_pages) + 1 == curpage: self.builds_in_current_page.append(build) elif len(self.named_builds) > 0: curpage = curpage or self.curpage if curpage not in self.named_builds: raise Exception("No builds set for page %s" % curpage) for build in self.named_builds[curpage]: self.builds_in_current_page.append(build) self.printer.print_current_page(curpage, total_pages) self.printer.print_jobs(self.builds_in_current_page) pulled_docker_images = defaultdict(lambda: False) # FIXME: Remove in Conan 1.3, https://github.com/conan-io/conan/issues/2787 abs_folder = os.path.realpath(os.getcwd()) for build in self.builds_in_current_page: base_profile_name = base_profile_name or os.getenv( "CONAN_BASE_PROFILE") if base_profile_name: self.printer.print_message( "**************************************************") self.printer.print_message("Using specified default " "base profile: %s" % base_profile_name) self.printer.print_message( "**************************************************") profile_text, base_profile_text = get_profiles( self.client_cache, build, base_profile_name) if not self.use_docker: profile_abs_path = save_profile_to_tmp(profile_text) r = CreateRunner( profile_abs_path, build.reference, self.conan_api, self.uploader, args=self.args, exclude_vcvars_precommand=self.exclude_vcvars_precommand, build_policy=self.build_policy, runner=self.runner, abs_folder=abs_folder, printer=self.printer, upload=self._upload_enabled(), test_folder=self.test_folder) r.run() else: docker_image = self._get_docker_image(build) r = DockerCreateRunner( profile_text, base_profile_text, base_profile_name, build.reference, args=self.args, conan_pip_package=self.conan_pip_package, docker_image=docker_image, sudo_docker_command=self.sudo_docker_command, sudo_pip_command=self.sudo_pip_command, docker_image_skip_update=self._docker_image_skip_update, docker_image_skip_pull=self._docker_image_skip_pull, build_policy=self.build_policy, always_update_conan_in_docker=self._update_conan_in_docker, upload=self._upload_enabled(), runner=self.runner, docker_shell=self.docker_shell, docker_conan_home=self.docker_conan_home, docker_platform_param=self.docker_platform_param, lcow_user_workaround=self.lcow_user_workaround, test_folder=self.test_folder) r.run(pull_image=not pulled_docker_images[docker_image], docker_entry_script=self.docker_entry_script) pulled_docker_images[docker_image] = True def _get_docker_image(self, build): if self._docker_image: docker_image = self._docker_image else: compiler_name = build.settings.get("compiler", "") compiler_version = build.settings.get("compiler.version", "") docker_image = self._autodetect_docker_base_image( compiler_name, compiler_version) arch = build.settings.get("arch", "") or build.settings.get( "arch_build", "") if self.docker_32_images and arch == "x86": build.settings["arch_build"] = "x86" docker_arch_suffix = "x86" elif arch != "x86" and arch != "x86_64": docker_arch_suffix = arch else: docker_arch_suffix = None if docker_arch_suffix and "-" not in docker_image: docker_image = "%s-%s" % (docker_image, docker_arch_suffix) return docker_image @staticmethod def _autodetect_docker_base_image(compiler_name, compiler_version): if compiler_name not in ["clang", "gcc"]: raise Exception("Docker image cannot be autodetected for " "the compiler %s" % compiler_name) if compiler_name == "gcc" and Version(compiler_version) > Version("5"): compiler_version = Version(compiler_version).major(fill=False) return "lasote/conan%s%s" % (compiler_name, compiler_version.replace(".", "")) def _get_channel(self, specified_channel, stable_channel): if self.stable_branch_pattern: stable_patterns = [self.stable_branch_pattern] else: stable_patterns = ["master$", "release*", "stable*"] branch = self.ci_manager.get_branch() self.printer.print_message("Branch detected", branch) for pattern in stable_patterns: prog = re.compile(pattern) if branch and prog.match(branch): self.printer.print_message( "Info", "Redefined channel by CI branch matching with '%s', " "setting CONAN_CHANNEL to '%s'" % (pattern, stable_channel)) return stable_channel return specified_channel
def setUp(self): self.conan_api = MockConanAPI() self.output = TestBufferConanOutput() self.printer = Printer(self.output.write)
def test_valid_config_with_args(self): manager = ConfigManager(self.conan_api, Printer()) manager.install('https://github.com/bincrafters/conan-config.git', '-b master')