def __init__(self, profile_text, base_profile_text, base_profile_name, reference,
args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=None,
sudo_pip_command=True,
docker_image_skip_update=False, build_policy=None,
docker_image_skip_pull=False,
always_update_conan_in_docker=False,
upload=False, runner=None,
docker_shell=None, docker_conan_home=None):
self.printer = Printer()
self._args = args
self._upload = upload
self._reference = reference
self._conan_pip_package = conan_pip_package
self._build_policy = build_policy
self._docker_image = docker_image
self._always_update_conan_in_docker = always_update_conan_in_docker
self._docker_image_skip_update = docker_image_skip_update
self._docker_image_skip_pull = docker_image_skip_pull
self._sudo_docker_command = sudo_docker_command or ""
self._sudo_pip_command = sudo_pip_command
self._profile_text = profile_text
self._base_profile_text = base_profile_text
self._base_profile_name = base_profile_name
self._docker_shell = docker_shell
self._docker_conan_home = docker_conan_home
self._runner = PrintRunner(runner, self.printer)
def __init__(self, profile_abs_path, reference, conan_api, uploader,
exclude_vcvars_precommand=False, build_policy=None, runner=None,
cwd=None, printer=None, upload=False, test_folder=None, config_url=None):
self.printer = printer or Printer()
self._cwd = cwd or os.getcwd()
self._uploader = uploader
self._upload = upload
self._conan_api = conan_api
self._profile_abs_path = profile_abs_path
self._reference = reference
self._exclude_vcvars_precommand = exclude_vcvars_precommand
self._build_policy = build_policy
self._runner = PrintRunner(runner or os.system, self.printer)
self._uploader.remote_manager.add_remotes_to_conan()
self._test_folder = test_folder
self._config_url = config_url
patch_default_base_profile(conan_api, profile_abs_path)
if Version(client_version) < Version("1.12.0"):
cache = self._conan_api._client_cache
else:
cache = self._conan_api._cache
self._profile = load_profile(profile_abs_path, cache)
def run():
conan_version = get_client_version()
if conan_version < Version("1.18.0"):
conan_api, client_cache, _ = Conan.factory()
else:
conan_api, _, _ = Conan.factory()
conan_api.create_app()
client_cache = conan_api.app.cache
printer = Printer()
remotes_manager = RemotesManager(conan_api, printer)
remotes_manager.add_remotes_to_conan()
default_username = os.getenv("CONAN_USERNAME")
auth_manager = AuthManager(conan_api,
printer,
default_username=default_username)
upload_retry = os.getenv("CPT_UPLOAD_RETRY")
upload_only_recipe = os.getenv("CPT_UPLOAD_ONLY_RECIPE")
upload_force = os.getenv("CPT_UPLOAD_FORCE")
uploader = Uploader(conan_api, remotes_manager, auth_manager, printer,
upload_retry, upload_force)
build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY"))
test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER"))
reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE"))
profile_text = unscape_env(os.getenv("CPT_PROFILE"))
abs_profile_path = save_profile_to_tmp(profile_text)
base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE"))
config_url = unscape_env(os.getenv("CPT_CONFIG_URL"))
config_args = unscape_env(os.getenv("CPT_CONFIG_ARGS"))
upload_dependencies = unscape_env(os.getenv("CPT_UPLOAD_DEPENDENCIES"))
update_dependencies = unscape_env(os.getenv("CPT_UPDATE_DEPENDENCIES"))
conanfile = unscape_env(os.getenv("CPT_CONANFILE"))
lockfile = unscape_env(os.getenv("CPT_LOCKFILE"))
skip_recipe_export = unscape_env(os.getenv("CPT_SKIP_RECIPE_EXPORT"))
if base_profile_text:
base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME"))
tools.save(os.path.join(client_cache.profiles_path, base_profile_name),
base_profile_text)
upload = os.getenv("CPT_UPLOAD_ENABLED")
runner = CreateRunner(abs_profile_path,
reference,
conan_api,
uploader,
build_policy=build_policy,
printer=printer,
upload=upload,
upload_only_recipe=upload_only_recipe,
test_folder=test_folder,
config_url=config_url,
config_args=config_args,
upload_dependencies=upload_dependencies,
conanfile=conanfile,
skip_recipe_export=skip_recipe_export,
update_dependencies=update_dependencies,
lockfile=lockfile)
runner.run()
def login(self):
if self.variables.docker_upload_only_when_stable:
printer = Printer()
ci_manager = CIManager(printer)
if ci_manager.get_branch() != "master" or ci_manager.is_pull_request():
logging.info("Skipped login, is not stable branch")
return
if not self.variables.docker_upload:
logging.info("Skipped login, DOCKER_UPLOAD is not activated")
return
if not self.variables.docker_password:
logging.warning("Skipped login, DOCKER_PASSWORD is missing!")
return
if not self.variables.docker_login_username:
logging.warning("Skipped login, DOCKER_LOGIN_USERNAME is missing!")
return
logging.info("Login to Docker hub account")
result = subprocess.call([
"docker", "login", "-u", self.variables.docker_login_username, "-p",
self.variables.docker_password
])
if result != os.EX_OK:
raise RuntimeError("Could not login username %s "
"to Docker hub." % self.variables.docker_login_username)
logging.info("Logged in Docker hub account with success")
self.loggedin = True
def run():
# Get all from environ
conan_api, client_cache, _ = Conan.factory()
printer = Printer()
if os.path.exists(client_cache.default_profile_path):
os.remove(client_cache.default_profile_path)
remotes_manager = RemotesManager(conan_api, printer)
default_username = os.getenv("CONAN_USERNAME", None)
auth_manager = AuthManager(conan_api, printer, default_username=default_username)
uploader = Uploader(conan_api, remotes_manager, auth_manager, printer)
args = os.getenv("CPT_ARGS", "")
build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY"))
reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE"))
profile_text = unscape_env(os.getenv("CPT_PROFILE"))
abs_profile_path = save_profile_to_tmp(profile_text)
base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE"))
if base_profile_text:
base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME"))
tools.save(os.path.join(client_cache.profiles_path, base_profile_name),
base_profile_text)
upload = os.getenv("CPT_UPLOAD_ENABLED", None)
runner = CreateRunner(abs_profile_path, reference, conan_api, uploader,
args=args,
build_policy=build_policy, printer=printer, upload=upload)
runner.run()
def __init__(self,
profile_abs_path,
reference,
conan_api,
uploader,
exclude_vcvars_precommand=False,
build_policy=None,
runner=None,
cwd=None,
printer=None,
upload=False,
upload_only_recipe=None,
test_folder=None,
config_url=None,
config_args=None,
upload_dependencies=None,
conanfile=None,
skip_recipe_export=False,
update_dependencies=False,
lockfile=None):
self.printer = printer or Printer()
self._cwd = cwd or os.getcwd()
self._uploader = uploader
self._upload = upload
self._conan_api = conan_api
self._profile_abs_path = profile_abs_path
self._reference = reference
self._exclude_vcvars_precommand = exclude_vcvars_precommand
self._build_policy = build_policy.split(",") if \
isinstance(build_policy, str) else \
build_policy
self._runner = PrintRunner(runner or os.system, self.printer)
self._test_folder = test_folder
self._config_url = config_url
self._config_args = config_args
self._upload_only_recipe = upload_only_recipe
self._conanfile = conanfile
self._lockfile = lockfile
self._upload_dependencies = upload_dependencies.split(",") if \
isinstance(upload_dependencies, str) else \
upload_dependencies
self._upload_dependencies = self._upload_dependencies or []
self.skip_recipe_export = skip_recipe_export
self._update_dependencies = update_dependencies
self._results = None
patch_default_base_profile(conan_api, profile_abs_path)
client_version = get_client_version()
if client_version < Version("1.12.0"):
cache = self._conan_api._client_cache
elif client_version < Version("1.18.0"):
cache = self._conan_api._cache
else:
if not conan_api.app:
conan_api.create_app()
cache = conan_api.app.cache
self._profile = load_profile(profile_abs_path, cache)
def run():
# Get all from environ
conan_api, client_cache, _ = Conan.factory()
printer = Printer()
remotes_manager = RemotesManager(conan_api, printer)
remotes_manager.add_remotes_to_conan()
default_username = os.getenv("CONAN_USERNAME")
auth_manager = AuthManager(conan_api, printer, default_username=default_username)
upload_retry = os.getenv("CPT_UPLOAD_RETRY")
upload_only_recipe = os.getenv("CPT_UPLOAD_ONLY_RECIPE")
uploader = Uploader(conan_api, remotes_manager, auth_manager, printer, upload_retry)
build_policy = unscape_env(os.getenv("CPT_BUILD_POLICY"))
test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER"))
reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE"))
profile_text = unscape_env(os.getenv("CPT_PROFILE"))
abs_profile_path = save_profile_to_tmp(profile_text)
base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE"))
config_url = unscape_env(os.getenv("CPT_CONFIG_URL"))
upload_dependencies = unscape_env(os.getenv("CPT_UPLOAD_DEPENDENCIES"))
conanfile = unscape_env(os.getenv("CPT_CONANFILE"))
if base_profile_text:
base_profile_name = unscape_env(os.getenv("CPT_BASE_PROFILE_NAME"))
tools.save(os.path.join(client_cache.profiles_path, base_profile_name),
base_profile_text)
upload = os.getenv("CPT_UPLOAD_ENABLED")
runner = CreateRunner(abs_profile_path, reference, conan_api, uploader,
build_policy=build_policy, printer=printer, upload=upload,
upload_only_recipe=upload_only_recipe,
test_folder=test_folder, config_url=config_url,
upload_dependencies=upload_dependencies, conanfile=conanfile)
runner.run()
def __init__(self,
profile_abs_path,
reference,
conan_api,
uploader,
args=None,
exclude_vcvars_precommand=False,
build_policy=None,
runner=None,
abs_folder=None,
printer=None,
upload=False):
self.printer = printer or Printer()
self._abs_folder = abs_folder or os.getcwd()
self._uploader = uploader
self._upload = upload
self._conan_api = conan_api
self._profile_abs_path = profile_abs_path
self._reference = reference
self._args = args
self._exclude_vcvars_precommand = exclude_vcvars_precommand
self._build_policy = build_policy
self._runner = PrintRunner(runner or os.system, self.printer)
self._uploader.remote_manager.add_remotes_to_conan()
patch_default_base_profile(conan_api, profile_abs_path)
self._profile = load_profile(profile_abs_path,
self._conan_api._client_cache)
def __init__(self,
profile_text,
base_profile_text,
base_profile_name,
reference,
conan_pip_package=None,
docker_image=None,
sudo_docker_command=None,
sudo_pip_command=False,
docker_image_skip_update=False,
build_policy=None,
docker_image_skip_pull=False,
always_update_conan_in_docker=False,
upload=False,
upload_retry=None,
runner=None,
docker_shell="",
docker_conan_home="",
docker_platform_param="",
lcow_user_workaround="",
test_folder=None,
pip_install=None,
config_url=None):
self.printer = Printer()
self._upload = upload
self._upload_retry = upload_retry
self._reference = reference
self._conan_pip_package = conan_pip_package
self._build_policy = build_policy
self._docker_image = docker_image
self._always_update_conan_in_docker = always_update_conan_in_docker
self._docker_image_skip_update = docker_image_skip_update
self._docker_image_skip_pull = docker_image_skip_pull
self._sudo_docker_command = sudo_docker_command or ""
self._sudo_pip_command = sudo_pip_command
self._profile_text = profile_text
self._base_profile_text = base_profile_text
self._base_profile_name = base_profile_name
self._docker_shell = docker_shell
self._docker_conan_home = docker_conan_home
self._docker_platform_param = docker_platform_param
self._lcow_user_workaround = lcow_user_workaround
self._runner = PrintRunner(runner, self.printer)
self._test_folder = test_folder
self._pip_install = pip_install
self._config_url = config_url
def __init__(self, profile_text, base_profile_text, base_profile_name, reference,
conan_pip_package=None, docker_image=None, sudo_docker_command=None,
sudo_pip_command=False,
docker_image_skip_update=False, build_policy=None,
docker_image_skip_pull=False,
always_update_conan_in_docker=False,
upload=False, upload_retry=None, upload_only_recipe=None,
runner=None,
docker_shell="", docker_conan_home="",
docker_platform_param="", docker_run_options="",
lcow_user_workaround="",
test_folder=None,
pip_install=None,
docker_pip_command=None,
config_url=None,
config_args=None,
printer=None,
upload_dependencies=None,
conanfile=None,
force_selinux=None,
skip_recipe_export=False,
update_dependencies=False,
lockfile=None):
self.printer = printer or Printer()
self._upload = upload
self._upload_retry = upload_retry
self._upload_only_recipe = upload_only_recipe
self._reference = reference
self._conan_pip_package = conan_pip_package
self._build_policy = build_policy
self._docker_image = docker_image
self._always_update_conan_in_docker = always_update_conan_in_docker
self._docker_image_skip_update = docker_image_skip_update
self._docker_image_skip_pull = docker_image_skip_pull
self._sudo_docker_command = sudo_docker_command or ""
self._sudo_pip_command = sudo_pip_command
self._profile_text = profile_text
self._base_profile_text = base_profile_text
self._base_profile_name = base_profile_name
self._docker_shell = docker_shell
self._docker_conan_home = docker_conan_home
self._docker_platform_param = docker_platform_param
self._docker_run_options = docker_run_options or ""
self._lcow_user_workaround = lcow_user_workaround
self._runner = PrintRunner(runner, self.printer)
self._test_folder = test_folder
self._pip_install = pip_install
self._docker_pip_command = docker_pip_command
self._config_url = config_url
self._config_args = config_args
self._upload_dependencies = upload_dependencies or []
self._conanfile = conanfile
self._lockfile = lockfile
self._force_selinux = force_selinux
self._skip_recipe_export = skip_recipe_export
self._update_dependencies = update_dependencies
def test_plain(self):
manager = RemotesManager(self.conan_api,
Printer(),
remotes_input="url1, url",
upload_input="url1")
expected = {
'CONAN_REMOTES': 'url1@True@remote0,url@True@remote1',
'CONAN_UPLOAD': 'url1@True@upload_repo'
}
self.assert_serial_deserial(manager, expected)
def test_valid_config(self):
manager = ConfigManager(self.api, Printer())
profiles = self.api.profile_list()
self.assertEquals(len(profiles), 0)
manager.install("https://github.com/bincrafters/bincrafters-config.git", "-b main")
profiles = self.api.profile_list()
self.assertGreater(len(profiles), 3)
def test_invalid_config(self):
manager = ConfigManager(self.api, Printer())
profiles = self.api.profile_list()
self.assertEquals(len(profiles), 0)
try:
manager.install("https://github.com/")
self.fail("Could not accept wrong URL")
except ConanException:
pass
def test_list(self):
remotes_input = [("url1", True, "remote1"), ("url2", False, "remote2")]
upload_input = ("url3", True, "remote1")
manager = RemotesManager(self.conan_api,
Printer(),
remotes_input=remotes_input,
upload_input=upload_input)
expected = {
'CONAN_REMOTES': 'url1@True@remote1,url2@False@remote2',
'CONAN_UPLOAD': 'url3@True@remote1'
}
self.assert_serial_deserial(manager, expected)
def setUp(self):
# Clean env first
new_env = {}
for var, value in os.environ.items():
if "travis" not in var.lower() and \
"appveyor" not in var.lower() and \
"bamboo" not in var.lower():
new_env[var] = value
os.environ = new_env
self.output = TestBufferConanOutput()
self.printer = Printer(self.output.write)
def test_duplicated_remotes_with_same_url(self):
self.api.remote_remove("conancenter")
self.api.remote_add("upload_repo", "url1", True)
manager = RemotesManager(self.api, Printer(), remotes_input="url1@True@upload_repo",
upload_input="url1@True@upload_repo")
remotes = self.api.remote_list()
self.assertIsNotNone(manager._get_remote_by_name(remotes, "upload_repo"))
manager.add_remotes_to_conan()
self.assertEquals(len(self.api.remote_list()), 1)
def get_version_from_ci():
printer = Printer(hidesensitive)
ci_man = CIManager(printer)
if ci_man.is_tag():
try:
version = subprocess.check_output(
"git describe --exact-match --tags",
shell=True).decode().strip()
except Exception:
return None
else:
version = ci_man.get_branch()
version = re.sub("^refs/tags/", "", version)
version = re.sub("^.*/v?", "", version)
version = re.sub("^v?", "", version)
return version
def upload(self):
""" Set taocpp repository to be used on upload.
The upload server address could be customized by env var
CONAN_UPLOAD. If not defined, the method will check the branch name.
Only master or CONAN_STABLE_BRANCH_PATTERN will be accepted.
The master branch will be pushed to testing channel, because it does
not match the stable pattern. Otherwise it will upload to stable
channel.
"""
if os.getenv("CONAN_UPLOAD", None) is not None:
return os.getenv("CONAN_UPLOAD")
printer = Printer(None)
ci_manager = CIManager(printer)
branch = ci_manager.get_branch()
patterns = [r"v?\d+\.\d+\.\d+-.*", self.stable_branch_pattern]
for pattern in patterns:
prog = re.compile(pattern)
if branch and prog.match(branch):
return "https://api.bintray.com/conan/taocpp/public-conan"
#!/usr/bin/env python3
import argparse
import errno
import tempfile
from pathlib import Path
import yaml
from cpt.printer import Printer
printer = Printer()
from ci_tools_gha.utils import get_builder_default, is_pure_c
gha_hack = True
import os
import shutil
def copytree(src, dst, symlinks=False, ignore=None):
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
if os.path.isdir(s):
shutil.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
def gha_hack_copy(src):
def _branch(self):
""" Get branch name from CI manager
"""
printer = Printer(None)
ci_manager = CIManager(printer)
return ci_manager.get_branch()
class DockerCreateRunner(object):
def __init__(self, profile_text, base_profile_text, base_profile_name, reference,
args=None, conan_pip_package=None, docker_image=None, sudo_docker_command=None,
sudo_pip_command=True,
docker_image_skip_update=False, build_policy=None,
always_update_conan_in_docker=False,
upload=False, runner=None):
self.printer = Printer()
self._args = args
self._upload = upload
self._reference = reference
self._conan_pip_package = conan_pip_package
self._build_policy = build_policy
self._docker_image = docker_image
self._always_update_conan_in_docker = always_update_conan_in_docker
self._docker_image_skip_update = docker_image_skip_update
self._sudo_docker_command = sudo_docker_command or ""
self._sudo_pip_command = sudo_pip_command
self._profile_text = profile_text
self._base_profile_text = base_profile_text
self._base_profile_name = base_profile_name
self._runner = PrintRunner(runner, self.printer)
def _pip_update_conan_command(self):
commands = []
# Hack for testing when retriving cpt from artifactory repo
if "conan-package-tools" not in self._conan_pip_package:
commands.append("%s pip install conan_package_tools==%s "
"--upgrade --no-cache" % (self._sudo_pip_command,
package_tools_version))
if self._conan_pip_package:
commands.append("%s pip install %s --no-cache" % (self._sudo_pip_command,
self._conan_pip_package))
else:
commands.append("%s pip install conan --upgrade --no-cache" % self._sudo_pip_command)
command = " && ".join(commands)
return command
def run(self, pull_image=True, docker_entry_script=None):
if pull_image:
self.pull_image()
if not self._docker_image_skip_update and not self._always_update_conan_in_docker:
# Update the downloaded image
with self.printer.foldable_output("update conan"):
command = '%s docker run --name conan_runner ' \
' %s /bin/sh -c "%s"' % (self._sudo_docker_command,
self._docker_image,
self._pip_update_conan_command())
ret = self._runner(command)
if ret != 0:
raise Exception("Error updating the image: %s" % command)
# Save the image with the updated installed
# packages and remove the intermediate container
command = "%s docker commit conan_runner %s" % (self._sudo_docker_command,
self._docker_image)
ret = self._runner(command)
if ret != 0:
raise Exception("Error commiting the image: %s" % command)
command = "%s docker rm conan_runner" % self._sudo_docker_command
ret = self._runner(command)
if ret != 0:
raise Exception("Error updating the image: %s" % command)
# Run the build
envs = self.get_env_vars()
env_vars_text = " ".join(['-e %s="%s"' % (key, value)
for key, value in envs.items() if value])
if self._always_update_conan_in_docker:
update_command = self._pip_update_conan_command() + " && "
else:
update_command = ""
command = ("%s docker run --rm -v%s:/home/conan/project %s %s /bin/sh "
"-c \"cd project && "
"%s run_create_in_docker \"" % (self._sudo_docker_command, os.getcwd(),
env_vars_text, self._docker_image,
update_command))
# Push entry command before to build
if docker_entry_script:
command = command.replace("run_create_in_docker",
"%s && run_create_in_docker" % docker_entry_script)
self.printer.print_in_docker(self._docker_image)
ret = self._runner(command)
if ret != 0:
raise Exception("Error building: %s" % command)
self.printer.print_message("Exiting docker...")
def pull_image(self):
with self.printer.foldable_output("docker pull"):
ret = self._runner("%s docker pull %s" % (self._sudo_docker_command, self._docker_image))
if ret != 0:
raise Exception("Error pulling the image: %s" % self._docker_image)
def get_env_vars(self):
ret = {key: value for key, value in os.environ.items() if key.startswith("CONAN_") and
key != "CONAN_USER_HOME"}
ret["CPT_ARGS"] = escape_env(self._args)
ret["CONAN_REFERENCE"] = self._reference
ret["CPT_PROFILE"] = escape_env(self._profile_text)
ret["CPT_BASE_PROFILE"] = escape_env(self._base_profile_text)
ret["CPT_BASE_PROFILE_NAME"] = escape_env(self._base_profile_name)
ret["CONAN_USERNAME"] = escape_env(self._reference.user)
ret["CONAN_TEMP_TEST_FOLDER"] = "1" # test package folder to a temp one
ret["CPT_UPLOAD_ENABLED"] = self._upload
ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)
return ret
def __init__(self,
username=None,
channel=None,
runner=None,
gcc_versions=None,
visual_versions=None,
visual_runtimes=None,
visual_toolsets=None,
apple_clang_versions=None,
archs=None,
options=None,
use_docker=None,
curpage=None,
total_pages=None,
docker_image=None,
reference=None,
password=None,
remotes=None,
upload=None,
stable_branch_pattern=None,
vs10_x86_64_enabled=False,
mingw_configurations=None,
stable_channel=None,
platform_info=None,
upload_retry=None,
clang_versions=None,
login_username=None,
upload_only_when_stable=None,
upload_only_when_tag=None,
upload_only_recipe=None,
build_types=None,
cppstds=None,
skip_check_credentials=False,
allow_gcc_minors=False,
exclude_vcvars_precommand=False,
docker_run_options=None,
docker_image_skip_update=False,
docker_image_skip_pull=False,
docker_entry_script=None,
docker_32_images=None,
docker_conan_home=None,
docker_shell=None,
pip_install=None,
build_policy=None,
always_update_conan_in_docker=False,
conan_api=None,
client_cache=None,
conanfile=None,
ci_manager=None,
out=None,
test_folder=None,
cwd=None,
config_url=None,
config_args=None,
upload_dependencies=None,
force_selinux=None,
skip_recipe_export=False,
update_dependencies=None,
lockfile=None):
conan_version = get_client_version()
self.printer = Printer(out)
self.printer.print_rule()
self.printer.print_ascci_art()
self.cwd = cwd or os.getcwd()
if not conan_api:
self.conan_api, _, _ = Conan.factory()
self.conan_api.create_app()
self.client_cache = self.conan_api.app.cache
else:
self.conan_api = conan_api
self.client_cache = client_cache
self.ci_manager = ci_manager or CIManager(self.printer)
self.remotes_manager = RemotesManager(self.conan_api, self.printer,
remotes, upload)
self.username = username or os.getenv("CONAN_USERNAME", None)
self.skip_check_credentials = skip_check_credentials or get_bool_from_env(
"CONAN_SKIP_CHECK_CREDENTIALS")
self.auth_manager = AuthManager(
self.conan_api,
self.printer,
login_username,
password,
default_username=self.username,
skip_check_credentials=self.skip_check_credentials)
# Upload related variables
self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3)
if upload_only_when_stable is not None:
self.upload_only_when_stable = upload_only_when_stable
else:
self.upload_only_when_stable = get_bool_from_env(
"CONAN_UPLOAD_ONLY_WHEN_STABLE")
if upload_only_when_tag is not None:
self.upload_only_when_tag = upload_only_when_tag
else:
self.upload_only_when_tag = get_bool_from_env(
"CONAN_UPLOAD_ONLY_WHEN_TAG")
self.upload_only_recipe = upload_only_recipe or get_bool_from_env(
"CONAN_UPLOAD_ONLY_RECIPE")
self.remotes_manager.add_remotes_to_conan()
self.uploader = Uploader(self.conan_api, self.remotes_manager,
self.auth_manager, self.printer,
self.upload_retry)
self._builds = []
self._named_builds = {}
self._packages_summary = []
self._update_conan_in_docker = always_update_conan_in_docker or get_bool_from_env(
"CONAN_ALWAYS_UPDATE_CONAN_DOCKER")
self._platform_info = platform_info or PlatformInfo()
self.stable_branch_pattern = stable_branch_pattern or \
os.getenv("CONAN_STABLE_BRANCH_PATTERN", None)
self.stable_channel = stable_channel or os.getenv(
"CONAN_STABLE_CHANNEL", "stable")
self.stable_channel = self.stable_channel.rstrip()
self.partial_reference = reference or os.getenv(
"CONAN_REFERENCE", None)
self.channel = self._get_specified_channel(channel, reference)
self.conanfile = conanfile or os.getenv("CONAN_CONANFILE",
"conanfile.py")
if self.partial_reference:
if "@" in self.partial_reference:
self.reference = ConanFileReference.loads(
self.partial_reference)
else:
name, version = self.partial_reference.split("/")
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
if not os.path.exists(os.path.join(self.cwd, self.conanfile)):
raise Exception("Conanfile not found, specify a 'reference' "
"parameter with name and version")
conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile),
self.conan_api)
name, version = conanfile.name, conanfile.version
if name and version:
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
self.reference = None
self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE",
None)
# If CONAN_DOCKER_IMAGE is specified, then use docker is True
self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False)
or self._docker_image is not None)
self.docker_conan_home = docker_conan_home or os.getenv(
"CONAN_DOCKER_HOME", None)
os_name = self._platform_info.system(
) if not self.use_docker else "Linux"
self.build_generator = BuildGenerator(
reference, os_name, gcc_versions, apple_clang_versions,
clang_versions, visual_versions, visual_runtimes, visual_toolsets,
vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors,
build_types, options, cppstds)
self.build_policy = (build_policy
or self.ci_manager.get_commit_build_policy()
or split_colon_env("CONAN_BUILD_POLICY"))
if isinstance(self.build_policy, list):
self.build_policy = ",".join(self.build_policy)
self.sudo_docker_command = ""
if "CONAN_DOCKER_USE_SUDO" in os.environ:
self.sudo_docker_command = "sudo -E" if get_bool_from_env(
"CONAN_DOCKER_USE_SUDO") else ""
elif platform.system() != "Windows":
self.sudo_docker_command = "sudo -E"
self.sudo_pip_command = ""
if "CONAN_PIP_USE_SUDO" in os.environ:
self.sudo_pip_command = "sudo -E" if get_bool_from_env(
"CONAN_PIP_USE_SUDO") else ""
elif platform.system(
) != "Windows" and self._docker_image and 'conanio/' not in str(
self._docker_image):
self.sudo_pip_command = "sudo -E"
self.pip_command = os.getenv("CONAN_PIP_COMMAND", "pip")
pip_found = True if tools.os_info.is_windows else tools.which(
self.pip_command)
if not pip_found or not "pip" in self.pip_command:
raise Exception(
"CONAN_PIP_COMMAND: '{}' is not a valid pip command.".format(
self.pip_command))
self.docker_pip_command = os.getenv("CONAN_DOCKER_PIP_COMMAND", "pip")
self.docker_shell = docker_shell or os.getenv("CONAN_DOCKER_SHELL")
if self.is_wcow:
if self.docker_conan_home is None:
self.docker_conan_home = "C:/Users/ContainerAdministrator"
self.docker_shell = docker_shell or "cmd /C"
else:
if self.docker_conan_home is None:
self.docker_conan_home = "/home/conan"
self.docker_shell = docker_shell or "/bin/sh -c"
self.docker_platform_param = ""
self.lcow_user_workaround = ""
if self.is_lcow:
self.docker_platform_param = "--platform=linux"
# With LCOW, Docker doesn't respect USER directive in dockerfile yet
self.lcow_user_workaround = "sudo su conan && "
self.exclude_vcvars_precommand = exclude_vcvars_precommand or \
get_bool_from_env("CONAN_EXCLUDE_VCVARS_PRECOMMAND")
self._docker_image_skip_update = docker_image_skip_update or \
get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_UPDATE")
self._docker_image_skip_pull = docker_image_skip_pull or \
get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_PULL")
self.runner = runner or os.system
self.output_runner = ConanOutputRunner()
self.docker_run_options = docker_run_options or split_colon_env(
"CONAN_DOCKER_RUN_OPTIONS")
if isinstance(self.docker_run_options, list):
self.docker_run_options = " ".join(self.docker_run_options)
self.docker_entry_script = docker_entry_script or os.getenv(
"CONAN_DOCKER_ENTRY_SCRIPT")
self.pip_install = pip_install or split_colon_env("CONAN_PIP_INSTALL")
self.upload_dependencies = upload_dependencies or split_colon_env(
"CONAN_UPLOAD_DEPENDENCIES") or ""
if isinstance(self.upload_dependencies, list):
self.upload_dependencies = ",".join(self.upload_dependencies)
if "all" in self.upload_dependencies and self.upload_dependencies != "all":
raise Exception(
"Upload dependencies only accepts or 'all' or package references. Do not mix both!"
)
self.update_dependencies = update_dependencies or get_bool_from_env(
"CONAN_UPDATE_DEPENDENCIES")
if self.channel:
os.environ["CONAN_CHANNEL"] = self.channel
if docker_32_images is not None:
self.docker_32_images = docker_32_images
else:
self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False)
self.force_selinux = force_selinux or get_bool_from_env(
"CONAN_FORCE_SELINUX")
self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1)
self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1)
self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE",
"conan==%s" % conan_version)
if self.conan_pip_package in ("0", "False"):
self.conan_pip_package = ""
self.vs10_x86_64_enabled = vs10_x86_64_enabled
self.builds_in_current_page = []
self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER")
self.config_url = config_url or os.getenv("CONAN_CONFIG_URL")
self.skip_recipe_export = skip_recipe_export or \
get_bool_from_env("CONAN_SKIP_RECIPE_EXPORT")
self.config_args = config_args or os.getenv("CONAN_CONFIG_ARGS")
self.lockfile = lockfile or os.getenv("CONAN_LOCKFILE")
def valid_pair(var, value):
return (isinstance(value, six.string_types)
or isinstance(value, bool) or isinstance(value, list)
) and not var.startswith("_") and "password" not in var
with self.printer.foldable_output("local_vars"):
self.printer.print_dict({
var: value
for var, value in self.__dict__.items()
if valid_pair(var, value)
})
self._newest_supported_conan_version = Version(
NEWEST_CONAN_SUPPORTED).minor(fill=False)
self._client_conan_version = conan_version
class ConanMultiPackager(object):
""" Help to generate common builds (setting's combinations), adjust the environment,
and run conan create command in docker containers"""
def __init__(self,
username=None,
channel=None,
runner=None,
gcc_versions=None,
visual_versions=None,
visual_runtimes=None,
visual_toolsets=None,
apple_clang_versions=None,
archs=None,
options=None,
use_docker=None,
curpage=None,
total_pages=None,
docker_image=None,
reference=None,
password=None,
remotes=None,
upload=None,
stable_branch_pattern=None,
vs10_x86_64_enabled=False,
mingw_configurations=None,
stable_channel=None,
platform_info=None,
upload_retry=None,
clang_versions=None,
login_username=None,
upload_only_when_stable=None,
upload_only_when_tag=None,
upload_only_recipe=None,
build_types=None,
cppstds=None,
skip_check_credentials=False,
allow_gcc_minors=False,
exclude_vcvars_precommand=False,
docker_run_options=None,
docker_image_skip_update=False,
docker_image_skip_pull=False,
docker_entry_script=None,
docker_32_images=None,
docker_conan_home=None,
docker_shell=None,
pip_install=None,
build_policy=None,
always_update_conan_in_docker=False,
conan_api=None,
client_cache=None,
conanfile=None,
ci_manager=None,
out=None,
test_folder=None,
cwd=None,
config_url=None,
config_args=None,
upload_dependencies=None,
force_selinux=None,
skip_recipe_export=False,
update_dependencies=None,
lockfile=None):
conan_version = get_client_version()
self.printer = Printer(out)
self.printer.print_rule()
self.printer.print_ascci_art()
self.cwd = cwd or os.getcwd()
if not conan_api:
self.conan_api, _, _ = Conan.factory()
self.conan_api.create_app()
self.client_cache = self.conan_api.app.cache
else:
self.conan_api = conan_api
self.client_cache = client_cache
self.ci_manager = ci_manager or CIManager(self.printer)
self.remotes_manager = RemotesManager(self.conan_api, self.printer,
remotes, upload)
self.username = username or os.getenv("CONAN_USERNAME", None)
self.skip_check_credentials = skip_check_credentials or get_bool_from_env(
"CONAN_SKIP_CHECK_CREDENTIALS")
self.auth_manager = AuthManager(
self.conan_api,
self.printer,
login_username,
password,
default_username=self.username,
skip_check_credentials=self.skip_check_credentials)
# Upload related variables
self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3)
if upload_only_when_stable is not None:
self.upload_only_when_stable = upload_only_when_stable
else:
self.upload_only_when_stable = get_bool_from_env(
"CONAN_UPLOAD_ONLY_WHEN_STABLE")
if upload_only_when_tag is not None:
self.upload_only_when_tag = upload_only_when_tag
else:
self.upload_only_when_tag = get_bool_from_env(
"CONAN_UPLOAD_ONLY_WHEN_TAG")
self.upload_only_recipe = upload_only_recipe or get_bool_from_env(
"CONAN_UPLOAD_ONLY_RECIPE")
self.remotes_manager.add_remotes_to_conan()
self.uploader = Uploader(self.conan_api, self.remotes_manager,
self.auth_manager, self.printer,
self.upload_retry)
self._builds = []
self._named_builds = {}
self._packages_summary = []
self._update_conan_in_docker = always_update_conan_in_docker or get_bool_from_env(
"CONAN_ALWAYS_UPDATE_CONAN_DOCKER")
self._platform_info = platform_info or PlatformInfo()
self.stable_branch_pattern = stable_branch_pattern or \
os.getenv("CONAN_STABLE_BRANCH_PATTERN", None)
self.stable_channel = stable_channel or os.getenv(
"CONAN_STABLE_CHANNEL", "stable")
self.stable_channel = self.stable_channel.rstrip()
self.partial_reference = reference or os.getenv(
"CONAN_REFERENCE", None)
self.channel = self._get_specified_channel(channel, reference)
self.conanfile = conanfile or os.getenv("CONAN_CONANFILE",
"conanfile.py")
if self.partial_reference:
if "@" in self.partial_reference:
self.reference = ConanFileReference.loads(
self.partial_reference)
else:
name, version = self.partial_reference.split("/")
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
if not os.path.exists(os.path.join(self.cwd, self.conanfile)):
raise Exception("Conanfile not found, specify a 'reference' "
"parameter with name and version")
conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile),
self.conan_api)
name, version = conanfile.name, conanfile.version
if name and version:
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
self.reference = None
self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE",
None)
# If CONAN_DOCKER_IMAGE is specified, then use docker is True
self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False)
or self._docker_image is not None)
self.docker_conan_home = docker_conan_home or os.getenv(
"CONAN_DOCKER_HOME", None)
os_name = self._platform_info.system(
) if not self.use_docker else "Linux"
self.build_generator = BuildGenerator(
reference, os_name, gcc_versions, apple_clang_versions,
clang_versions, visual_versions, visual_runtimes, visual_toolsets,
vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors,
build_types, options, cppstds)
self.build_policy = (build_policy
or self.ci_manager.get_commit_build_policy()
or split_colon_env("CONAN_BUILD_POLICY"))
if isinstance(self.build_policy, list):
self.build_policy = ",".join(self.build_policy)
self.sudo_docker_command = ""
if "CONAN_DOCKER_USE_SUDO" in os.environ:
self.sudo_docker_command = "sudo -E" if get_bool_from_env(
"CONAN_DOCKER_USE_SUDO") else ""
elif platform.system() != "Windows":
self.sudo_docker_command = "sudo -E"
self.sudo_pip_command = ""
if "CONAN_PIP_USE_SUDO" in os.environ:
self.sudo_pip_command = "sudo -E" if get_bool_from_env(
"CONAN_PIP_USE_SUDO") else ""
elif platform.system(
) != "Windows" and self._docker_image and 'conanio/' not in str(
self._docker_image):
self.sudo_pip_command = "sudo -E"
self.pip_command = os.getenv("CONAN_PIP_COMMAND", "pip")
pip_found = True if tools.os_info.is_windows else tools.which(
self.pip_command)
if not pip_found or not "pip" in self.pip_command:
raise Exception(
"CONAN_PIP_COMMAND: '{}' is not a valid pip command.".format(
self.pip_command))
self.docker_pip_command = os.getenv("CONAN_DOCKER_PIP_COMMAND", "pip")
self.docker_shell = docker_shell or os.getenv("CONAN_DOCKER_SHELL")
if self.is_wcow:
if self.docker_conan_home is None:
self.docker_conan_home = "C:/Users/ContainerAdministrator"
self.docker_shell = docker_shell or "cmd /C"
else:
if self.docker_conan_home is None:
self.docker_conan_home = "/home/conan"
self.docker_shell = docker_shell or "/bin/sh -c"
self.docker_platform_param = ""
self.lcow_user_workaround = ""
if self.is_lcow:
self.docker_platform_param = "--platform=linux"
# With LCOW, Docker doesn't respect USER directive in dockerfile yet
self.lcow_user_workaround = "sudo su conan && "
self.exclude_vcvars_precommand = exclude_vcvars_precommand or \
get_bool_from_env("CONAN_EXCLUDE_VCVARS_PRECOMMAND")
self._docker_image_skip_update = docker_image_skip_update or \
get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_UPDATE")
self._docker_image_skip_pull = docker_image_skip_pull or \
get_bool_from_env("CONAN_DOCKER_IMAGE_SKIP_PULL")
self.runner = runner or os.system
self.output_runner = ConanOutputRunner()
self.docker_run_options = docker_run_options or split_colon_env(
"CONAN_DOCKER_RUN_OPTIONS")
if isinstance(self.docker_run_options, list):
self.docker_run_options = " ".join(self.docker_run_options)
self.docker_entry_script = docker_entry_script or os.getenv(
"CONAN_DOCKER_ENTRY_SCRIPT")
self.pip_install = pip_install or split_colon_env("CONAN_PIP_INSTALL")
self.upload_dependencies = upload_dependencies or split_colon_env(
"CONAN_UPLOAD_DEPENDENCIES") or ""
if isinstance(self.upload_dependencies, list):
self.upload_dependencies = ",".join(self.upload_dependencies)
if "all" in self.upload_dependencies and self.upload_dependencies != "all":
raise Exception(
"Upload dependencies only accepts or 'all' or package references. Do not mix both!"
)
self.update_dependencies = update_dependencies or get_bool_from_env(
"CONAN_UPDATE_DEPENDENCIES")
if self.channel:
os.environ["CONAN_CHANNEL"] = self.channel
if docker_32_images is not None:
self.docker_32_images = docker_32_images
else:
self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False)
self.force_selinux = force_selinux or get_bool_from_env(
"CONAN_FORCE_SELINUX")
self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1)
self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1)
self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE",
"conan==%s" % conan_version)
if self.conan_pip_package in ("0", "False"):
self.conan_pip_package = ""
self.vs10_x86_64_enabled = vs10_x86_64_enabled
self.builds_in_current_page = []
self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER")
self.config_url = config_url or os.getenv("CONAN_CONFIG_URL")
self.skip_recipe_export = skip_recipe_export or \
get_bool_from_env("CONAN_SKIP_RECIPE_EXPORT")
self.config_args = config_args or os.getenv("CONAN_CONFIG_ARGS")
self.lockfile = lockfile or os.getenv("CONAN_LOCKFILE")
def valid_pair(var, value):
return (isinstance(value, six.string_types)
or isinstance(value, bool) or isinstance(value, list)
) and not var.startswith("_") and "password" not in var
with self.printer.foldable_output("local_vars"):
self.printer.print_dict({
var: value
for var, value in self.__dict__.items()
if valid_pair(var, value)
})
self._newest_supported_conan_version = Version(
NEWEST_CONAN_SUPPORTED).minor(fill=False)
self._client_conan_version = conan_version
def _check_conan_version(self):
tmp = self._newest_supported_conan_version
if Version(self._client_conan_version).minor(fill=False) > tmp:
msg = "Conan/CPT version mismatch. Conan version installed: " \
"%s . This version of CPT supports only Conan < %s" \
"" % (self._client_conan_version, str(tmp))
self.printer.print_message(msg)
raise Exception(msg)
# For Docker on Windows, including Linux containers on Windows
@property
def is_lcow(self):
return self.container_os == "linux" and platform.system() == "Windows"
@property
def is_wcow(self):
return self.container_os == "windows" and platform.system(
) == "Windows"
@property
def container_os(self):
# CONAN_DOCKER_PLATFORM=linux must be specified for LCOW
if self.use_docker:
if "CONAN_DOCKER_PLATFORM" in os.environ:
return os.getenv("CONAN_DOCKER_PLATFORM", "windows").lower()
else:
return "windows"
else:
return ""
@property
def packages_summary(self):
return self._packages_summary
def save_packages_summary(self, file):
self.printer.print_message("Saving packages summary to " + file)
import json
import datetime
def default(o):
if isinstance(o, (datetime.date, datetime.datetime)):
return o.isoformat()
with open(file, 'w') as outfile:
json.dump(self.packages_summary, outfile, default=default)
@property
def items(self):
return self._builds
@items.setter
def items(self, confs):
self.builds = confs
@property
def builds(self):
# Retrocompatibility iterating
self.printer.print_message(
"WARNING",
"\n\n\n******* ITERATING THE CONAN_PACKAGE_TOOLS BUILDS WITH "
".builds is deprecated use '.items' instead (unpack 5 elements: "
"settings, options, env_vars, build_requires, reference *******"
"**\n\n\n")
return [elem[0:4] for elem in self._builds]
@builds.setter
def builds(self, confs):
"""For retro compatibility directly assigning builds"""
self._named_builds = {}
self._builds = []
for values in confs:
if len(values) == 2:
self._builds.append(
BuildConf(values[0], values[1], {}, {}, self.reference))
elif len(values) == 4:
self._builds.append(
BuildConf(values[0], values[1], values[2], values[3],
self.reference))
elif len(values) != 5:
raise Exception(
"Invalid build configuration, has to be a tuple of "
"(settings, options, env_vars, build_requires, reference)")
else:
self._builds.append(BuildConf(*values))
@property
def named_builds(self):
return self._named_builds
@named_builds.setter
def named_builds(self, confs):
self._builds = []
self._named_builds = {}
for key, pages in confs.items():
for values in pages:
if len(values) == 2:
bc = BuildConf(values[0], values[1], {}, {},
self.reference)
self._named_builds.setdefault(key, []).append(bc)
elif len(values) == 4:
bc = BuildConf(values[0], values[1], values[2], values[3],
self.reference)
self._named_builds.setdefault(key, []).append(bc)
elif len(values) != 5:
raise Exception(
"Invalid build configuration, has to be a tuple of "
"(settings, options, env_vars, build_requires, reference)"
)
else:
self._named_builds.setdefault(key, []).append(
BuildConf(*values))
def login(self, remote_name):
self.auth_manager.login(remote_name)
def add_common_builds(self,
shared_option_name=None,
pure_c=True,
dll_with_static_runtime=False,
reference=None,
header_only=True,
build_all_options_values=None):
if reference:
if "@" in reference:
reference = ConanFileReference.loads(reference)
else:
name, version = reference.split("/")
reference = ConanFileReference(name, version, self.username,
self.channel)
else:
reference = self.reference
if not reference:
raise Exception(
"Specify a CONAN_REFERENCE or name and version fields in the recipe"
)
if shared_option_name is None:
env_shared_option_name = os.getenv("CONAN_SHARED_OPTION_NAME",
None)
shared_option_name = env_shared_option_name if str(
env_shared_option_name).lower() != "false" else False
build_all_options_values = build_all_options_values or split_colon_env(
"CONAN_BUILD_ALL_OPTIONS_VALUES") or []
if not isinstance(build_all_options_values, list):
raise Exception(
"'build_all_options_values' must be a list. e.g. ['foo:opt', 'foo:bar']"
)
conanfile = None
if os.path.exists(os.path.join(self.cwd, self.conanfile)):
conanfile = load_cf_class(os.path.join(self.cwd, self.conanfile),
self.conan_api)
header_only_option = None
if conanfile:
if hasattr(
conanfile, "options"
) and conanfile.options and "header_only" in conanfile.options:
header_only_option = "%s:header_only" % reference.name
if shared_option_name is None:
if conanfile:
if hasattr(
conanfile, "options"
) and conanfile.options and "shared" in conanfile.options:
shared_option_name = "%s:shared" % reference.name
# filter only valid options
raw_options_for_building = [
opt[opt.find(":") + 1:] for opt in build_all_options_values
]
for raw_option in reversed(raw_options_for_building):
if hasattr(conanfile, "options") and conanfile.options and \
not isinstance(conanfile.options.get(raw_option), list):
raw_options_for_building.remove(raw_option)
if raw_options_for_building and conanfile:
# get option and its values
cloned_options = copy.copy(conanfile.options)
for key, value in conanfile.options.items():
if key == "shared" and shared_option_name:
continue
elif key not in raw_options_for_building:
del cloned_options[key]
cloned_options2 = {}
for key, value in cloned_options.items():
# add package reference to the option name
if not key.startswith("{}:".format(reference.name)):
cloned_options2["{}:{}".format(reference.name,
key)] = value
# combine all options x values (cartesian product)
build_all_options_values = [
dict(zip(cloned_options2, v))
for v in product(*cloned_options2.values())
]
builds = self.build_generator.get_builds(pure_c, shared_option_name,
dll_with_static_runtime,
reference,
build_all_options_values)
if header_only_option and header_only:
if conanfile.default_options.get("header_only"):
cloned_builds = copy.deepcopy(builds)
for settings, options, env_vars, build_requires, reference in cloned_builds:
options.update({header_only_option: False})
builds.extend(cloned_builds)
else:
settings, options, env_vars, build_requires, reference = builds[
0]
cloned_options = copy.copy(options)
cloned_options.update({header_only_option: True})
builds.append(
BuildConf(copy.copy(settings), cloned_options,
copy.copy(env_vars), copy.copy(build_requires),
reference))
self._builds.extend(builds)
def add(self,
settings=None,
options=None,
env_vars=None,
build_requires=None,
reference=None):
settings = settings or {}
options = options or {}
env_vars = env_vars or {}
build_requires = build_requires or {}
if reference:
reference = ConanFileReference.loads(
"%s@%s/%s" % (reference, self.username, self.channel))
reference = reference or self.reference
self._builds.append(
BuildConf(settings, options, env_vars, build_requires, reference))
def remove_build_if(self, predicate):
filtered_builds = []
for build in self.items:
if not predicate(build):
filtered_builds.append(build)
self._builds = filtered_builds
def update_build_if(self,
predicate,
new_settings=None,
new_options=None,
new_env_vars=None,
new_build_requires=None,
new_reference=None):
updated_builds = []
for build in self.items:
if predicate(build):
if new_settings:
build.settings.update(new_settings)
if new_options:
build.options.update(new_options)
if new_build_requires:
build.build_requires.update(new_build_requires)
if new_env_vars:
build.env_vars.update(new_env_vars)
if new_reference:
build.reference = new_reference
updated_builds.append(build)
self._builds = updated_builds
def run(self, base_profile_name=None, summary_file=None):
self._check_conan_version()
env_vars = self.auth_manager.env_vars()
env_vars.update(self.remotes_manager.env_vars())
with tools.environment_append(env_vars):
self.printer.print_message("Running builds...")
if self.ci_manager.skip_builds():
self.printer.print_message(
"Skipped builds due [skip ci] commit message")
return 99
if not self.skip_check_credentials and self._upload_enabled():
self.auth_manager.login(
self.remotes_manager.upload_remote_name)
if self.conan_pip_package and not self.use_docker:
with self.printer.foldable_output("pip_update"):
self.runner('%s %s install -q %s' %
(self.sudo_pip_command, self.pip_command,
self.conan_pip_package))
if self.pip_install:
packages = " ".join(self.pip_install)
self.printer.print_message(
"Install extra python packages: {}".format(
packages))
self.runner('%s %s install -q %s' %
(self.sudo_pip_command, self.pip_command,
packages))
self.run_builds(base_profile_name=base_profile_name)
summary_file = summary_file or os.getenv("CPT_SUMMARY_FILE", None)
if summary_file:
self.save_packages_summary(summary_file)
def _upload_enabled(self):
if not self.remotes_manager.upload_remote_name:
return False
if not self.auth_manager.credentials_ready(
self.remotes_manager.upload_remote_name):
return False
if self.upload_only_when_tag and not self.ci_manager.is_tag():
self.printer.print_message("Skipping upload, not tag branch")
return False
st_channel = self.stable_channel or "stable"
if self.upload_only_when_stable and self.channel != st_channel and not self.upload_only_when_tag:
self.printer.print_message("Skipping upload, not stable channel")
return False
if not os.getenv("CONAN_TEST_SUITE", False):
if self.ci_manager.is_pull_request():
# PENDING! can't found info for gitlab/bamboo
self.printer.print_message(
"Skipping upload, this is a Pull Request")
return False
def raise_error(field):
raise Exception("Upload not possible, '%s' is missing!" % field)
if not self.channel and "@" not in self.partial_reference:
raise_error("channel")
if not self.username and "@" not in self.partial_reference:
raise_error("username")
return True
def run_builds(self,
curpage=None,
total_pages=None,
base_profile_name=None):
if len(self.named_builds) > 0 and len(self.items) > 0:
raise Exception(
"Both bulk and named builds are set. Only one is allowed.")
self.builds_in_current_page = []
if len(self.items) > 0:
curpage = curpage or int(self.curpage)
total_pages = total_pages or int(self.total_pages)
for index, build in enumerate(self.items):
if curpage is None or total_pages is None or (
index % total_pages) + 1 == curpage:
self.builds_in_current_page.append(build)
elif len(self.named_builds) > 0:
curpage = curpage or self.curpage
if curpage not in self.named_builds:
raise Exception("No builds set for page %s" % curpage)
for build in self.named_builds[curpage]:
self.builds_in_current_page.append(build)
self.printer.print_current_page(curpage, total_pages)
self.printer.print_jobs(self.builds_in_current_page)
pulled_docker_images = defaultdict(lambda: False)
skip_recipe_export = False
# FIXME: Remove in Conan 1.3, https://github.com/conan-io/conan/issues/2787
for index, build in enumerate(self.builds_in_current_page):
self.printer.print_message(
"Build: %s/%s" % (index + 1, len(self.builds_in_current_page)))
base_profile_name = base_profile_name or os.getenv(
"CONAN_BASE_PROFILE")
if base_profile_name:
self.printer.print_message(
"**************************************************")
self.printer.print_message("Using specified default "
"base profile: %s" %
base_profile_name)
self.printer.print_message(
"**************************************************")
profile_text, base_profile_text = get_profiles(
self.client_cache, build, base_profile_name)
if not self.use_docker:
profile_abs_path = save_profile_to_tmp(profile_text)
r = CreateRunner(
profile_abs_path,
build.reference,
self.conan_api,
self.uploader,
exclude_vcvars_precommand=self.exclude_vcvars_precommand,
build_policy=self.build_policy,
runner=self.runner,
cwd=self.cwd,
printer=self.printer,
upload=self._upload_enabled(),
upload_only_recipe=self.upload_only_recipe,
test_folder=self.test_folder,
config_url=self.config_url,
config_args=self.config_args,
upload_dependencies=self.upload_dependencies,
conanfile=self.conanfile,
lockfile=self.lockfile,
skip_recipe_export=skip_recipe_export,
update_dependencies=self.update_dependencies)
r.run()
self._packages_summary.append({
"configuration": build,
"package": r.results
})
else:
docker_image = self._get_docker_image(build)
r = DockerCreateRunner(
profile_text,
base_profile_text,
base_profile_name,
build.reference,
conan_pip_package=self.conan_pip_package,
docker_image=docker_image,
sudo_docker_command=self.sudo_docker_command,
sudo_pip_command=self.sudo_pip_command,
docker_image_skip_update=self._docker_image_skip_update,
docker_image_skip_pull=self._docker_image_skip_pull,
build_policy=self.build_policy,
always_update_conan_in_docker=self._update_conan_in_docker,
upload=self._upload_enabled(),
upload_retry=self.upload_retry,
upload_only_recipe=self.upload_only_recipe,
runner=self.runner,
docker_shell=self.docker_shell,
docker_conan_home=self.docker_conan_home,
docker_platform_param=self.docker_platform_param,
docker_run_options=self.docker_run_options,
lcow_user_workaround=self.lcow_user_workaround,
test_folder=self.test_folder,
pip_install=self.pip_install,
docker_pip_command=self.docker_pip_command,
config_url=self.config_url,
config_args=self.config_args,
printer=self.printer,
upload_dependencies=self.upload_dependencies,
conanfile=self.conanfile,
lockfile=self.lockfile,
force_selinux=self.force_selinux,
skip_recipe_export=skip_recipe_export,
update_dependencies=self.update_dependencies)
r.run(pull_image=not pulled_docker_images[docker_image],
docker_entry_script=self.docker_entry_script)
pulled_docker_images[docker_image] = True
skip_recipe_export = self.skip_recipe_export
def _get_docker_image(self, build):
if self._docker_image:
docker_image = self._docker_image
else:
compiler_name = build.settings.get("compiler", "")
compiler_version = build.settings.get("compiler.version", "")
docker_image = self._autodetect_docker_base_image(
compiler_name, compiler_version)
arch = build.settings.get("arch", "") or build.settings.get(
"arch_build", "")
if self.docker_32_images and arch == "x86":
build.settings["arch_build"] = "x86"
docker_arch_suffix = "x86"
elif arch != "x86" and arch != "x86_64":
docker_arch_suffix = arch
else:
docker_arch_suffix = None
if docker_arch_suffix and "-" not in docker_image:
docker_image = "%s-%s" % (docker_image, docker_arch_suffix)
return docker_image
@staticmethod
def _autodetect_docker_base_image(compiler_name, compiler_version):
if compiler_name not in ["clang", "gcc"]:
raise Exception("Docker image cannot be autodetected for "
"the compiler %s" % compiler_name)
if compiler_name == "gcc" and Version(compiler_version) > Version("5"):
compiler_version = Version(compiler_version).major(fill=False)
return "conanio/%s%s" % (compiler_name,
compiler_version.replace(".", ""))
def _get_channel(self, specified_channel, stable_channel, upload_when_tag):
if not specified_channel:
return
if self.stable_branch_pattern:
stable_patterns = [self.stable_branch_pattern]
else:
stable_patterns = ["master$", "release*", "stable*"]
branch = self.ci_manager.get_branch()
self.printer.print_message("Branch detected", branch)
for pattern in stable_patterns:
prog = re.compile(pattern)
if branch and prog.match(branch):
self.printer.print_message(
"Info",
"Redefined channel by CI branch matching with '%s', "
"setting CONAN_CHANNEL to '%s'" %
(pattern, stable_channel))
return stable_channel
if self.ci_manager.is_tag() and upload_when_tag:
self.printer.print_message(
"Info", "Redefined channel by branch tag, "
"setting CONAN_CHANNEL to '%s'" % stable_channel)
return stable_channel
return specified_channel
def _get_specified_channel(self, channel, reference):
partial_reference = reference or os.getenv("CONAN_REFERENCE", None)
specified_channel = None
# without name/channel e.g. zlib/1.2.11@
if partial_reference:
if "@" in partial_reference:
specified_channel = channel or os.getenv("CONAN_CHANNEL", None)
else:
specified_channel = channel or os.getenv(
"CONAN_CHANNEL", "testing")
specified_channel = specified_channel.rstrip()
else:
if self.username:
specified_channel = channel or os.getenv(
"CONAN_CHANNEL", "testing")
specified_channel = specified_channel.rstrip()
else:
specified_channel = channel or os.getenv("CONAN_CHANNEL", None)
return self._get_channel(specified_channel, self.stable_channel,
self.upload_only_when_tag)
def is_tag():
printer = Printer()
ci_manager = CIManager(printer)
return ci_manager.is_tag()
def test_valid_config(self):
manager = ConfigManager(self.conan_api, Printer())
manager.install('https://github.com/bincrafters/bincrafters-config.git')
def assert_serial_deserial(self, manager, expected):
self.assertEquals(manager.env_vars(), expected)
with tools.environment_append(expected):
manager = RemotesManager(self.conan_api, Printer())
self.assertEquals(manager.env_vars(), expected)
def __init__(self,
args=None,
username=None,
channel=None,
runner=None,
gcc_versions=None,
visual_versions=None,
visual_runtimes=None,
apple_clang_versions=None,
archs=None,
use_docker=None,
curpage=None,
total_pages=None,
docker_image=None,
reference=None,
password=None,
remotes=None,
upload=None,
stable_branch_pattern=None,
vs10_x86_64_enabled=False,
mingw_configurations=None,
stable_channel=None,
platform_info=None,
upload_retry=None,
clang_versions=None,
login_username=None,
upload_only_when_stable=None,
build_types=None,
skip_check_credentials=False,
allow_gcc_minors=False,
exclude_vcvars_precommand=False,
docker_image_skip_update=False,
docker_image_skip_pull=False,
docker_entry_script=None,
docker_32_images=None,
build_policy=None,
always_update_conan_in_docker=False,
conan_api=None,
client_cache=None,
ci_manager=None,
out=None,
test_folder=None):
self.printer = Printer(out)
self.printer.print_rule()
self.printer.print_ascci_art()
if not conan_api:
self.conan_api, self.client_cache, _ = Conan.factory()
else:
self.conan_api = conan_api
self.client_cache = client_cache
self.ci_manager = ci_manager or CIManager(self.printer)
self.remotes_manager = RemotesManager(self.conan_api, self.printer,
remotes, upload)
self.username = username or os.getenv("CONAN_USERNAME", None)
if not self.username:
raise Exception(
"Instance ConanMultiPackage with 'username' parameter or use "
"CONAN_USERNAME env variable")
self.skip_check_credentials = skip_check_credentials or \
os.getenv("CONAN_SKIP_CHECK_CREDENTIALS", False)
self.auth_manager = AuthManager(
self.conan_api,
self.printer,
login_username,
password,
default_username=self.username,
skip_check_credentials=self.skip_check_credentials)
self.uploader = Uploader(self.conan_api, self.remotes_manager,
self.auth_manager, self.printer)
self._builds = []
self._named_builds = {}
self._update_conan_in_docker = (
always_update_conan_in_docker
or os.getenv("CONAN_ALWAYS_UPDATE_CONAN_DOCKER", False))
self._platform_info = platform_info or PlatformInfo()
self.stable_branch_pattern = stable_branch_pattern or \
os.getenv("CONAN_STABLE_BRANCH_PATTERN", None)
self.specified_channel = channel or os.getenv("CONAN_CHANNEL",
"testing")
self.specified_channel = self.specified_channel.rstrip()
self.stable_channel = stable_channel or os.getenv(
"CONAN_STABLE_CHANNEL", "stable")
self.stable_channel = self.stable_channel.rstrip()
self.channel = self._get_channel(self.specified_channel,
self.stable_channel)
self.partial_reference = reference or os.getenv(
"CONAN_REFERENCE", None)
if self.partial_reference:
if "@" in self.partial_reference:
self.reference = ConanFileReference.loads(
self.partial_reference)
else:
name, version = self.partial_reference.split("/")
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
if not os.path.exists("conanfile.py"):
raise Exception(
"Conanfile not found, specify a 'reference' parameter with name and version"
)
conanfile = load_conanfile_class("./conanfile.py")
name, version = conanfile.name, conanfile.version
if name and version:
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
self.reference = None
# If CONAN_DOCKER_IMAGE is speified, then use docker is True
self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False)
or os.getenv("CONAN_DOCKER_IMAGE",
None) is not None)
os_name = self._platform_info.system(
) if not self.use_docker else "Linux"
self.build_generator = BuildGenerator(
reference, os_name, gcc_versions, apple_clang_versions,
clang_versions, visual_versions, visual_runtimes,
vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors,
build_types)
build_policy = (build_policy
or self.ci_manager.get_commit_build_policy()
or os.getenv("CONAN_BUILD_POLICY", None))
if build_policy:
if build_policy.lower() not in ("never", "outdated", "missing"):
raise Exception(
"Invalid build policy, valid values: never, outdated, missing"
)
self.build_policy = build_policy
self.sudo_docker_command = ""
if "CONAN_DOCKER_USE_SUDO" in os.environ:
self.sudo_docker_command = "sudo -E" if get_bool_from_env(
"CONAN_DOCKER_USE_SUDO") else ""
elif platform.system() != "Windows":
self.sudo_docker_command = "sudo -E"
self.sudo_pip_command = ""
if "CONAN_PIP_USE_SUDO" in os.environ:
self.sudo_pip_command = "sudo -E" if get_bool_from_env(
"CONAN_PIP_USE_SUDO") else ""
elif platform.system() != "Windows":
self.sudo_pip_command = "sudo -E"
self.docker_shell = ""
self.docker_conan_home = ""
if self.is_wcow:
self.docker_conan_home = "C:/Users/ContainerAdministrator"
self.docker_shell = "cmd /C"
else:
self.docker_conan_home = "/home/conan"
self.docker_shell = "/bin/sh -c"
self.docker_platform_param = ""
self.lcow_user_workaround = ""
if self.is_lcow:
self.docker_platform_param = "--platform=linux"
# With LCOW, Docker doesn't respect USER directive in dockerfile yet
self.lcow_user_workaround = "sudo su conan && "
self.exclude_vcvars_precommand = (
exclude_vcvars_precommand
or os.getenv("CONAN_EXCLUDE_VCVARS_PRECOMMAND", False))
self._docker_image_skip_update = (
docker_image_skip_update
or os.getenv("CONAN_DOCKER_IMAGE_SKIP_UPDATE", False))
self._docker_image_skip_pull = (docker_image_skip_pull or os.getenv(
"CONAN_DOCKER_IMAGE_SKIP_PULL", False))
self.runner = runner or os.system
self.output_runner = ConanOutputRunner()
self.args = " ".join(args) if args else " ".join(sys.argv[1:])
# Upload related variables
self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3)
if upload_only_when_stable is not None:
self.upload_only_when_stable = upload_only_when_stable
else:
self.upload_only_when_stable = get_bool_from_env(
"CONAN_UPLOAD_ONLY_WHEN_STABLE")
self.docker_entry_script = docker_entry_script or os.getenv(
"CONAN_DOCKER_ENTRY_SCRIPT")
os.environ["CONAN_CHANNEL"] = self.channel
# If CONAN_DOCKER_IMAGE is speified, then use docker is True
self.use_docker = use_docker or os.getenv("CONAN_USE_DOCKER", False) or \
(os.getenv("CONAN_DOCKER_IMAGE", None) is not None)
if docker_32_images is not None:
self.docker_32_images = docker_32_images
else:
self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False)
self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1)
self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1)
self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE",
None)
self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE",
"conan==%s" % client_version)
if self.conan_pip_package in ("0", "False"):
self.conan_pip_package = False
self.vs10_x86_64_enabled = vs10_x86_64_enabled
self.builds_in_current_page = []
self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER", None)
def valid_pair(var, value):
return (isinstance(value, six.string_types)
or isinstance(value, bool) or isinstance(value, list)
) and not var.startswith("_") and "password" not in var
with self.printer.foldable_output("local_vars"):
self.printer.print_dict({
var: value
for var, value in self.__dict__.items()
if valid_pair(var, value)
})
class ConanMultiPackager(object):
""" Help to generate common builds (setting's combinations), adjust the environment,
and run conan create command in docker containers"""
def __init__(self,
args=None,
username=None,
channel=None,
runner=None,
gcc_versions=None,
visual_versions=None,
visual_runtimes=None,
apple_clang_versions=None,
archs=None,
use_docker=None,
curpage=None,
total_pages=None,
docker_image=None,
reference=None,
password=None,
remotes=None,
upload=None,
stable_branch_pattern=None,
vs10_x86_64_enabled=False,
mingw_configurations=None,
stable_channel=None,
platform_info=None,
upload_retry=None,
clang_versions=None,
login_username=None,
upload_only_when_stable=None,
build_types=None,
skip_check_credentials=False,
allow_gcc_minors=False,
exclude_vcvars_precommand=False,
docker_image_skip_update=False,
docker_image_skip_pull=False,
docker_entry_script=None,
docker_32_images=None,
build_policy=None,
always_update_conan_in_docker=False,
conan_api=None,
client_cache=None,
ci_manager=None,
out=None,
test_folder=None):
self.printer = Printer(out)
self.printer.print_rule()
self.printer.print_ascci_art()
if not conan_api:
self.conan_api, self.client_cache, _ = Conan.factory()
else:
self.conan_api = conan_api
self.client_cache = client_cache
self.ci_manager = ci_manager or CIManager(self.printer)
self.remotes_manager = RemotesManager(self.conan_api, self.printer,
remotes, upload)
self.username = username or os.getenv("CONAN_USERNAME", None)
if not self.username:
raise Exception(
"Instance ConanMultiPackage with 'username' parameter or use "
"CONAN_USERNAME env variable")
self.skip_check_credentials = skip_check_credentials or \
os.getenv("CONAN_SKIP_CHECK_CREDENTIALS", False)
self.auth_manager = AuthManager(
self.conan_api,
self.printer,
login_username,
password,
default_username=self.username,
skip_check_credentials=self.skip_check_credentials)
self.uploader = Uploader(self.conan_api, self.remotes_manager,
self.auth_manager, self.printer)
self._builds = []
self._named_builds = {}
self._update_conan_in_docker = (
always_update_conan_in_docker
or os.getenv("CONAN_ALWAYS_UPDATE_CONAN_DOCKER", False))
self._platform_info = platform_info or PlatformInfo()
self.stable_branch_pattern = stable_branch_pattern or \
os.getenv("CONAN_STABLE_BRANCH_PATTERN", None)
self.specified_channel = channel or os.getenv("CONAN_CHANNEL",
"testing")
self.specified_channel = self.specified_channel.rstrip()
self.stable_channel = stable_channel or os.getenv(
"CONAN_STABLE_CHANNEL", "stable")
self.stable_channel = self.stable_channel.rstrip()
self.channel = self._get_channel(self.specified_channel,
self.stable_channel)
self.partial_reference = reference or os.getenv(
"CONAN_REFERENCE", None)
if self.partial_reference:
if "@" in self.partial_reference:
self.reference = ConanFileReference.loads(
self.partial_reference)
else:
name, version = self.partial_reference.split("/")
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
if not os.path.exists("conanfile.py"):
raise Exception(
"Conanfile not found, specify a 'reference' parameter with name and version"
)
conanfile = load_conanfile_class("./conanfile.py")
name, version = conanfile.name, conanfile.version
if name and version:
self.reference = ConanFileReference(name, version,
self.username,
self.channel)
else:
self.reference = None
# If CONAN_DOCKER_IMAGE is speified, then use docker is True
self.use_docker = (use_docker or os.getenv("CONAN_USE_DOCKER", False)
or os.getenv("CONAN_DOCKER_IMAGE",
None) is not None)
os_name = self._platform_info.system(
) if not self.use_docker else "Linux"
self.build_generator = BuildGenerator(
reference, os_name, gcc_versions, apple_clang_versions,
clang_versions, visual_versions, visual_runtimes,
vs10_x86_64_enabled, mingw_configurations, archs, allow_gcc_minors,
build_types)
build_policy = (build_policy
or self.ci_manager.get_commit_build_policy()
or os.getenv("CONAN_BUILD_POLICY", None))
if build_policy:
if build_policy.lower() not in ("never", "outdated", "missing"):
raise Exception(
"Invalid build policy, valid values: never, outdated, missing"
)
self.build_policy = build_policy
self.sudo_docker_command = ""
if "CONAN_DOCKER_USE_SUDO" in os.environ:
self.sudo_docker_command = "sudo -E" if get_bool_from_env(
"CONAN_DOCKER_USE_SUDO") else ""
elif platform.system() != "Windows":
self.sudo_docker_command = "sudo -E"
self.sudo_pip_command = ""
if "CONAN_PIP_USE_SUDO" in os.environ:
self.sudo_pip_command = "sudo -E" if get_bool_from_env(
"CONAN_PIP_USE_SUDO") else ""
elif platform.system() != "Windows":
self.sudo_pip_command = "sudo -E"
self.docker_shell = ""
self.docker_conan_home = ""
if self.is_wcow:
self.docker_conan_home = "C:/Users/ContainerAdministrator"
self.docker_shell = "cmd /C"
else:
self.docker_conan_home = "/home/conan"
self.docker_shell = "/bin/sh -c"
self.docker_platform_param = ""
self.lcow_user_workaround = ""
if self.is_lcow:
self.docker_platform_param = "--platform=linux"
# With LCOW, Docker doesn't respect USER directive in dockerfile yet
self.lcow_user_workaround = "sudo su conan && "
self.exclude_vcvars_precommand = (
exclude_vcvars_precommand
or os.getenv("CONAN_EXCLUDE_VCVARS_PRECOMMAND", False))
self._docker_image_skip_update = (
docker_image_skip_update
or os.getenv("CONAN_DOCKER_IMAGE_SKIP_UPDATE", False))
self._docker_image_skip_pull = (docker_image_skip_pull or os.getenv(
"CONAN_DOCKER_IMAGE_SKIP_PULL", False))
self.runner = runner or os.system
self.output_runner = ConanOutputRunner()
self.args = " ".join(args) if args else " ".join(sys.argv[1:])
# Upload related variables
self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3)
if upload_only_when_stable is not None:
self.upload_only_when_stable = upload_only_when_stable
else:
self.upload_only_when_stable = get_bool_from_env(
"CONAN_UPLOAD_ONLY_WHEN_STABLE")
self.docker_entry_script = docker_entry_script or os.getenv(
"CONAN_DOCKER_ENTRY_SCRIPT")
os.environ["CONAN_CHANNEL"] = self.channel
# If CONAN_DOCKER_IMAGE is speified, then use docker is True
self.use_docker = use_docker or os.getenv("CONAN_USE_DOCKER", False) or \
(os.getenv("CONAN_DOCKER_IMAGE", None) is not None)
if docker_32_images is not None:
self.docker_32_images = docker_32_images
else:
self.docker_32_images = os.getenv("CONAN_DOCKER_32_IMAGES", False)
self.curpage = curpage or os.getenv("CONAN_CURRENT_PAGE", 1)
self.total_pages = total_pages or os.getenv("CONAN_TOTAL_PAGES", 1)
self._docker_image = docker_image or os.getenv("CONAN_DOCKER_IMAGE",
None)
self.conan_pip_package = os.getenv("CONAN_PIP_PACKAGE",
"conan==%s" % client_version)
if self.conan_pip_package in ("0", "False"):
self.conan_pip_package = False
self.vs10_x86_64_enabled = vs10_x86_64_enabled
self.builds_in_current_page = []
self.test_folder = test_folder or os.getenv("CPT_TEST_FOLDER", None)
def valid_pair(var, value):
return (isinstance(value, six.string_types)
or isinstance(value, bool) or isinstance(value, list)
) and not var.startswith("_") and "password" not in var
with self.printer.foldable_output("local_vars"):
self.printer.print_dict({
var: value
for var, value in self.__dict__.items()
if valid_pair(var, value)
})
# For Docker on Windows, including Linux containers on Windows
@property
def is_lcow(self):
return self.container_os == "linux" and platform.system() == "Windows"
@property
def is_wcow(self):
return self.container_os == "windows" and platform.system(
) == "Windows"
@property
def container_os(self):
# CONAN_DOCKER_PLATFORM=linux must be specified for LCOW
if self.use_docker:
if "CONAN_DOCKER_PLATFORM" in os.environ:
return os.getenv("CONAN_DOCKER_PLATFORM", "windows").lower()
else:
return "windows"
else:
return ""
@property
def items(self):
return self._builds
@items.setter
def items(self, confs):
self.builds = confs
@property
def builds(self):
# Retrocompatibility iterating
self.printer.print_message(
"WARNING",
"\n\n\n******* ITERATING THE CONAN_PACKAGE_TOOLS BUILDS WITH "
".builds is deprecated use .items() instead (unpack 5 elements: "
"settings, options, env_vars, build_requires, reference *******"
"**\n\n\n")
return [elem[0:4] for elem in self._builds]
@builds.setter
def builds(self, confs):
"""For retro compatibility directly assigning builds"""
self._named_builds = {}
self._builds = []
for values in confs:
if len(values) == 2:
self._builds.append(
BuildConf(values[0], values[1], {}, {}, self.reference))
elif len(values) == 4:
self._builds.append(
BuildConf(values[0], values[1], values[2], values[3],
self.reference))
elif len(values) != 5:
raise Exception(
"Invalid build configuration, has to be a tuple of "
"(settings, options, env_vars, build_requires, reference)")
else:
self._builds.append(BuildConf(*values))
@property
def named_builds(self):
return self._named_builds
@named_builds.setter
def named_builds(self, confs):
self._builds = []
self._named_builds = {}
for key, pages in confs.items():
for values in pages:
if len(values) == 2:
bc = BuildConf(values[0], values[1], {}, {},
self.reference)
self._named_builds.setdefault(key, []).append(bc)
elif len(values) == 4:
bc = BuildConf(values[0], values[1], values[2], values[3],
self.reference)
self._named_builds.setdefault(key, []).append(bc)
elif len(values) != 5:
raise Exception(
"Invalid build configuration, has to be a tuple of "
"(settings, options, env_vars, build_requires, reference)"
)
else:
self._named_builds.setdefault(key, []).append(
BuildConf(*values))
def add_common_builds(self,
shared_option_name=None,
pure_c=True,
dll_with_static_runtime=False,
reference=None):
if not reference and not self.reference:
raise Exception(
"Specify a CONAN_REFERENCE or name and version fields in the recipe"
)
if shared_option_name is None:
if os.path.exists("conanfile.py"):
conanfile = load_conanfile_class("./conanfile.py")
if hasattr(conanfile,
"options") and "shared" in conanfile.options:
shared_option_name = "%s:shared" % self.reference.name
tmp = self.build_generator.get_builds(pure_c, shared_option_name,
dll_with_static_runtime,
reference or self.reference)
self._builds.extend(tmp)
def add(self,
settings=None,
options=None,
env_vars=None,
build_requires=None,
reference=None):
settings = settings or {}
options = options or {}
env_vars = env_vars or {}
build_requires = build_requires or {}
if reference:
reference = ConanFileReference.loads(
"%s@%s/%s" % (reference, self.username, self.channel))
reference = reference or self.reference
self._builds.append(
BuildConf(settings, options, env_vars, build_requires, reference))
def run(self, base_profile_name=None):
env_vars = self.auth_manager.env_vars()
env_vars.update(self.remotes_manager.env_vars())
with tools.environment_append(env_vars):
self.printer.print_message("Running builds...")
if self.ci_manager.skip_builds():
print("Skipped builds due [skip ci] commit message")
return 99
if not self.skip_check_credentials and self._upload_enabled():
self.remotes_manager.add_remotes_to_conan()
self.auth_manager.login(
self.remotes_manager.upload_remote_name)
if self.conan_pip_package and not self.use_docker:
with self.printer.foldable_output("pip_update"):
self.runner(
'%s pip install %s' %
(self.sudo_pip_command, self.conan_pip_package))
self.run_builds(base_profile_name=base_profile_name)
def _upload_enabled(self):
if not self.remotes_manager.upload_remote_name:
return False
if not self.auth_manager.credentials_ready(
self.remotes_manager.upload_remote_name):
return False
st_channel = self.stable_channel or "stable"
if self.upload_only_when_stable and self.channel != st_channel:
self.printer.print_message("Skipping upload, not stable channel")
return False
if not os.getenv("CONAN_TEST_SUITE", False):
if self.ci_manager.is_pull_request():
# PENDING! can't found info for gitlab/bamboo
self.printer.print_message(
"Skipping upload, this is a Pull Request")
return False
def raise_error(field):
raise Exception("Upload not possible, '%s' is missing!" % field)
if not self.channel:
raise_error("channel")
if not self.username:
raise_error("username")
return True
def run_builds(self,
curpage=None,
total_pages=None,
base_profile_name=None):
if len(self.named_builds) > 0 and len(self.items) > 0:
raise Exception(
"Both bulk and named builds are set. Only one is allowed.")
self.builds_in_current_page = []
if len(self.items) > 0:
curpage = curpage or int(self.curpage)
total_pages = total_pages or int(self.total_pages)
for index, build in enumerate(self.items):
if curpage is None or total_pages is None or (
index % total_pages) + 1 == curpage:
self.builds_in_current_page.append(build)
elif len(self.named_builds) > 0:
curpage = curpage or self.curpage
if curpage not in self.named_builds:
raise Exception("No builds set for page %s" % curpage)
for build in self.named_builds[curpage]:
self.builds_in_current_page.append(build)
self.printer.print_current_page(curpage, total_pages)
self.printer.print_jobs(self.builds_in_current_page)
pulled_docker_images = defaultdict(lambda: False)
# FIXME: Remove in Conan 1.3, https://github.com/conan-io/conan/issues/2787
abs_folder = os.path.realpath(os.getcwd())
for build in self.builds_in_current_page:
base_profile_name = base_profile_name or os.getenv(
"CONAN_BASE_PROFILE")
if base_profile_name:
self.printer.print_message(
"**************************************************")
self.printer.print_message("Using specified default "
"base profile: %s" %
base_profile_name)
self.printer.print_message(
"**************************************************")
profile_text, base_profile_text = get_profiles(
self.client_cache, build, base_profile_name)
if not self.use_docker:
profile_abs_path = save_profile_to_tmp(profile_text)
r = CreateRunner(
profile_abs_path,
build.reference,
self.conan_api,
self.uploader,
args=self.args,
exclude_vcvars_precommand=self.exclude_vcvars_precommand,
build_policy=self.build_policy,
runner=self.runner,
abs_folder=abs_folder,
printer=self.printer,
upload=self._upload_enabled(),
test_folder=self.test_folder)
r.run()
else:
docker_image = self._get_docker_image(build)
r = DockerCreateRunner(
profile_text,
base_profile_text,
base_profile_name,
build.reference,
args=self.args,
conan_pip_package=self.conan_pip_package,
docker_image=docker_image,
sudo_docker_command=self.sudo_docker_command,
sudo_pip_command=self.sudo_pip_command,
docker_image_skip_update=self._docker_image_skip_update,
docker_image_skip_pull=self._docker_image_skip_pull,
build_policy=self.build_policy,
always_update_conan_in_docker=self._update_conan_in_docker,
upload=self._upload_enabled(),
runner=self.runner,
docker_shell=self.docker_shell,
docker_conan_home=self.docker_conan_home,
docker_platform_param=self.docker_platform_param,
lcow_user_workaround=self.lcow_user_workaround,
test_folder=self.test_folder)
r.run(pull_image=not pulled_docker_images[docker_image],
docker_entry_script=self.docker_entry_script)
pulled_docker_images[docker_image] = True
def _get_docker_image(self, build):
if self._docker_image:
docker_image = self._docker_image
else:
compiler_name = build.settings.get("compiler", "")
compiler_version = build.settings.get("compiler.version", "")
docker_image = self._autodetect_docker_base_image(
compiler_name, compiler_version)
arch = build.settings.get("arch", "") or build.settings.get(
"arch_build", "")
if self.docker_32_images and arch == "x86":
build.settings["arch_build"] = "x86"
docker_arch_suffix = "x86"
elif arch != "x86" and arch != "x86_64":
docker_arch_suffix = arch
else:
docker_arch_suffix = None
if docker_arch_suffix and "-" not in docker_image:
docker_image = "%s-%s" % (docker_image, docker_arch_suffix)
return docker_image
@staticmethod
def _autodetect_docker_base_image(compiler_name, compiler_version):
if compiler_name not in ["clang", "gcc"]:
raise Exception("Docker image cannot be autodetected for "
"the compiler %s" % compiler_name)
if compiler_name == "gcc" and Version(compiler_version) > Version("5"):
compiler_version = Version(compiler_version).major(fill=False)
return "lasote/conan%s%s" % (compiler_name,
compiler_version.replace(".", ""))
def _get_channel(self, specified_channel, stable_channel):
if self.stable_branch_pattern:
stable_patterns = [self.stable_branch_pattern]
else:
stable_patterns = ["master$", "release*", "stable*"]
branch = self.ci_manager.get_branch()
self.printer.print_message("Branch detected", branch)
for pattern in stable_patterns:
prog = re.compile(pattern)
if branch and prog.match(branch):
self.printer.print_message(
"Info",
"Redefined channel by CI branch matching with '%s', "
"setting CONAN_CHANNEL to '%s'" %
(pattern, stable_channel))
return stable_channel
return specified_channel
def setUp(self):
self.conan_api = MockConanAPI()
self.output = TestBufferConanOutput()
self.printer = Printer(self.output.write)
def test_valid_config_with_args(self):
manager = ConfigManager(self.conan_api, Printer())
manager.install('https://github.com/bincrafters/conan-config.git',
'-b master')
