def test_PackageCollection_configuration():
config = yaml.load('''
global:
export:
owner : initech
channel : devel
setting_overrides:
version : devel
checkout : master
package_instances:
- conanfile: PackA/conanfile.py
- conanfile: PackB/conanfile.py
dependency_overrides:
- 'PackA/2.6@initech/stable'
- conanfile : PackC/conanfile.py
setting_overrides:
version : '1.1'
checkout : 'v1.1'
- conanfile: PackA/conanfile.py
channel : 'stable'
setting_overrides:
version : '2.6'
checkout : 'v2.6'
''',
Loader=yaml.SafeLoader)
pc = util.PackageCollection()
pc.load(config)
assert len(pc.package_instances) == 4
assert pc.package_instances[0].conanfile == 'PackA/conanfile.py'
assert pc.package_instances[0].setting_overrides['version'] == 'devel'
assert pc.package_instances[0].setting_overrides['checkout'] == 'master'
assert pc.package_instances[1].conanfile == 'PackB/conanfile.py'
assert pc.package_instances[1].setting_overrides['version'] == 'devel'
assert pc.package_instances[1].setting_overrides['checkout'] == 'master'
assert pc.package_instances[2].conanfile == 'PackC/conanfile.py'
assert pc.package_instances[2].setting_overrides['version'] == '1.1'
assert pc.package_instances[2].setting_overrides['checkout'] == 'v1.1'
assert pc.package_instances[3].conanfile == 'PackA/conanfile.py'
assert pc.package_instances[3].setting_overrides['version'] == '2.6'
assert pc.package_instances[3].setting_overrides['checkout'] == 'v2.6'
def main(print_default_configuration, print_configuration, packages, owner,
channel, create, config_file):
"""
Export conan packages to the local cache.
This script exports conan recipes to the local cache by default. Consumers
can then use the packages, but must specify `--build missing` the first time
they install.
This script allows conan package recipes to be generated from a base recipe. For example, if you wanted to create
a package for the last three major releases of LibA, the recipe files for each version would probably look identical
expect for the version number and the commit that gets checked out (unless the build system changes). Rather than
write three different recipes, you can just write a single recipe for the latest release. This script can then
generate recipes for the other versions.
To do this, every package that gets exported by this script starts with a
base recipe. You can then specify settings to override (see Configuration
Settings below) to create a new recipe instance. If no override settings are
specified, then the recipe instance will just be the base recipe.
base recipe -> [override settigns] -> recipe instance
This is convenient because it allows you to maintain a collection of valid
recipe files that can be tested and used with just conan, and then use this
script to export variations on these recipes.
Configuration:
The exported packages can be configured using a YAML file and passing it as an argument to this script.
Configuration Settings:
\b
global
a namespace for global package settings (setting that will be applied to all packages)
global.export
a namespace for setting related to exporting packages.
global.export.owner
the default package owner
global.export.channel
the default package channel
package_instances
a namespace used for overriding package settings on a per-package basis.
package_instances[i].conanfile
path to the conan recipe to use.
package_instances[i].name
name used to refer to package in the script.
this is not the name that will be used in the package recipe (see settings_overrides)
package_instances[i].setting_overrides
a dict of key/value pairs that will be added to the recipe instance to override settings in the base recipe.
the key/value pairs are arbitrary. if the key is a setting in the base recipe, it will be overridden. if it is
not, it will be added.
package_instances[i].setting_overrides.name
override the package name
package_instances[i].setting_overrides.version
override the package version
package_instances[i].setting_overrides.checkout
override the git reference that will be checked out to build the package. must be supported by the base recipe.
export-tool
a namespace used for setting for this script
export-tool.scratch-folder
the name of the directory used for writing data to disk (logs, temp files, etc)
export-tool.packages_to_export
a list of packages to export. by default, all packages will be exported.
this will be overridden by the --packages option.
Example Configuration File:
\b
export-packages:
packages_to_export: all
scratch-folder: _package-exports.d
global:
export:
channel: devel
owner: MyOrg
package_instances:
- conanfile: /path/to/LibA/conanfile.py
name: LibA
setting_overrides:
version: "3.1"
checkout: "v3.1"
- conanfile: /path/to/LibA/conanfile.py
name: LibA
setting_overrides:
version: "3.2"
checkout: "v3.2"
This will create and export a recipe instance for every conanfile.py found in the recipies directory,
and two recipe instances for the LibA package.
"""
default_configuration_file = prog_path.parent / f"{prog_path.stem}-default-config.yaml"
if default_configuration_file.exists():
default_configuration_text = default_configuration_file.read_text()
else:
default_configuration_text = ""
print(
util.WARN +
f"WARNING: did not find default configuration file '{str(default_configuration_file)}'."
+ util.EOL)
config = yaml.load(default_configuration_text, Loader=yaml.BaseLoader)
if print_default_configuration:
print("# Default Configuration")
print(yaml.dump(config))
sys.exit(0)
for file in config_file:
util.update_dict(
config, yaml.load(Path(file).read_text(), Loader=yaml.BaseLoader))
if not 'global' in config:
config['global'] = dict()
if not 'export' in config['global']:
config['global']['export'] = dict()
if owner:
config['global']['export']['owner'] = owner
if channel:
config['global']['export']['channel'] = channel
if packages:
if packages in ['all', 'instances-only']:
config[prog_path.stem]['packages_to_export'] = packages
else:
config[prog_path.stem]['packages_to_export'] = packages.split(",")
packages_to_export = config[prog_path.stem].get('packages_to_export',
'all')
if not 'package_instances' in config:
config['package_instances'] = list()
# sort of a hack. we want to let the user specify that only the instances explicitly listed
# should be exported
if packages_to_export != "instances-only":
for file in Path("recipes").glob("*/conanfile.py"):
config['package_instances'].append({
'conanfile':
str(file.absolute()),
'name':
str(file.parent.stem)
})
for file in Path("recipes").glob("*/conanfile-latest.py"):
config['package_instances'].append({
'conanfile':
str(file.absolute()),
'name':
str(file.parent.stem)
})
else:
packages_to_export = "all"
# make sure all conanfiles are specified with absolute path
for instance in config["package_instances"]:
instance["conanfile"] = str(Path(instance["conanfile"]).absolute())
if print_configuration:
print("# Complete Configuration")
print(yaml.dump(config))
sys.exit(0)
pc = util.PackageCollection()
pc.load(config)
scratch_folder_path = Path(pc.config[prog_path.stem]["scratch-folder"])
if scratch_folder_path.exists():
shutil.rmtree(str(scratch_folder_path))
scratch_folder_path.mkdir()
packages_to_export = [
p.name
for p in util.filter_packages(packages_to_export, pc.package_instances)
]
if create:
print("Creating packages")
with (Path(pc.config[prog_path.stem]["scratch-folder"]) /
"conan_export.out").open('w') as f:
pc.create_packages(config=packages_to_export, stdout=f)
print("Done")
else:
print("Exporting packages")
with (Path(pc.config[prog_path.stem]["scratch-folder"]) /
"conan_export.out").open('w') as f:
pc.export_packages(config=packages_to_export, stdout=f)
print("Done")
def main(
print_configuration,
tests,
profile,
unit_tests,
skip_export,
clear_cache,
owner,
channel,
config_file,
):
"""
Test conan-based build of packages.
This script will export each of the packages in the repository to the local cache and then attempt to build them and run their unit tests.
This is useful for testing that the latest commits on a given component did not break any of the components that depend on it. For example,
if libA is updated, this script will run an verify that all of the packages dependeing on libA can still build and pass their unit tests.
To do this, every package that gets exported by this script starts with a
base recipe. You can then specify settings to override (see Configuration
Settings below) to create a new recipe instance. If no override settings are
specified, then the recipe instance will just be the base recipe.
base recipe -> [override settigns] -> recipe instance
Configuration:
The exported packages can be configured using a YAML file and passing it as an argument to this script.
Configuration Settings:
\b
global
a namespace for global package settings (setting that will be applied to all packages)
global.export
a namespace for setting related to exporting packages.
global.export.owner
the default package owner for every exported package.
global.export.channel
the default package channel for every exported packages.
package_instances
a namespace used for overriding package settings on a per-package basis.
package_instances[i].conanfile
path to the conan recipe to use.
package_instances[i].name
name used to refer to package in the script.
this is not the name that will be used in the package recipe (see settings_overrides)
package_instances[i].setting_overrides
a dict of key/value pairs that will be added to the recipe instance to override settings in the base recipe.
the key/value pairs are arbitrary. if the key is a setting in the base recipe, it will be overridden. if it is
not, it will be added.
package_instances[i].setting_overrides.name
override the package name
package_instances[i].setting_overrides.version
override the package version
package_instances[i].setting_overrides.checkout
override the git reference that will be checked out to build the package. must be supported by the base recipe.
test-integrations
a namespace used for setting for this script
test-integrations.scratch-folder
the name of the directory used for writing data to disk (logs, temp files, etc)
test-integrations.packages_to_test
a list of packages to test. by default, all 'marked' packages will be tested.
to mark a package, place an empty file named 'test-integrations' in the package
recipe folder next to the conanfile.py.
Example Configuration Files:
To test all package recipes in their current state:
\b
test-integrations:
packages_to_test: all
scratch-folder: _test-integrations.d
global:
export:
channel: devel
owner: MyOrg
This will export all packages in the repository and then try to build and test each package
that is marked with a test-integrations file.
To test the latest commits on master for all packages.
\b
test-integrations:
packages_to_test: all
scratch-folder: _test-integrations.d
global:
setting_overrides:
version: latest
checkout: master
export:
channel: devel
owner: MyOrg
dependency_overrides:
- '*/*@MyOrg/devel -> [name]/latest@MyOrg/devel'
This will export all packages in the repository using the latest commit on master with the version
set to 'latest'. Since the package recipes (probably) won't refer to the 'latest' version,
We specify a dependency override that will replace all dependencies from the MyOrg/devel
owner/channel with a reference to the latest version.
"""
config = util.load_config(
str(prog_path.parent / f"{prog_path.stem}-default-config.yaml"),
config_file)
if owner:
config['global']['export']['owner'] = owner
if channel:
config['global']['export']['channel'] = channel
# collect all packages to export.
for file in Path("recipes").glob("*/conanfile.py"):
config['package_instances'].append({
'conanfile': str(file.absolute()),
'name': str(file.parent.stem)
})
for file in Path("recipes").glob("*/conanfile-latest.py"):
config['package_instances'].append({
'conanfile': str(file.absolute()),
'name': str(file.parent.stem)
})
# make sure all conanfiles are specified with absolute path
for instance in config["package_instances"]:
instance["conanfile"] = str(Path(instance["conanfile"]).absolute())
if tests:
if tests in ['all']:
config[prog_path.stem]['packages_to_test'] = tests
else:
config[prog_path.stem]['packages_to_test'] = tests.split(",")
if config[prog_path.stem].get('packages_to_test', 'all') == 'all':
packages = list()
for marker in Path("recipes").glob("*/test-integrations"):
packages.append(marker.parent.stem)
config[prog_path.stem]['packages_to_test'] = packages
if print_configuration:
print("# Complete Configuration")
print(yaml.dump(config))
sys.exit(0)
pc = util.PackageCollection()
pc.load(config)
scratch_folder_path = Path(pc.config[prog_path.stem]["scratch-folder"])
if not scratch_folder_path.exists():
scratch_folder_path.mkdir()
scratch_build_folder_path = scratch_folder_path / "builds"
util.remove_path(scratch_build_folder_path)
scratch_build_folder_path.mkdir()
conan_cache_path = scratch_folder_path.absolute() / "conan_cache"
# set teh CONAN_USER_HOME environment variable to a local directory so we don't
# interfere with the global cache
os.environ["CONAN_USER_HOME"] = str(conan_cache_path)
if skip_export:
print(
"Skipping export step. Packages currently in the local cache will be tested."
)
else:
if clear_cache:
print(
f"Removing conan cache (removing directory {str(conan_cache_path/'data')})."
)
util.remove_path(conan_cache_path / "data")
print("Exporting packages")
pc.export_packages()
with (Path(pc.config[prog_path.stem]["scratch-folder"]) /
"conan_export.out").open('w') as f:
pc.export_packages(stdout=f)
print("Done")
packages_to_test = util.filter_packages(
config[prog_path.stem]['packages_to_test'], pc.package_instances)
print(util.EMPH + "Testing packages: " +
", ".join([p.name for p in packages_to_test]) + util.EOL)
with util.working_directory(scratch_build_folder_path):
results = [
test_package(package, profile, unit_tests)
for package in packages_to_test
]
print("Done")
num_tests = len(results)
num_failed = sum(results)
print("\n")
print("Tested " + str(num_tests) + " Packages")
if num_failed > 0:
print(str(num_failed) + " Failed")
else:
print("All Tests Passed")
def test_filter_packages():
config = yaml.load('''
global:
export:
owner : initech
channel : devel
setting_overrides:
version : devel
checkout : master
''',
Loader=yaml.SafeLoader)
pc = util.PackageCollection()
pc.load(config)
res = subprocess.run("conan remove -f MyPackage",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
res = subprocess.run("conan remove -f PackA",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
res = subprocess.run("conan remove -f PackB",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
res = subprocess.run("conan remove -f PackC",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
conanfile_text = '''
from conans import ConanFile, CMake
import os, glob
class ConanPackage(ConanFile):
name = "Name"
version = "master"
checkout = "master"
generators = "virtualenv"
requires = "boost/1.69.0@conan/stable"
build_requires = "cmake_installer/3.13.0@conan/stable"
git_url_basename = "Missing"
repo_name = None
def source(self):
pass
def build(self):
pass
def package(self):
pass
def package_info(self):
pass
'''
with util.in_temporary_directory() as d:
path = Path('PackA')
conan = path / 'conanfile.py'
path.mkdir()
conan.touch()
with open(conan, 'w') as f:
f.write(conanfile_text.replace("Name", "PackA"))
path = Path('PackB')
conan = path / 'conanfile.py'
path.mkdir()
conan.touch()
conanfile_text = conanfile_text.replace("boost/1.69.0@conan/stable",
"PackA/devel@initech/devel")
with open(conan, 'w') as f:
f.write(conanfile_text.replace("Name", "PackB"))
path = Path('PackC')
conan = path / 'conanfile.py'
path.mkdir()
conan.touch()
conanfile_text = conanfile_text.replace("PackA/devel@initech/devel",
"PackB/devel@initech/devel")
with open(conan, 'w') as f:
f.write(conanfile_text.replace("Name", "PackC"))
pc.add_from_conan_recipe_collection('.')
pc.export_packages()
assert len(util.filter_packages('all', pc.package_instances)) == 3
assert len(util.filter_packages('none', pc.package_instances)) == 0
assert len(util.filter_packages(['PackA', 'PackB'],
pc.package_instances)) == 2
assert len(
util.filter_packages(['PackA', 'PackB', 'PackD'],
pc.package_instances)) == 2
assert len(util.filter_packages({'include': 'Pack'},
pc.package_instances)) == 3
assert len(
util.filter_packages({'include': 'Pack.*'}, pc.package_instances)) == 3
assert len(util.filter_packages({'include': '.*A'},
pc.package_instances)) == 1
assert len(util.filter_packages({'exclude': 'None'},
pc.package_instances)) == 0
assert len(
util.filter_packages({
'include': '.*',
'exclude': 'None'
}, pc.package_instances)) == 3
assert len(
util.filter_packages({
'include': '.*',
'exclude': '.*A$'
}, pc.package_instances)) == 2
assert util.a_deps_on_b("PackA/devel@initech/devel",
"boost/1.69.0@conan/stable")
assert not util.a_deps_on_b("PackA/devel@initech/devel",
"PackB/devel@initech/devel")
assert not util.a_deps_on_b("PackA/devel@initech/devel",
"PackC/devel@initech/devel")
assert util.a_deps_on_b("PackB/devel@initech/devel",
"PackA/devel@initech/devel")
assert util.a_deps_on_b("PackB/devel@initech/devel",
"boost/1.69.0@conan/stable"
) # PackA depends on boost, so PackB will too
assert util.a_deps_on_b("PackC/devel@initech/devel",
"PackB/devel@initech/devel")
assert util.a_deps_on_b("PackC/devel@initech/devel",
"PackA/devel@initech/devel")
assert util.a_deps_on_b("PackC/devel@initech/devel",
"boost/1.69.0@conan/stable")
def test_PackageCollection_export_packages():
conanfile_text = '''
from conans import ConanFile, CMake
import os, glob
class ConanPackage(ConanFile):
name = "Name Here"
version = "Unknown"
checkout = "Unknown"
generators = "virtualenv"
requires = "boost/1.69.0@conan/stable"
build_requires = "cmake_installer/3.13.0@conan/stable"
git_url_basename = "Missing"
repo_name = None
def source(self):
pass
def build(self):
pass
def package(self):
pass
def package_info(self):
pass
'''
res = subprocess.run("conan remove PackA -f",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
res = subprocess.run("conan remove PackB -f",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
res = subprocess.run("conan remove PackC -f",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
with util.in_temporary_directory() as d:
path_a = Path('PackA')
path_b = Path('PackB')
path_c = Path('PackC')
conan_a = path_a / 'conanfile.py'
conan_b = path_b / 'conanfile.py'
conan_c = path_c / 'conanfile.py'
path_a.mkdir()
conan_a.touch()
path_b.mkdir()
conan_b.touch()
path_c.mkdir()
conan_c.touch()
with open(conan_a, 'w') as f:
f.write(conanfile_text.replace("Name Here", "PackA"))
with open(conan_b, 'w') as f:
f.write(conanfile_text.replace("Name Here", "PackB"))
with open(conan_c, 'w') as f:
f.write(conanfile_text.replace("Name Here", "PackC"))
config = yaml.load('''
global:
export:
owner : initech
channel : devel
setting_overrides:
version : master
checkout : master
''',
Loader=yaml.SafeLoader)
pc = util.PackageCollection()
pc.load(config)
pc.add_from_conan_recipe_collection('.')
pc.export_packages()
res = subprocess.run("conan search PackA",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
assert re.match(".*PackA/master@initech/devel", str(res.stdout))
res = subprocess.run("conan search PackB",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
assert re.match(".*PackB/master@initech/devel", str(res.stdout))
res = subprocess.run("conan search PackC",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
assert re.match(".*PackC/master@initech/devel", str(res.stdout))
def test_PackageCollection_build():
config = yaml.load('''
global:
export:
owner : initech
channel : devel
setting_overrides:
version : devel
checkout : master
''',
Loader=yaml.SafeLoader)
pc = util.PackageCollection()
pc.load(config)
with util.in_temporary_directory() as d:
path_a = Path('PackA')
path_b = Path('PackB')
path_c = Path('PackC')
conan_a = path_a / 'conanfile.py'
conan_b = path_b / 'conan-recipe.py' # should NOT get picked up
conan_c = path_c / 'conanfile.py'
path_a.mkdir()
conan_a.touch()
path_b.mkdir()
conan_b.touch()
path_c.mkdir()
conan_c.touch()
with pytest.raises(Exception):
pc.add_from_conan_recipe_collection('/missing')
names = [p.parent.name for p in Path('.').glob('*/conanfile.py')]
pc.add_from_conan_recipe_collection('.')
assert len(pc.package_instances) == 2
idx = names.index('PackA')
assert pc.package_instances[idx].conanfile == str(
Path('PackA/conanfile.py').resolve())
assert pc.package_instances[idx].setting_overrides[
'version'] == 'devel'
assert pc.package_instances[idx].setting_overrides[
'checkout'] == 'master'
idx = names.index('PackC')
assert pc.package_instances[idx].conanfile == str(
Path('PackC/conanfile.py').resolve())
assert pc.package_instances[idx].setting_overrides[
'version'] == 'devel'
assert pc.package_instances[idx].setting_overrides[
'checkout'] == 'master'
pc.add_from_conan_recipe_collection('.')
assert len(pc.package_instances) == 4
idx = names.index('PackA')
assert pc.package_instances[idx].conanfile == str(
Path('PackA/conanfile.py').resolve())
assert pc.package_instances[idx - 2].conanfile == str(
Path('PackA/conanfile.py').resolve())
idx = names.index('PackC')
assert pc.package_instances[idx].conanfile == str(
Path('PackC/conanfile.py').resolve())
assert pc.package_instances[idx - 2].conanfile == str(
Path('PackC/conanfile.py').resolve())