def test_conda_env_file_for_only_selected_py():
'''
Tests that the conda env file is generated only for selected configurations.
'''
python_versions = "3.7"
build_types = "cpu,cuda"
mpi_types = "openmpi,system"
mock_build_tree = TestBuildTree([], python_versions, build_types,
mpi_types, external_deps)
mock_build_tree.build_commands = sample_build_commands[
2:4] # Build cmds for py3.7
output_dir = os.path.join(test_dir, '../condabuild')
mock_conda_env_file_generator = TestCondaEnvFileGenerator(
python_versions, build_types, mpi_types, None, output_dir)
expected_channels = ["file:/{}".format(output_dir), "defaults"]
actual_channels = mock_conda_env_file_generator.channels
assert actual_channels == expected_channels
variants = utils.make_variants(python_versions, build_types, mpi_types)
expected_keys = [
utils.variant_string(variant['python'], variant['build_type'],
variant['mpi_type']) for variant in variants
]
actual_keys = list(mock_conda_env_file_generator.dependency_dict.keys())
assert Counter(actual_keys) == Counter(expected_keys)
for build_command in mock_build_tree:
mock_conda_env_file_generator.update_conda_env_file_content(
build_command, mock_build_tree)
mock_conda_env_file_generator.write_conda_env_files(TMP_OPENCE_DIR)
# Conda env files should be generated only for py3.7-cpu-openmpi and py3.7-cuda-system variants
expected_files_keys = [
utils.variant_string("3.7", "cpu", "openmpi"),
utils.variant_string("3.7", "cuda", "system")
]
# Check if conda env files are created for expected_files_keys
for file_keys in expected_files_keys:
cuda_env_file = os.path.join(
TMP_OPENCE_DIR, "{}{}.yaml".format(utils.CONDA_ENV_FILENAME_PREFIX,
file_keys))
assert os.path.exists(cuda_env_file)
# Check that no other env file exists other than the two expected ones
for (_, _, files) in os.walk(TMP_OPENCE_DIR, topdown=True):
assert len(files) == 2
cleanup()
assert not os.path.exists(TMP_OPENCE_DIR)
def get_external_dependencies(self, variant):
'''Return the list of external dependencies for the given variant.'''
variant_string = utils.variant_string(variant["python"],
variant["build_type"],
variant["mpi_type"],
variant["cudatoolkit"])
return self._external_dependencies.get(variant_string, [])
def validate_env_config(conda_build_config, env_config_files, variants,
repository_folder):
'''
Validates a lits of Open-CE env files against a conda build config
for a given set of variants.
'''
for variant in variants:
for env_file in env_config_files:
print('Validating {} for {} : {}'.format(conda_build_config,
env_file, variant))
try:
recipes = build_tree.BuildTree(
[env_file],
variant['python'],
variant['build_type'],
variant['mpi_type'],
variant['cudatoolkit'],
repository_folder=repository_folder,
conda_build_config=conda_build_config)
variant_string = utils.variant_string(variant["python"],
variant["build_type"],
variant["mpi_type"],
variant["cudatoolkit"])
validate_build_tree(recipes, variant_string)
except OpenCEError as err:
raise OpenCEError(Error.VALIDATE_CONFIG, conda_build_config,
env_file, variant, err.msg) from err
print('Successfully validated {} for {} : {}'.format(
conda_build_config, env_file, variant))
def main(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
variants = utils.make_variants(args.python_versions, args.build_types,
args.mpi_types, args.cuda_versions)
pr_branch = utils.get_output("git log -1 --format='%H'")
utils.run_and_log("git remote set-head origin -a")
default_branch = utils.get_output(
"git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'"
)
variant_build_results = dict()
for variant in variants:
utils.run_and_log("git checkout {}".format(default_branch))
master_build_config_data, master_config = _get_configs(variant)
master_build_numbers = _get_build_numbers(master_build_config_data,
master_config, variant)
utils.run_and_log("git checkout {}".format(pr_branch))
pr_build_config_data, pr_config = _get_configs(variant)
current_pr_build_numbers = _get_build_numbers(pr_build_config_data,
pr_config, variant)
print("Build Info for Variant: {}".format(variant))
print("Current PR Build Info: {}".format(current_pr_build_numbers))
print("Master Branch Build Info: {}".format(master_build_numbers))
#No build numbers can go backwards without a version change.
for package in master_build_numbers:
if package in current_pr_build_numbers and current_pr_build_numbers[
package]["version"] == master_build_numbers[package][
"version"]:
assert int(current_pr_build_numbers[package]["number"]) >= int(
master_build_numbers[package]["number"]
), "If the version doesn't change, the build number can't be reduced."
#If packages are added or removed, don't require a version change
if set(master_build_numbers.keys()) != set(
current_pr_build_numbers.keys()):
return
#At least one package needs to increase the build number or change the version.
checks = [
current_pr_build_numbers[package]["version"] !=
master_build_numbers[package]["version"]
or int(current_pr_build_numbers[package]["number"]) > int(
master_build_numbers[package]["number"])
for package in master_build_numbers
]
variant_build_results[utils.variant_string(
variant["python"], variant["build_type"], variant["mpi_type"],
variant["cudatoolkit"])] = any(checks)
assert any(
variant_build_results.values()
), "At least one package needs to increase the build number or change the version in at least one variant."
def _initialize_dependency_dict(self):
variants = utils.make_variants(self.python_versions, self.build_types,
self.mpi_types)
for variant in variants:
key = utils.variant_string(variant['python'],
variant['build_type'],
variant['mpi_type'])
self.dependency_dict[key] = set()
def name(self):
"""
Returns a name representing the Build Command
"""
result = self.recipe
variant_string = utils.variant_string(self.python, self.build_type, self.mpi_type, self.cudatoolkit)
if variant_string:
result += "-" + variant_string
result = result.replace(".", "-")
result = result.replace("_", "-")
return result
def __init__(self,
env_config_files,
python_versions,
build_types,
mpi_types,
cuda_versions,
repository_folder="./",
git_location=utils.DEFAULT_GIT_LOCATION,
git_tag_for_env=utils.DEFAULT_GIT_TAG,
conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG,
test_labels=None):
self._env_config_files = env_config_files
self._repository_folder = repository_folder
self._git_location = git_location
self._git_tag_for_env = git_tag_for_env
self._conda_build_config = conda_build_config
self._external_dependencies = dict()
self._conda_env_files = dict()
self._test_commands = dict()
self._test_labels = test_labels
# Create a dependency tree that includes recipes for every combination
# of variants.
self._possible_variants = utils.make_variants(python_versions,
build_types, mpi_types,
cuda_versions)
self.build_commands = []
for variant in self._possible_variants:
try:
build_commands, external_deps, test_commands = self._create_all_commands(
variant)
except OpenCEError as exc:
raise OpenCEError(Error.CREATE_BUILD_TREE, exc.msg) from exc
variant_string = utils.variant_string(variant["python"],
variant["build_type"],
variant["mpi_type"],
variant["cudatoolkit"])
self._external_dependencies[variant_string] = external_deps
self._test_commands[variant_string] = test_commands
# Add dependency tree information to the packages list and
# remove build commands from build_commands that are already in self.build_commands
build_commands = _add_build_command_dependencies(
build_commands, self.build_commands, len(self.build_commands))
self.build_commands += build_commands
installable_packages = self.get_installable_packages(
variant_string)
self._conda_env_files[variant_string] = CondaEnvFileGenerator(
installable_packages)
self._detect_cycle()
def validate_conda_env_files(py_versions=utils.DEFAULT_PYTHON_VERS,
build_types=utils.DEFAULT_BUILD_TYPES,
mpi_types=utils.DEFAULT_MPI_TYPES):
# Check if conda env files are created for given python versions and build variants
variants = utils.make_variants(py_versions, build_types, mpi_types)
for variant in variants:
cuda_env_file = os.path.join(os.getcwd(), utils.DEFAULT_OUTPUT_FOLDER,
"{}{}.yaml".format(utils.CONDA_ENV_FILENAME_PREFIX,
utils.variant_string(variant['python'], variant['build_type'], variant['mpi_type'], variant['cudatoolkit'])))
assert os.path.exists(cuda_env_file)
# Remove the file once it's existence is verified
os.remove(cuda_env_file)
def test_conda_env_file_content():
'''
Tests that the conda env file content are being populated correctly
'''
python_versions = "3.6,3.7"
build_types = "cpu,cuda"
mpi_types = "openmpi,system"
mock_build_tree = TestBuildTree([], python_versions, build_types,
mpi_types, external_deps)
mock_build_tree.build_commands = sample_build_commands
output_dir = os.path.join(test_dir, '../condabuild')
mock_conda_env_file_generator = TestCondaEnvFileGenerator(
python_versions, build_types, mpi_types, ["some channel"], output_dir)
expected_channels = [
"file:/{}".format(output_dir), "some channel", "defaults"
]
actual_channels = mock_conda_env_file_generator.channels
assert actual_channels == expected_channels
variants = utils.make_variants(python_versions, build_types, mpi_types)
expected_keys = [
utils.variant_string(variant['python'], variant['build_type'],
variant['mpi_type']) for variant in variants
]
actual_keys = list(mock_conda_env_file_generator.dependency_dict.keys())
assert Counter(actual_keys) == Counter(expected_keys)
for build_command in mock_build_tree:
mock_conda_env_file_generator.update_conda_env_file_content(
build_command, mock_build_tree)
files_generated_for_keys = []
validate_dependencies(mock_conda_env_file_generator, expected_keys,
files_generated_for_keys)
mock_conda_env_file_generator.write_conda_env_files(TMP_OPENCE_DIR)
# Check if conda env files are created for all variants
for key in files_generated_for_keys:
cuda_env_file = os.path.join(
TMP_OPENCE_DIR, "{}{}.yaml".format(utils.CONDA_ENV_FILENAME_PREFIX,
key))
assert os.path.exists(cuda_env_file)
cleanup()
assert not os.path.exists(TMP_OPENCE_DIR)
def update_conda_env_file_content(self, build_command, build_tree):
"""
This function updates dependency dictionary for each build command with
its dependencies both internal and external.
"""
key = utils.variant_string(build_command.python,
build_command.build_type,
build_command.mpi_type)
self._update_deps_lists(build_command.run_dependencies, key)
self._update_deps_lists(build_command.packages, key)
variant = {
'python': build_command.python,
'build_type': build_command.build_type,
'mpi_type': build_command.mpi_type
}
self._update_deps_lists(
build_tree.get_external_dependencies(str(variant)), key)
