def test_conda_env_file_for_only_selected_py(): ''' Tests that the conda env file is generated only for selected configurations. ''' python_versions = "3.7" build_types = "cpu,cuda" mpi_types = "openmpi,system" mock_build_tree = TestBuildTree([], python_versions, build_types, mpi_types, external_deps) mock_build_tree.build_commands = sample_build_commands[ 2:4] # Build cmds for py3.7 output_dir = os.path.join(test_dir, '../condabuild') mock_conda_env_file_generator = TestCondaEnvFileGenerator( python_versions, build_types, mpi_types, None, output_dir) expected_channels = ["file:/{}".format(output_dir), "defaults"] actual_channels = mock_conda_env_file_generator.channels assert actual_channels == expected_channels variants = utils.make_variants(python_versions, build_types, mpi_types) expected_keys = [ utils.variant_string(variant['python'], variant['build_type'], variant['mpi_type']) for variant in variants ] actual_keys = list(mock_conda_env_file_generator.dependency_dict.keys()) assert Counter(actual_keys) == Counter(expected_keys) for build_command in mock_build_tree: mock_conda_env_file_generator.update_conda_env_file_content( build_command, mock_build_tree) mock_conda_env_file_generator.write_conda_env_files(TMP_OPENCE_DIR) # Conda env files should be generated only for py3.7-cpu-openmpi and py3.7-cuda-system variants expected_files_keys = [ utils.variant_string("3.7", "cpu", "openmpi"), utils.variant_string("3.7", "cuda", "system") ] # Check if conda env files are created for expected_files_keys for file_keys in expected_files_keys: cuda_env_file = os.path.join( TMP_OPENCE_DIR, "{}{}.yaml".format(utils.CONDA_ENV_FILENAME_PREFIX, file_keys)) assert os.path.exists(cuda_env_file) # Check that no other env file exists other than the two expected ones for (_, _, files) in os.walk(TMP_OPENCE_DIR, topdown=True): assert len(files) == 2 cleanup() assert not os.path.exists(TMP_OPENCE_DIR)
def get_external_dependencies(self, variant): '''Return the list of external dependencies for the given variant.''' variant_string = utils.variant_string(variant["python"], variant["build_type"], variant["mpi_type"], variant["cudatoolkit"]) return self._external_dependencies.get(variant_string, [])
def validate_env_config(conda_build_config, env_config_files, variants, repository_folder): ''' Validates a lits of Open-CE env files against a conda build config for a given set of variants. ''' for variant in variants: for env_file in env_config_files: print('Validating {} for {} : {}'.format(conda_build_config, env_file, variant)) try: recipes = build_tree.BuildTree( [env_file], variant['python'], variant['build_type'], variant['mpi_type'], variant['cudatoolkit'], repository_folder=repository_folder, conda_build_config=conda_build_config) variant_string = utils.variant_string(variant["python"], variant["build_type"], variant["mpi_type"], variant["cudatoolkit"]) validate_build_tree(recipes, variant_string) except OpenCEError as err: raise OpenCEError(Error.VALIDATE_CONFIG, conda_build_config, env_file, variant, err.msg) from err print('Successfully validated {} for {} : {}'.format( conda_build_config, env_file, variant))
def main(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) variants = utils.make_variants(args.python_versions, args.build_types, args.mpi_types, args.cuda_versions) pr_branch = utils.get_output("git log -1 --format='%H'") utils.run_and_log("git remote set-head origin -a") default_branch = utils.get_output( "git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'" ) variant_build_results = dict() for variant in variants: utils.run_and_log("git checkout {}".format(default_branch)) master_build_config_data, master_config = _get_configs(variant) master_build_numbers = _get_build_numbers(master_build_config_data, master_config, variant) utils.run_and_log("git checkout {}".format(pr_branch)) pr_build_config_data, pr_config = _get_configs(variant) current_pr_build_numbers = _get_build_numbers(pr_build_config_data, pr_config, variant) print("Build Info for Variant: {}".format(variant)) print("Current PR Build Info: {}".format(current_pr_build_numbers)) print("Master Branch Build Info: {}".format(master_build_numbers)) #No build numbers can go backwards without a version change. for package in master_build_numbers: if package in current_pr_build_numbers and current_pr_build_numbers[ package]["version"] == master_build_numbers[package][ "version"]: assert int(current_pr_build_numbers[package]["number"]) >= int( master_build_numbers[package]["number"] ), "If the version doesn't change, the build number can't be reduced." #If packages are added or removed, don't require a version change if set(master_build_numbers.keys()) != set( current_pr_build_numbers.keys()): return #At least one package needs to increase the build number or change the version. checks = [ current_pr_build_numbers[package]["version"] != master_build_numbers[package]["version"] or int(current_pr_build_numbers[package]["number"]) > int( master_build_numbers[package]["number"]) for package in master_build_numbers ] variant_build_results[utils.variant_string( variant["python"], variant["build_type"], variant["mpi_type"], variant["cudatoolkit"])] = any(checks) assert any( variant_build_results.values() ), "At least one package needs to increase the build number or change the version in at least one variant."
def _initialize_dependency_dict(self): variants = utils.make_variants(self.python_versions, self.build_types, self.mpi_types) for variant in variants: key = utils.variant_string(variant['python'], variant['build_type'], variant['mpi_type']) self.dependency_dict[key] = set()
def name(self): """ Returns a name representing the Build Command """ result = self.recipe variant_string = utils.variant_string(self.python, self.build_type, self.mpi_type, self.cudatoolkit) if variant_string: result += "-" + variant_string result = result.replace(".", "-") result = result.replace("_", "-") return result
def __init__(self, env_config_files, python_versions, build_types, mpi_types, cuda_versions, repository_folder="./", git_location=utils.DEFAULT_GIT_LOCATION, git_tag_for_env=utils.DEFAULT_GIT_TAG, conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG, test_labels=None): self._env_config_files = env_config_files self._repository_folder = repository_folder self._git_location = git_location self._git_tag_for_env = git_tag_for_env self._conda_build_config = conda_build_config self._external_dependencies = dict() self._conda_env_files = dict() self._test_commands = dict() self._test_labels = test_labels # Create a dependency tree that includes recipes for every combination # of variants. self._possible_variants = utils.make_variants(python_versions, build_types, mpi_types, cuda_versions) self.build_commands = [] for variant in self._possible_variants: try: build_commands, external_deps, test_commands = self._create_all_commands( variant) except OpenCEError as exc: raise OpenCEError(Error.CREATE_BUILD_TREE, exc.msg) from exc variant_string = utils.variant_string(variant["python"], variant["build_type"], variant["mpi_type"], variant["cudatoolkit"]) self._external_dependencies[variant_string] = external_deps self._test_commands[variant_string] = test_commands # Add dependency tree information to the packages list and # remove build commands from build_commands that are already in self.build_commands build_commands = _add_build_command_dependencies( build_commands, self.build_commands, len(self.build_commands)) self.build_commands += build_commands installable_packages = self.get_installable_packages( variant_string) self._conda_env_files[variant_string] = CondaEnvFileGenerator( installable_packages) self._detect_cycle()
def validate_conda_env_files(py_versions=utils.DEFAULT_PYTHON_VERS, build_types=utils.DEFAULT_BUILD_TYPES, mpi_types=utils.DEFAULT_MPI_TYPES): # Check if conda env files are created for given python versions and build variants variants = utils.make_variants(py_versions, build_types, mpi_types) for variant in variants: cuda_env_file = os.path.join(os.getcwd(), utils.DEFAULT_OUTPUT_FOLDER, "{}{}.yaml".format(utils.CONDA_ENV_FILENAME_PREFIX, utils.variant_string(variant['python'], variant['build_type'], variant['mpi_type'], variant['cudatoolkit']))) assert os.path.exists(cuda_env_file) # Remove the file once it's existence is verified os.remove(cuda_env_file)
def test_conda_env_file_content(): ''' Tests that the conda env file content are being populated correctly ''' python_versions = "3.6,3.7" build_types = "cpu,cuda" mpi_types = "openmpi,system" mock_build_tree = TestBuildTree([], python_versions, build_types, mpi_types, external_deps) mock_build_tree.build_commands = sample_build_commands output_dir = os.path.join(test_dir, '../condabuild') mock_conda_env_file_generator = TestCondaEnvFileGenerator( python_versions, build_types, mpi_types, ["some channel"], output_dir) expected_channels = [ "file:/{}".format(output_dir), "some channel", "defaults" ] actual_channels = mock_conda_env_file_generator.channels assert actual_channels == expected_channels variants = utils.make_variants(python_versions, build_types, mpi_types) expected_keys = [ utils.variant_string(variant['python'], variant['build_type'], variant['mpi_type']) for variant in variants ] actual_keys = list(mock_conda_env_file_generator.dependency_dict.keys()) assert Counter(actual_keys) == Counter(expected_keys) for build_command in mock_build_tree: mock_conda_env_file_generator.update_conda_env_file_content( build_command, mock_build_tree) files_generated_for_keys = [] validate_dependencies(mock_conda_env_file_generator, expected_keys, files_generated_for_keys) mock_conda_env_file_generator.write_conda_env_files(TMP_OPENCE_DIR) # Check if conda env files are created for all variants for key in files_generated_for_keys: cuda_env_file = os.path.join( TMP_OPENCE_DIR, "{}{}.yaml".format(utils.CONDA_ENV_FILENAME_PREFIX, key)) assert os.path.exists(cuda_env_file) cleanup() assert not os.path.exists(TMP_OPENCE_DIR)
def update_conda_env_file_content(self, build_command, build_tree): """ This function updates dependency dictionary for each build command with its dependencies both internal and external. """ key = utils.variant_string(build_command.python, build_command.build_type, build_command.mpi_type) self._update_deps_lists(build_command.run_dependencies, key) self._update_deps_lists(build_command.packages, key) variant = { 'python': build_command.python, 'build_type': build_command.build_type, 'mpi_type': build_command.mpi_type } self._update_deps_lists( build_tree.get_external_dependencies(str(variant)), key)