def validate_env(args): '''Entry Function''' variants = utils.make_variants(args.python_versions, args.build_types, args.mpi_types, args.cuda_versions) for variant in variants: try: env_config.load_env_config_files(args.env_config_file, variant) except OpenCEError as exc: raise OpenCEError(Error.VALIDATE_ENV, args.env_config_file, str(variant), exc.msg) from exc
def _validate_config_file(env_file, variants): '''Perform some validation on the environment file after loading it.''' # pylint: disable=import-outside-toplevel from open_ce import conda_utils try: if utils.is_url(env_file): env_file = utils.download_file(env_file) meta_obj = conda_utils.render_yaml(env_file, variants=variants, schema=_ENV_CONFIG_SCHEMA) if not (Key.packages.name in meta_obj.keys() or Key.imported_envs.name in meta_obj.keys()): raise OpenCEError(Error.CONFIG_CONTENT) meta_obj[Key.opence_env_file_path.name] = env_file return meta_obj except (Exception, SystemExit) as exc: #pylint: disable=broad-except raise OpenCEError(Error.ERROR, "Error in {}:\n {}".format(env_file, str(exc))) from exc
def _execute_in_container(container_name, command, container_tool): container_cmd = container_tool + " exec " if _use_root_user(container_tool): container_cmd += "--user root " container_cmd += container_name + " " # Change to home directory container_cmd += "bash -c 'cd " + _home_path( container_tool) + "; " + command + "'" if os.system(container_cmd): raise OpenCEError(Error.BUILD_IN_CONTAINER, container_name)
def _detect_cycle(self): cycle_print = "" cycles = networkx.simple_cycles(self._tree) for cycle in cycles: if any(node.build_command for node in cycle): cycle_print += " -> ".join( node.build_command.recipe if node. build_command else str(node.packages) for node in cycle + [cycle[0]]) + "\n" if cycle_print: raise OpenCEError(Error.BUILD_TREE_CYCLE, cycle_print)
def _clone_repo(self, git_url, repo_dir, env_config_data, package): """ Clone the git repo at repository. """ # Priority is given to command line specified tag, if it is not # specified then package specific tag, and when even that is not specified # then top level git tag specified for env in the env file. And if nothing is # at all specified then fall back to default branch of the repo. git_tag = self._git_tag_for_env git_tag_for_package = None if git_tag is None: git_tag_for_package = package.get(env_config.Key.git_tag.name, None) if package else None if git_tag_for_package: git_tag = git_tag_for_package else: git_tag = env_config_data.get( env_config.Key.git_tag_for_env.name, None) if env_config_data else None clone_successful = utils.git_clone( git_url, git_tag, repo_dir, self._git_up_to_date and not git_tag_for_package) if clone_successful: patches = package.get(env_config.Key.patches.name, []) if package else [] if len(patches) > 0: cur_dir = os.getcwd() os.chdir(repo_dir) for patch in patches: if os.path.isabs(patch) and os.path.exists(patch): patch_file = patch else: # Look for patch relative to where the Open-CE environment file is patch_file = os.path.join( os.path.dirname( env_config_data.get( env_config.Key.opence_env_file_path.name)), patch) if utils.is_url(patch_file): patch_file = utils.download_file(patch_file) patch_apply_cmd = "git apply {}".format(patch_file) log.info("Patch apply command: %s", patch_apply_cmd) patch_apply_res = os.system(patch_apply_cmd) if patch_apply_res != 0: os.chdir(cur_dir) shutil.rmtree(repo_dir) raise OpenCEError( Error.PATCH_APPLICATION, patch, package[env_config.Key.feedstock.name]) os.chdir(cur_dir)
def _create_remote_deps(self, dep_graph): #pylint: disable=import-outside-toplevel from open_ce import conda_utils deps = {dep for dep in dep_graph.nodes() if dep.build_command is None} seen = set() try: while deps: node = deps.pop() ancestor_build_cmds = { x.build_command for x in networkx.ancestors(dep_graph, node) if x.build_command is not None } channels = [] ancestor_channels = [] for cmd in ancestor_build_cmds: ancestor_channels += cmd.channels for channel in node.channels + ancestor_channels + self._channels: if not channel in channels: channels += [channel] for package in node.packages: package_name = utils.remove_version(package) if package_name in seen: continue seen.add(package_name) # Pass in channels ordered by priority. package_info = conda_utils.get_latest_package_info( channels, package) # package_info is empty for a virtual package. # As of now, this is just one case of package_info being empty. if package_info == "": continue dep_graph.add_node(DependencyNode({package})) for dep in package_info['dependencies']: dep_name = utils.remove_version(dep) local_dest = { dest_node for dest_node in dep_graph.nodes() if dep_name in map(utils.remove_version, dest_node.packages) } if local_dest: dep_graph.add_edge(node, local_dest.pop()) else: new_dep = DependencyNode({dep}) dep_graph.add_edge(node, new_dep) deps.add(new_dep) return dep_graph except OpenCEError as err: raise OpenCEError(Error.REMOTE_PACKAGE_DEPENDENCIES, deps, err.msg) from err
def add_licenses(self, conda_env_file): """ Add all of the license information for every package within a given conda environment file. """ # Create a conda environment from the provided file time_stamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") conda_env_path = os.path.join(os.getcwd(), "license_env_file_" + time_stamp) cli = "conda env create -p {} -f {}".format(conda_env_path, conda_env_file) ret_code, std_out, std_err = utils.run_command_capture(cli) if not ret_code: raise OpenCEError(Error.GET_LICENSES, cli, std_out, std_err) # Get all of the licenses from the file self._add_licenses_from_environment(conda_env_path) # Delete the generated conda environment cli = "conda env remove -p {}".format(conda_env_path) ret_code, std_out, std_err = utils.run_command_capture(cli) if not ret_code: raise OpenCEError(Error.GET_LICENSES, cli, std_out, std_err)
def build_runtime_container_image(args): """ Create a runtime image which will have a conda environment created using locally built conda packages and environment file. """ if not args.container_tool: raise OpenCEError(Error.NO_CONTAINER_TOOL_FOUND) local_conda_channel = os.path.abspath(args.local_conda_channel) if not os.path.exists(local_conda_channel): raise OpenCEError(Error.INCORRECT_INPUT_PATHS) if not os.path.exists(os.path.join(local_conda_channel, TEMP_FILES)): os.mkdir(os.path.join(local_conda_channel, TEMP_FILES)) for conda_env_file in parse_arg_list(args.conda_env_files): conda_env_file = os.path.abspath(conda_env_file) if not os.path.exists(conda_env_file): raise OpenCEError(Error.INCORRECT_INPUT_PATHS) # Copy the conda environment file into the TEMP_FILES dir inside local # conda channel with a new name and modify it conda_env_runtime_filename = os.path.splitext( os.path.basename(conda_env_file))[0] + '-runtime.yaml' conda_env_runtime_file = os.path.join(local_conda_channel, TEMP_FILES, conda_env_runtime_filename) create_copy(conda_env_file, conda_env_runtime_file) utils.replace_conda_env_channels(conda_env_runtime_file, r'file:.*', "file:/{}".format(TARGET_DIR)) image_version = utils.get_open_ce_version(conda_env_file) image_name = build_image(args.local_conda_channel, os.path.basename(conda_env_runtime_file), args.container_tool, image_version, args.container_build_args) print("Docker image with name {} is built successfully.".format( image_name)) cleanup(local_conda_channel)
def validate_dict_schema(dictionary, schema): '''Recursively validate a dictionary's schema.''' for k, (schema_type, required) in schema.items(): if k not in dictionary: if required: raise OpenCEError( Error.ERROR, "Required key {} was not found in {}".format( k, dictionary)) continue if isinstance(schema_type, list): if dictionary[ k] is not None: #Handle if the yaml file has an empty list for this key. validate_type(dictionary[k], list) for value in dictionary[k]: validate_type(value, schema_type[0]) else: validate_type(dictionary[k], schema_type) for k in dictionary: if not k in schema: raise OpenCEError( Error.ERROR, "Unexpected key {} was found in {}".format(k, dictionary))
def _validate_config_file(env_file, variants): '''Perform some validation on the environment file after loading it.''' # pylint: disable=import-outside-toplevel from open_ce import conda_utils try: if utils.is_url(env_file): env_file = utils.download_file(env_file) # First, partially render yaml to validate builder version number. version_check_obj = conda_utils.render_yaml( env_file, permit_undefined_jinja=True) if Key.builder_version.name in version_check_obj.keys(): if not conda_utils.version_matches_spec( version_check_obj.get(Key.builder_version.name)): raise OpenCEError( Error.SCHEMA_VERSION_MISMATCH, env_file, version_check_obj.get(Key.builder_version.name), open_ce_version) meta_obj = None try: meta_obj = conda_utils.render_yaml(env_file, variants=variants, schema=_ENV_CONFIG_SCHEMA) if not (Key.packages.name in meta_obj.keys() or Key.imported_envs.name in meta_obj.keys()): raise OpenCEError(Error.CONFIG_CONTENT) meta_obj[Key.opence_env_file_path.name] = env_file except OpenCEError as exc: if Key.builder_version.name not in version_check_obj.keys(): show_warning(Error.SCHEMA_VERSION_NOT_FOUND, env_file, Key.builder_version.name) raise exc return meta_obj except (Exception, SystemExit) as exc: #pylint: disable=broad-except raise OpenCEError(Error.ERROR, "Error in {}:\n {}".format(env_file, str(exc))) from exc
def _generate_dockerfile_name(build_types, cuda_version): ''' Ensure we have valid combinations. I.e. Specify a valid cuda version ''' if 'cuda' in build_types: dockerfile = os.path.join(BUILD_CUDA_IMAGE_PATH, "Dockerfile.cuda-" + cuda_version) build_image_path = BUILD_CUDA_IMAGE_PATH if not os.path.isfile(dockerfile): raise OpenCEError(Error.UNSUPPORTED_CUDA, cuda_version) else: #Build with cpu based image dockerfile = os.path.join(BUILD_IMAGE_PATH, "Dockerfile") build_image_path = BUILD_IMAGE_PATH return build_image_path, dockerfile
def test_feedstock_entry(args): '''Entry Function''' if not args.conda_env_files: raise OpenCEError(Error.CONDA_ENV_FILE_REQUIRED) if args.working_directory: feedstock = os.path.basename(os.path.abspath(args.working_directory)) else: feedstock = os.path.basename(os.getcwd()) test_results = {feedstock: []} for conda_env_file in inputs.parse_arg_list(args.conda_env_files): test_results[feedstock] += test_feedstock(conda_env_file, args.test_labels, args.test_working_dir, args.working_directory) process_test_results(test_results, args.test_working_dir, args.test_labels)
def download_file(url, filename=None): ''' Downloads a file from a url string. Raises an OpenCE Error if an exception is encountered. ''' retval = None try: if not filename: download_path = tempfile.NamedTemporaryFile( suffix=os.path.basename(url), delete=False).name else: download_path = tempfile.NamedTemporaryFile(suffix=filename, delete=False).name retval, _ = urllib.request.urlretrieve(url, filename=download_path) except Exception as exc: # pylint: disable=broad-except raise OpenCEError(Error.FILE_DOWNLOAD, url, str(exc)) from exc return retval
def get_licenses(args): """ Entry point for `get licenses`. """ if not args.conda_env_files: raise OpenCEError(Error.CONDA_ENV_FILE_REQUIRED) gen = LicenseGenerator() for conda_env_file in parse_arg_list(args.conda_env_files): gen.add_licenses(conda_env_file) gen.write_licenses_file(args.output_folder) if args.template_files: for template_file in parse_arg_list(args.template_files): gen.gen_file_from_template(template_file, args.output_folder)
def validate_config(args): '''Entry Function Validates a lits of Open-CE env files against a conda build config for a given set of variants. ''' # Importing BuildTree is intentionally done here because it checks for the # existence of conda-build as BuildTree uses conda_build APIs. from open_ce.build_tree import construct_build_tree # pylint: disable=import-outside-toplevel for env_file in list(args.env_config_file): #make a copy of the env_file list log.info('Validating %s for %s', args.conda_build_configs, env_file) try: args.env_config_file = [env_file] _ = construct_build_tree(args) except OpenCEError as err: raise OpenCEError(Error.VALIDATE_CONFIG, args.conda_build_configs, env_file, err.msg) from err log.info('Successfully validated %s for %s', args.conda_build_configs, env_file)
def build_image(local_conda_channel, conda_env_file): """ Build a docker image from the Dockerfile in RUNTIME_IMAGE_PATH. Returns a result code and the name of the new image. """ image_name = REPO_NAME + ":" + IMAGE_NAME + "-" + str(os.getuid()) build_cmd = DOCKER_TOOL + " build " build_cmd += "-f " + os.path.join(RUNTIME_IMAGE_PATH, "Dockerfile") + " " build_cmd += "-t " + image_name + " " build_cmd += "--build-arg OPENCE_USER="******" " build_cmd += "--build-arg LOCAL_CONDA_CHANNEL=" + local_conda_channel + " " build_cmd += "--build-arg CONDA_ENV_FILE=" + conda_env_file + " " build_cmd += "--build-arg TARGET_DIR=" + TARGET_DIR + " " build_cmd += BUILD_CONTEXT print("Docker build command: ", build_cmd) if os.system(build_cmd): raise OpenCEError(Error.BUILD_IMAGE, image_name) return image_name
def _detect_cycle(self, max_cycles=10): extract_build_tree = [ x.build_command_dependencies for x in self.build_commands ] cycles = [] for start in range( len(self.build_commands) ): # Check to see if there are any cycles that start anywhere in the tree. cycles += find_all_cycles(extract_build_tree, start) if len(cycles) >= max_cycles: break if cycles: cycle_print = "\n".join([ " -> ".join([self.build_commands[i].recipe for i in cycle]) for cycle in cycles[:min(max_cycles, len(cycles))] ]) if len(cycles) > max_cycles: cycle_print += "\nCycles truncated after {}...".format( max_cycles) raise OpenCEError(Error.BUILD_TREE_CYCLE, cycle_print)
def load_package_config(config_file=None, variants=None, recipe_path=None, permit_undefined_jinja=False): ''' Check for a config file. If the user does not provide a recipe config file as an argument, it will be assumed that there is only one recipe to build, and it is in the directory called 'recipe'. ''' # pylint: disable=import-outside-toplevel from open_ce import conda_utils if recipe_path: recipe_name = os.path.basename(os.getcwd()) build_config_data = { 'recipes': [{ 'name': recipe_name, 'path': recipe_path }] } elif not config_file and not os.path.exists( utils.DEFAULT_RECIPE_CONFIG_FILE): recipe_name = os.path.basename(os.getcwd()) build_config_data = { 'recipes': [{ 'name': recipe_name, 'path': 'recipe' }] } else: if not config_file: config_file = utils.DEFAULT_RECIPE_CONFIG_FILE if not os.path.exists(config_file): raise OpenCEError(Error.CONFIG_FILE, config_file) build_config_data = conda_utils.render_yaml( config_file, variants, permit_undefined_jinja=permit_undefined_jinja) return build_config_data, config_file
def validate_build_tree(tree, external_deps, start_nodes=None): ''' Check a build tree for dependency compatability. ''' # Importing BuildTree is intentionally done here because it checks for the # existence of conda-build as BuildTree uses conda_build APIs. from open_ce import build_tree # pylint: disable=import-outside-toplevel packages = [package for recipe in build_tree.traverse_build_commands(tree, start_nodes) for package in recipe.packages] channels = {channel for recipe in build_tree.traverse_build_commands(tree, start_nodes) for channel in recipe.channels} env_channels = {channel for node in tree.nodes() for channel in node.channels} deps = build_tree.get_installable_packages(tree, external_deps, start_nodes, True) pkg_args = " ".join(["\"{}\"".format(utils.generalize_version(dep)) for dep in deps if not utils.remove_version(dep) in packages]) channel_args = " ".join({"-c \"{}\"".format(channel) for channel in channels.union(env_channels)}) cli = "conda create --dry-run -n test_conda_dependencies {} {}".format(channel_args, pkg_args) ret_code, std_out, std_err = utils.run_command_capture(cli) if not ret_code: raise OpenCEError(Error.VALIDATE_BUILD_TREE, cli, std_out, std_err)
def conda_package_info(channels, package): ''' Get conda package info. ''' # Call "conda search --info" through conda's cli.python_api channel_args = sum((["-c", channel] for channel in channels), []) search_args = ["--info", generalize_version(package)] + channel_args # Setting the logging level allows us to ignore unnecessary output conda_logger = getLogger("conda.common.io") conda_logger.setLevel(ERROR) std_out, _, ret_code = conda.cli.python_api.run_command( conda.cli.python_api.Commands.SEARCH, search_args, use_exception_handler=True, stderr=None) # Parsing the normal output from "conda search --info" instead of using the json flag. Using the json # flag adds a lot of extra time due to a slow regex in the conda code that is attempting to parse out # URL tokens. entries = [] for entry in std_out.split("\n\n"): _, file_name, rest = entry.partition("file name") if file_name: entry = open_ce.yaml_utils.load(file_name + rest) # Convert time string into a timestamp (if there is a timestamp) if "timestamp" in entry: entry["timestamp"] = datetime.timestamp( datetime.strptime(entry["timestamp"], '%Y-%m-%d %H:%M:%S %Z')) else: entry["timestamp"] = 0 if not entry["dependencies"]: entry["dependencies"] = [] entries.append(entry) if ret_code: raise OpenCEError(Error.CONDA_PACKAGE_INFO, str(search_args), std_out) return entries
def validate_env_config(conda_build_config, env_config_files, variants, repository_folder): ''' Validates a lits of Open-CE env files against a conda build config for a given set of variants. ''' for variant in variants: for env_file in env_config_files: print('Validating {} for {} : {}'.format(conda_build_config, env_file, variant)) try: _ = build_tree.BuildTree([env_file], variant['python'], variant['build_type'], variant['mpi_type'], variant['cudatoolkit'], repository_folder=repository_folder, conda_build_config=conda_build_config) except OpenCEError as err: raise OpenCEError(Error.VALIDATE_CONFIG, conda_build_config, env_file, variant, err.msg) from err print('Successfully validated {} for {} : {}'.format( conda_build_config, env_file, variant))
def process_test_results(test_results, output_folder="./", test_labels=None): """ This function writes test results to a file, displays failed tests to stdout, and throws an exception if there are test failures. """ label_string = "" if test_labels: label_string = "with labels: {}".format(str(test_labels)) test_suites = [ TestSuite("Open-CE tests for {} {}".format(feedstock, label_string), test_results[feedstock]) for feedstock in test_results ] with open(os.path.join(output_folder, utils.DEFAULT_TEST_RESULT_FILE), 'w') as outfile: outfile.write(to_xml_report_string(test_suites)) failed_tests = [ x for key in test_results for x in test_results[key] if x.is_failure() ] if failed_tests: raise OpenCEError( Error.FAILED_TESTS, len(failed_tests), str([failed_test.name for failed_test in failed_tests])) log.info("All tests passed!")
def build_image(build_image_path, dockerfile, container_tool, cuda_version=None, container_build_args=""): """ Build a container image from the Dockerfile in BUILD_IMAGE_PATH. Returns a result code and the name of the new image. """ if cuda_version: image_name = REPO_NAME + ":" + IMAGE_NAME + "-cuda" + cuda_version else: image_name = REPO_NAME + ":" + IMAGE_NAME + "-cpu" build_cmd = container_tool + " build " build_cmd += "-f " + dockerfile + " " build_cmd += "-t " + image_name + " " if not _use_root_user(container_tool): build_cmd += "--build-arg BUILD_ID=" + str(os.getuid()) + " " build_cmd += "--build-arg GROUP_ID=" + str(os.getgid()) + " " build_cmd += container_build_args + " " build_cmd += build_image_path if os.system(build_cmd): raise OpenCEError(Error.BUILD_IMAGE, image_name) return image_name
def build_image(local_conda_channel, conda_env_file, container_tool, image_version, container_build_args=""): """ Build a container image from the Dockerfile in RUNTIME_IMAGE_PATH. Returns a result code and the name of the new image. """ variant = os.path.splitext(conda_env_file)[0].replace( utils.CONDA_ENV_FILENAME_PREFIX, "", 1) variant = variant.replace("-runtime", "") image_name = REPO_NAME + ":" + image_version + "-" + variant # Docker version on ppc64le rhel7 doesn't allow Dockerfiles to be out of build context. # Hence, copying it in temp_dir inside the build context. This isn't needed with newer # docker versions or podman but to be consistent, doing this in all cases. dockerfile_path = os.path.join(local_conda_channel, TEMP_FILES, "Dockerfile") runtime_img_file = _get_runtime_image_file(container_tool) create_copy(runtime_img_file, dockerfile_path) build_cmd = container_tool + " build " build_cmd += "-f " + dockerfile_path + " " build_cmd += "-t " + image_name + " " build_cmd += "--build-arg OPENCE_USER="******" " build_cmd += "--build-arg LOCAL_CONDA_CHANNEL=" + "./ " build_cmd += "--build-arg CONDA_ENV_FILE=" + os.path.join( TEMP_FILES, conda_env_file) + " " build_cmd += "--build-arg TARGET_DIR=" + TARGET_DIR + " " build_cmd += container_build_args + " " build_cmd += local_conda_channel print("Container build command: ", build_cmd) if os.system(build_cmd): raise OpenCEError(Error.BUILD_IMAGE, image_name) return image_name
def _run_tests(build_tree, test_labels, conda_env_files): """ Run through all of the tests within a build tree for the given conda environment files. Args: build_tree (BuildTree): The build tree containing the tests conda_env_files (dict): A dictionary where the key is a variant string and the value is the name of a conda environment file. """ failed_tests = [] # Run test commands for each conda environment that was generated for variant_string, conda_env_file in conda_env_files.items(): test_feedstocks = build_tree.get_test_feedstocks(variant_string) if test_feedstocks: print("\n*** Running tests within the " + os.path.basename(conda_env_file) + " conda environment ***\n") for feedstock in test_feedstocks: print("Running tests for " + feedstock) failed_tests += test_feedstock.test_feedstock(conda_env_file, test_labels=test_labels, working_directory=feedstock) test_feedstock.display_failed_tests(failed_tests) if failed_tests: raise OpenCEError(Error.FAILED_TESTS, len(failed_tests))
def _set_local_src_dir(local_src_dir_arg, recipe, recipe_config_file): """ Set the LOCAL_SRC_DIR environment variable if local_src_dir is specified. """ # Local source directory provided as command line argument has higher priority # than what is specified in build-config.yaml if local_src_dir_arg: local_src_dir = os.path.expanduser(local_src_dir_arg) elif 'local_src_dir' in recipe: local_src_dir = os.path.expanduser(recipe.get('local_src_dir')) # If a relative path is specified, it should be in relation to the config file if not os.path.isabs(local_src_dir): local_src_dir = os.path.join(os.path.dirname(os.path.abspath(recipe_config_file)), local_src_dir) else: local_src_dir = None if local_src_dir: if not os.path.exists(local_src_dir): raise OpenCEError(Error.LOCAL_SRC_DIR, local_src_dir) os.environ["LOCAL_SRC_DIR"] = local_src_dir else: if 'LOCAL_SRC_DIR' in os.environ: del os.environ['LOCAL_SRC_DIR']
def git_clone(git_url, git_tag, location, up_to_date=False): ''' Clone a git repository and checkout a certain branch. ''' clone_cmd = "git clone " + git_url + " " + location print("Clone cmd: ", clone_cmd) clone_result = os.system(clone_cmd) cur_dir = os.getcwd() clone_successful = clone_result == 0 if clone_successful: if git_tag: os.chdir(location) if up_to_date: git_tag = get_branch_of_tag(git_tag) checkout_cmd = "git checkout " + git_tag print("Checkout branch/tag command: ", checkout_cmd) checkout_res = os.system(checkout_cmd) os.chdir(cur_dir) clone_successful = checkout_res == 0 else: raise OpenCEError(Error.CLONE_REPO, git_url) return clone_successful
def _start_container(container_name, container_tool): if os.system(container_tool + " start " + container_name): raise OpenCEError(Error.START_CONTAINER, container_name)
def _copy_to_container(src, dest, container_name, container_tool): if os.system(container_tool + " cp " + src + " " + container_name + ":" + dest): raise OpenCEError(Error.COPY_DIR_TO_CONTAINER, src, container_name)
def __init__(self, env_config_files, python_versions, build_types, mpi_types, cuda_versions, repository_folder="./", channels=None, git_location=utils.DEFAULT_GIT_LOCATION, git_tag_for_env=utils.DEFAULT_GIT_TAG, conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG, packages=None): self._env_config_files = env_config_files self._repository_folder = repository_folder self._channels = channels if channels else [] self._git_location = git_location self._git_tag_for_env = git_tag_for_env self._conda_build_config = conda_build_config self._external_dependencies = dict() self._conda_env_files = dict() self._test_feedstocks = dict() self._initial_package_indices = [] # Create a dependency tree that includes recipes for every combination # of variants. self._possible_variants = utils.make_variants(python_versions, build_types, mpi_types, cuda_versions) self.build_commands = [] for variant in self._possible_variants: try: build_commands, external_deps = self._create_all_commands( variant) except OpenCEError as exc: raise OpenCEError(Error.CREATE_BUILD_TREE, exc.msg) from exc variant_string = utils.variant_string(variant["python"], variant["build_type"], variant["mpi_type"], variant["cudatoolkit"]) self._external_dependencies[variant_string] = external_deps # Add dependency tree information to the packages list and # remove build commands from build_commands that are already in self.build_commands build_commands, package_indices = _add_build_command_dependencies( build_commands, self.build_commands, len(self.build_commands)) self.build_commands += build_commands self._detect_cycle() # If the packages argument is provided, find the indices into the build_commands for all # of the packages that were requested. variant_package_indices = [] if packages: for package in packages: if package in package_indices: variant_package_indices += package_indices[package] else: print("INFO: No recipes were found for " + package + " for variant " + variant_string) else: for package in package_indices: variant_package_indices += package_indices[package] self._initial_package_indices += variant_package_indices validate_config.validate_build_tree(self.build_commands, external_deps, variant_package_indices) installable_packages = get_installable_packages( self.build_commands, external_deps, variant_package_indices) filtered_packages = [ package for package in installable_packages if utils.remove_version(package) in package_indices or utils.remove_version(package) in utils.KNOWN_VARIANT_PACKAGES ] self._conda_env_files[variant_string] = CondaEnvFileGenerator( filtered_packages) self._test_feedstocks[variant_string] = [] for build_command in traverse_build_commands( self.build_commands, variant_package_indices): self._test_feedstocks[variant_string].append( build_command.repository)