def add_licenses_from_info_files(self, license_data):
"""
Add all of the license information from a list of open-ce-info licence info.
"""
info_file_licenses = utils.run_in_parallel(
self._get_licenses_from_info_file_helper, set(license_data))
self._licenses.update(filter(None, info_file_licenses))
def _create_nodes(self, variants):
'''
Create a recipe dictionary for each recipe needed for a given environment file.
'''
env_config_data_list = env_config.load_env_config_files(
self._env_config_files, variants)
feedstocks_seen = set()
external_deps = []
retval = graph.OpenCEGraph()
create_commands_args = []
# Find all conda_build_configs listed in environment files
conda_build_configs = []
for env_config_data in env_config_data_list:
conda_build_configs += [
utils.expanded_path(
config,
relative_to=env_config_data[
env_config.Key.opence_env_file_path.name])
for config in env_config_data.get(
env_config.Key.conda_build_configs.name, [])
]
utils.check_conda_build_configs_exist(conda_build_configs)
# Create recipe dictionaries for each repository in the environment file
for env_config_data in env_config_data_list:
channels = self._channels + env_config_data.get(
env_config.Key.channels.name, [])
feedstocks = env_config_data.get(env_config.Key.packages.name, [])
if not feedstocks:
feedstocks = []
for feedstock in feedstocks:
if _make_hash(feedstock) in feedstocks_seen:
continue
# Create arguments for call to _create_commands_helper
create_commands_args.append((variants, env_config_data,
conda_build_configs, feedstock))
feedstocks_seen.add(_make_hash(feedstock))
current_deps = env_config_data.get(
env_config.Key.external_dependencies.name, [])
for dep in current_deps:
#Add external dependencies as top level nodes in the graph.
new_dep = DependencyNode({dep}, channels=channels)
retval.add_node(new_dep)
if current_deps:
external_deps += current_deps
# Execute _create_commands_helper in parallel
commands = utils.run_in_parallel(self._create_commands_helper,
create_commands_args)
# Add the results of _create_commands_helper to the graph
for command in commands:
retval = networkx.compose(retval, command)
return retval, external_deps
def _add_licenses_from_environment(self, conda_env):
# For each meta-pkg within an environment, find its about.json file.
meta_file_args = [
(meta_file, conda_env)
for meta_file in os.listdir(os.path.join(conda_env, "conda-meta"))
if meta_file.endswith('.json')
]
licenses = utils.run_in_parallel(
self._add_licenses_from_environment_helper, meta_file_args)
local_licenses, info_file_packages = zip(*licenses)
self._licenses.update(filter(None, local_licenses))
self.add_licenses_from_info_files(
functools.reduce(operator.iconcat, info_file_packages, []))
def __init__(self,
env_config_files,
python_versions,
build_types,
mpi_types,
cuda_versions,
repository_folder="./",
channels=None,
git_location=utils.DEFAULT_GIT_LOCATION,
git_tag_for_env=utils.DEFAULT_GIT_TAG,
git_up_to_date=False,
conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG,
packages=None):
self._env_config_files = env_config_files
self._repository_folder = repository_folder
self._channels = channels if channels else []
self._git_location = git_location
self._git_tag_for_env = git_tag_for_env
self._git_up_to_date = git_up_to_date
self._conda_build_config = conda_build_config
self._external_dependencies = dict()
self._conda_env_files = dict()
self._test_feedstocks = dict()
self._initial_nodes = []
# Create a dependency tree that includes recipes for every combination
# of variants.
self._possible_variants = utils.make_variants(python_versions, build_types, mpi_types, cuda_versions)
self._tree = networkx.DiGraph()
validate_args = []
for variant in self._possible_variants:
try:
variant_tree, external_deps = self._create_nodes(variant)
variant_tree = _create_edges(variant_tree)
variant_tree = self._create_remote_deps(variant_tree)
self._tree = networkx.compose(self._tree, variant_tree)
except OpenCEError as exc:
raise OpenCEError(Error.CREATE_BUILD_TREE, exc.msg) from exc
variant_string = utils.variant_string(variant["python"], variant["build_type"],
variant["mpi_type"], variant["cudatoolkit"])
self._external_dependencies[variant_string] = external_deps
self._detect_cycle()
variant_start_nodes = {n for n,d in variant_tree.in_degree() if d==0}
# If the packages argument is provided, find the indices into the build_commands for all
# of the packages that were requested.
if packages:
for package in packages:
if not {n for n in traverse_build_commands(variant_tree, return_node=True) if package in n.packages}:
print("INFO: No recipes were found for " + package + " for variant " + variant_string)
variant_start_nodes = {n for n in traverse_build_commands(variant_tree, return_node=True)
if n.packages.intersection(packages)}
self._initial_nodes += variant_start_nodes
validate_args.append((self._tree, external_deps, variant_start_nodes))
self._conda_env_files[variant_string] = get_conda_file_packages(self._tree, external_deps, variant_start_nodes)
self._test_feedstocks[variant_string] = []
for build_command in traverse_build_commands(self._tree, variant_start_nodes):
self._test_feedstocks[variant_string].append(build_command.repository)
# Execute validate_build_tree in parallel
utils.run_in_parallel(validate_config.validate_build_tree, validate_args)