def validate_env(args):
'''Entry Function'''
variants = utils.make_variants(args.python_versions, args.build_types,
args.mpi_types, args.cuda_versions)
for variant in variants:
try:
env_config.load_env_config_files(args.env_config_file, variant)
except OpenCEError as exc:
raise OpenCEError(Error.VALIDATE_ENV, args.env_config_file,
str(variant), exc.msg) from exc
def _validate_config_file(env_file, variants):
'''Perform some validation on the environment file after loading it.'''
# pylint: disable=import-outside-toplevel
from open_ce import conda_utils
try:
if utils.is_url(env_file):
env_file = utils.download_file(env_file)
meta_obj = conda_utils.render_yaml(env_file,
variants=variants,
schema=_ENV_CONFIG_SCHEMA)
if not (Key.packages.name in meta_obj.keys()
or Key.imported_envs.name in meta_obj.keys()):
raise OpenCEError(Error.CONFIG_CONTENT)
meta_obj[Key.opence_env_file_path.name] = env_file
return meta_obj
except (Exception, SystemExit) as exc: #pylint: disable=broad-except
raise OpenCEError(Error.ERROR,
"Error in {}:\n {}".format(env_file,
str(exc))) from exc
def _execute_in_container(container_name, command, container_tool):
container_cmd = container_tool + " exec "
if _use_root_user(container_tool):
container_cmd += "--user root "
container_cmd += container_name + " "
# Change to home directory
container_cmd += "bash -c 'cd " + _home_path(
container_tool) + "; " + command + "'"
if os.system(container_cmd):
raise OpenCEError(Error.BUILD_IN_CONTAINER, container_name)
def _detect_cycle(self):
cycle_print = ""
cycles = networkx.simple_cycles(self._tree)
for cycle in cycles:
if any(node.build_command for node in cycle):
cycle_print += " -> ".join(
node.build_command.recipe if node.
build_command else str(node.packages)
for node in cycle + [cycle[0]]) + "\n"
if cycle_print:
raise OpenCEError(Error.BUILD_TREE_CYCLE, cycle_print)
def _clone_repo(self, git_url, repo_dir, env_config_data, package):
"""
Clone the git repo at repository.
"""
# Priority is given to command line specified tag, if it is not
# specified then package specific tag, and when even that is not specified
# then top level git tag specified for env in the env file. And if nothing is
# at all specified then fall back to default branch of the repo.
git_tag = self._git_tag_for_env
git_tag_for_package = None
if git_tag is None:
git_tag_for_package = package.get(env_config.Key.git_tag.name,
None) if package else None
if git_tag_for_package:
git_tag = git_tag_for_package
else:
git_tag = env_config_data.get(
env_config.Key.git_tag_for_env.name,
None) if env_config_data else None
clone_successful = utils.git_clone(
git_url, git_tag, repo_dir, self._git_up_to_date
and not git_tag_for_package)
if clone_successful:
patches = package.get(env_config.Key.patches.name,
[]) if package else []
if len(patches) > 0:
cur_dir = os.getcwd()
os.chdir(repo_dir)
for patch in patches:
if os.path.isabs(patch) and os.path.exists(patch):
patch_file = patch
else:
# Look for patch relative to where the Open-CE environment file is
patch_file = os.path.join(
os.path.dirname(
env_config_data.get(
env_config.Key.opence_env_file_path.name)),
patch)
if utils.is_url(patch_file):
patch_file = utils.download_file(patch_file)
patch_apply_cmd = "git apply {}".format(patch_file)
log.info("Patch apply command: %s", patch_apply_cmd)
patch_apply_res = os.system(patch_apply_cmd)
if patch_apply_res != 0:
os.chdir(cur_dir)
shutil.rmtree(repo_dir)
raise OpenCEError(
Error.PATCH_APPLICATION, patch,
package[env_config.Key.feedstock.name])
os.chdir(cur_dir)
def _create_remote_deps(self, dep_graph):
#pylint: disable=import-outside-toplevel
from open_ce import conda_utils
deps = {dep for dep in dep_graph.nodes() if dep.build_command is None}
seen = set()
try:
while deps:
node = deps.pop()
ancestor_build_cmds = {
x.build_command
for x in networkx.ancestors(dep_graph, node)
if x.build_command is not None
}
channels = []
ancestor_channels = []
for cmd in ancestor_build_cmds:
ancestor_channels += cmd.channels
for channel in node.channels + ancestor_channels + self._channels:
if not channel in channels:
channels += [channel]
for package in node.packages:
package_name = utils.remove_version(package)
if package_name in seen:
continue
seen.add(package_name)
# Pass in channels ordered by priority.
package_info = conda_utils.get_latest_package_info(
channels, package)
# package_info is empty for a virtual package.
# As of now, this is just one case of package_info being empty.
if package_info == "":
continue
dep_graph.add_node(DependencyNode({package}))
for dep in package_info['dependencies']:
dep_name = utils.remove_version(dep)
local_dest = {
dest_node
for dest_node in dep_graph.nodes() if dep_name in
map(utils.remove_version, dest_node.packages)
}
if local_dest:
dep_graph.add_edge(node, local_dest.pop())
else:
new_dep = DependencyNode({dep})
dep_graph.add_edge(node, new_dep)
deps.add(new_dep)
return dep_graph
except OpenCEError as err:
raise OpenCEError(Error.REMOTE_PACKAGE_DEPENDENCIES, deps,
err.msg) from err
def add_licenses(self, conda_env_file):
"""
Add all of the license information for every package within a given conda
environment file.
"""
# Create a conda environment from the provided file
time_stamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
conda_env_path = os.path.join(os.getcwd(), "license_env_file_" + time_stamp)
cli = "conda env create -p {} -f {}".format(conda_env_path, conda_env_file)
ret_code, std_out, std_err = utils.run_command_capture(cli)
if not ret_code:
raise OpenCEError(Error.GET_LICENSES, cli, std_out, std_err)
# Get all of the licenses from the file
self._add_licenses_from_environment(conda_env_path)
# Delete the generated conda environment
cli = "conda env remove -p {}".format(conda_env_path)
ret_code, std_out, std_err = utils.run_command_capture(cli)
if not ret_code:
raise OpenCEError(Error.GET_LICENSES, cli, std_out, std_err)
def build_runtime_container_image(args):
"""
Create a runtime image which will have a conda environment created
using locally built conda packages and environment file.
"""
if not args.container_tool:
raise OpenCEError(Error.NO_CONTAINER_TOOL_FOUND)
local_conda_channel = os.path.abspath(args.local_conda_channel)
if not os.path.exists(local_conda_channel):
raise OpenCEError(Error.INCORRECT_INPUT_PATHS)
if not os.path.exists(os.path.join(local_conda_channel, TEMP_FILES)):
os.mkdir(os.path.join(local_conda_channel, TEMP_FILES))
for conda_env_file in parse_arg_list(args.conda_env_files):
conda_env_file = os.path.abspath(conda_env_file)
if not os.path.exists(conda_env_file):
raise OpenCEError(Error.INCORRECT_INPUT_PATHS)
# Copy the conda environment file into the TEMP_FILES dir inside local
# conda channel with a new name and modify it
conda_env_runtime_filename = os.path.splitext(
os.path.basename(conda_env_file))[0] + '-runtime.yaml'
conda_env_runtime_file = os.path.join(local_conda_channel, TEMP_FILES,
conda_env_runtime_filename)
create_copy(conda_env_file, conda_env_runtime_file)
utils.replace_conda_env_channels(conda_env_runtime_file, r'file:.*',
"file:/{}".format(TARGET_DIR))
image_version = utils.get_open_ce_version(conda_env_file)
image_name = build_image(args.local_conda_channel,
os.path.basename(conda_env_runtime_file),
args.container_tool, image_version,
args.container_build_args)
print("Docker image with name {} is built successfully.".format(
image_name))
cleanup(local_conda_channel)
def validate_dict_schema(dictionary, schema):
'''Recursively validate a dictionary's schema.'''
for k, (schema_type, required) in schema.items():
if k not in dictionary:
if required:
raise OpenCEError(
Error.ERROR, "Required key {} was not found in {}".format(
k, dictionary))
continue
if isinstance(schema_type, list):
if dictionary[
k] is not None: #Handle if the yaml file has an empty list for this key.
validate_type(dictionary[k], list)
for value in dictionary[k]:
validate_type(value, schema_type[0])
else:
validate_type(dictionary[k], schema_type)
for k in dictionary:
if not k in schema:
raise OpenCEError(
Error.ERROR,
"Unexpected key {} was found in {}".format(k, dictionary))
def _validate_config_file(env_file, variants):
'''Perform some validation on the environment file after loading it.'''
# pylint: disable=import-outside-toplevel
from open_ce import conda_utils
try:
if utils.is_url(env_file):
env_file = utils.download_file(env_file)
# First, partially render yaml to validate builder version number.
version_check_obj = conda_utils.render_yaml(
env_file, permit_undefined_jinja=True)
if Key.builder_version.name in version_check_obj.keys():
if not conda_utils.version_matches_spec(
version_check_obj.get(Key.builder_version.name)):
raise OpenCEError(
Error.SCHEMA_VERSION_MISMATCH, env_file,
version_check_obj.get(Key.builder_version.name),
open_ce_version)
meta_obj = None
try:
meta_obj = conda_utils.render_yaml(env_file,
variants=variants,
schema=_ENV_CONFIG_SCHEMA)
if not (Key.packages.name in meta_obj.keys()
or Key.imported_envs.name in meta_obj.keys()):
raise OpenCEError(Error.CONFIG_CONTENT)
meta_obj[Key.opence_env_file_path.name] = env_file
except OpenCEError as exc:
if Key.builder_version.name not in version_check_obj.keys():
show_warning(Error.SCHEMA_VERSION_NOT_FOUND, env_file,
Key.builder_version.name)
raise exc
return meta_obj
except (Exception, SystemExit) as exc: #pylint: disable=broad-except
raise OpenCEError(Error.ERROR,
"Error in {}:\n {}".format(env_file,
str(exc))) from exc
def _generate_dockerfile_name(build_types, cuda_version):
'''
Ensure we have valid combinations. I.e. Specify a valid cuda version
'''
if 'cuda' in build_types:
dockerfile = os.path.join(BUILD_CUDA_IMAGE_PATH, "Dockerfile.cuda-" + cuda_version)
build_image_path = BUILD_CUDA_IMAGE_PATH
if not os.path.isfile(dockerfile):
raise OpenCEError(Error.UNSUPPORTED_CUDA, cuda_version)
else:
#Build with cpu based image
dockerfile = os.path.join(BUILD_IMAGE_PATH, "Dockerfile")
build_image_path = BUILD_IMAGE_PATH
return build_image_path, dockerfile
def test_feedstock_entry(args):
'''Entry Function'''
if not args.conda_env_files:
raise OpenCEError(Error.CONDA_ENV_FILE_REQUIRED)
if args.working_directory:
feedstock = os.path.basename(os.path.abspath(args.working_directory))
else:
feedstock = os.path.basename(os.getcwd())
test_results = {feedstock: []}
for conda_env_file in inputs.parse_arg_list(args.conda_env_files):
test_results[feedstock] += test_feedstock(conda_env_file,
args.test_labels,
args.test_working_dir,
args.working_directory)
process_test_results(test_results, args.test_working_dir, args.test_labels)
def download_file(url, filename=None):
'''
Downloads a file from a url string.
Raises an OpenCE Error if an exception is encountered.
'''
retval = None
try:
if not filename:
download_path = tempfile.NamedTemporaryFile(
suffix=os.path.basename(url), delete=False).name
else:
download_path = tempfile.NamedTemporaryFile(suffix=filename,
delete=False).name
retval, _ = urllib.request.urlretrieve(url, filename=download_path)
except Exception as exc: # pylint: disable=broad-except
raise OpenCEError(Error.FILE_DOWNLOAD, url, str(exc)) from exc
return retval
def get_licenses(args):
"""
Entry point for `get licenses`.
"""
if not args.conda_env_files:
raise OpenCEError(Error.CONDA_ENV_FILE_REQUIRED)
gen = LicenseGenerator()
for conda_env_file in parse_arg_list(args.conda_env_files):
gen.add_licenses(conda_env_file)
gen.write_licenses_file(args.output_folder)
if args.template_files:
for template_file in parse_arg_list(args.template_files):
gen.gen_file_from_template(template_file, args.output_folder)
def validate_config(args):
'''Entry Function
Validates a lits of Open-CE env files against a conda build config
for a given set of variants.
'''
# Importing BuildTree is intentionally done here because it checks for the
# existence of conda-build as BuildTree uses conda_build APIs.
from open_ce.build_tree import construct_build_tree # pylint: disable=import-outside-toplevel
for env_file in list(args.env_config_file): #make a copy of the env_file list
log.info('Validating %s for %s', args.conda_build_configs, env_file)
try:
args.env_config_file = [env_file]
_ = construct_build_tree(args)
except OpenCEError as err:
raise OpenCEError(Error.VALIDATE_CONFIG, args.conda_build_configs, env_file, err.msg) from err
log.info('Successfully validated %s for %s', args.conda_build_configs, env_file)
def build_image(local_conda_channel, conda_env_file):
"""
Build a docker image from the Dockerfile in RUNTIME_IMAGE_PATH.
Returns a result code and the name of the new image.
"""
image_name = REPO_NAME + ":" + IMAGE_NAME + "-" + str(os.getuid())
build_cmd = DOCKER_TOOL + " build "
build_cmd += "-f " + os.path.join(RUNTIME_IMAGE_PATH, "Dockerfile") + " "
build_cmd += "-t " + image_name + " "
build_cmd += "--build-arg OPENCE_USER="******" "
build_cmd += "--build-arg LOCAL_CONDA_CHANNEL=" + local_conda_channel + " "
build_cmd += "--build-arg CONDA_ENV_FILE=" + conda_env_file + " "
build_cmd += "--build-arg TARGET_DIR=" + TARGET_DIR + " "
build_cmd += BUILD_CONTEXT
print("Docker build command: ", build_cmd)
if os.system(build_cmd):
raise OpenCEError(Error.BUILD_IMAGE, image_name)
return image_name
def _detect_cycle(self, max_cycles=10):
extract_build_tree = [
x.build_command_dependencies for x in self.build_commands
]
cycles = []
for start in range(
len(self.build_commands)
): # Check to see if there are any cycles that start anywhere in the tree.
cycles += find_all_cycles(extract_build_tree, start)
if len(cycles) >= max_cycles:
break
if cycles:
cycle_print = "\n".join([
" -> ".join([self.build_commands[i].recipe for i in cycle])
for cycle in cycles[:min(max_cycles, len(cycles))]
])
if len(cycles) > max_cycles:
cycle_print += "\nCycles truncated after {}...".format(
max_cycles)
raise OpenCEError(Error.BUILD_TREE_CYCLE, cycle_print)
def load_package_config(config_file=None,
variants=None,
recipe_path=None,
permit_undefined_jinja=False):
'''
Check for a config file. If the user does not provide a recipe config
file as an argument, it will be assumed that there is only one
recipe to build, and it is in the directory called 'recipe'.
'''
# pylint: disable=import-outside-toplevel
from open_ce import conda_utils
if recipe_path:
recipe_name = os.path.basename(os.getcwd())
build_config_data = {
'recipes': [{
'name': recipe_name,
'path': recipe_path
}]
}
elif not config_file and not os.path.exists(
utils.DEFAULT_RECIPE_CONFIG_FILE):
recipe_name = os.path.basename(os.getcwd())
build_config_data = {
'recipes': [{
'name': recipe_name,
'path': 'recipe'
}]
}
else:
if not config_file:
config_file = utils.DEFAULT_RECIPE_CONFIG_FILE
if not os.path.exists(config_file):
raise OpenCEError(Error.CONFIG_FILE, config_file)
build_config_data = conda_utils.render_yaml(
config_file,
variants,
permit_undefined_jinja=permit_undefined_jinja)
return build_config_data, config_file
def validate_build_tree(tree, external_deps, start_nodes=None):
'''
Check a build tree for dependency compatability.
'''
# Importing BuildTree is intentionally done here because it checks for the
# existence of conda-build as BuildTree uses conda_build APIs.
from open_ce import build_tree # pylint: disable=import-outside-toplevel
packages = [package for recipe in build_tree.traverse_build_commands(tree, start_nodes)
for package in recipe.packages]
channels = {channel for recipe in build_tree.traverse_build_commands(tree, start_nodes) for channel in recipe.channels}
env_channels = {channel for node in tree.nodes() for channel in node.channels}
deps = build_tree.get_installable_packages(tree, external_deps, start_nodes, True)
pkg_args = " ".join(["\"{}\"".format(utils.generalize_version(dep)) for dep in deps
if not utils.remove_version(dep) in packages])
channel_args = " ".join({"-c \"{}\"".format(channel) for channel in channels.union(env_channels)})
cli = "conda create --dry-run -n test_conda_dependencies {} {}".format(channel_args, pkg_args)
ret_code, std_out, std_err = utils.run_command_capture(cli)
if not ret_code:
raise OpenCEError(Error.VALIDATE_BUILD_TREE, cli, std_out, std_err)
def conda_package_info(channels, package):
'''
Get conda package info.
'''
# Call "conda search --info" through conda's cli.python_api
channel_args = sum((["-c", channel] for channel in channels), [])
search_args = ["--info", generalize_version(package)] + channel_args
# Setting the logging level allows us to ignore unnecessary output
conda_logger = getLogger("conda.common.io")
conda_logger.setLevel(ERROR)
std_out, _, ret_code = conda.cli.python_api.run_command(
conda.cli.python_api.Commands.SEARCH,
search_args,
use_exception_handler=True,
stderr=None)
# Parsing the normal output from "conda search --info" instead of using the json flag. Using the json
# flag adds a lot of extra time due to a slow regex in the conda code that is attempting to parse out
# URL tokens.
entries = []
for entry in std_out.split("\n\n"):
_, file_name, rest = entry.partition("file name")
if file_name:
entry = open_ce.yaml_utils.load(file_name + rest)
# Convert time string into a timestamp (if there is a timestamp)
if "timestamp" in entry:
entry["timestamp"] = datetime.timestamp(
datetime.strptime(entry["timestamp"],
'%Y-%m-%d %H:%M:%S %Z'))
else:
entry["timestamp"] = 0
if not entry["dependencies"]:
entry["dependencies"] = []
entries.append(entry)
if ret_code:
raise OpenCEError(Error.CONDA_PACKAGE_INFO, str(search_args), std_out)
return entries
def validate_env_config(conda_build_config, env_config_files, variants,
repository_folder):
'''
Validates a lits of Open-CE env files against a conda build config
for a given set of variants.
'''
for variant in variants:
for env_file in env_config_files:
print('Validating {} for {} : {}'.format(conda_build_config,
env_file, variant))
try:
_ = build_tree.BuildTree([env_file],
variant['python'],
variant['build_type'],
variant['mpi_type'],
variant['cudatoolkit'],
repository_folder=repository_folder,
conda_build_config=conda_build_config)
except OpenCEError as err:
raise OpenCEError(Error.VALIDATE_CONFIG, conda_build_config,
env_file, variant, err.msg) from err
print('Successfully validated {} for {} : {}'.format(
conda_build_config, env_file, variant))
def process_test_results(test_results, output_folder="./", test_labels=None):
"""
This function writes test results to a file, displays failed tests to stdout,
and throws an exception if there are test failures.
"""
label_string = ""
if test_labels:
label_string = "with labels: {}".format(str(test_labels))
test_suites = [
TestSuite("Open-CE tests for {} {}".format(feedstock, label_string),
test_results[feedstock]) for feedstock in test_results
]
with open(os.path.join(output_folder, utils.DEFAULT_TEST_RESULT_FILE),
'w') as outfile:
outfile.write(to_xml_report_string(test_suites))
failed_tests = [
x for key in test_results for x in test_results[key] if x.is_failure()
]
if failed_tests:
raise OpenCEError(
Error.FAILED_TESTS, len(failed_tests),
str([failed_test.name for failed_test in failed_tests]))
log.info("All tests passed!")
def build_image(build_image_path, dockerfile, container_tool, cuda_version=None, container_build_args=""):
"""
Build a container image from the Dockerfile in BUILD_IMAGE_PATH.
Returns a result code and the name of the new image.
"""
if cuda_version:
image_name = REPO_NAME + ":" + IMAGE_NAME + "-cuda" + cuda_version
else:
image_name = REPO_NAME + ":" + IMAGE_NAME + "-cpu"
build_cmd = container_tool + " build "
build_cmd += "-f " + dockerfile + " "
build_cmd += "-t " + image_name + " "
if not _use_root_user(container_tool):
build_cmd += "--build-arg BUILD_ID=" + str(os.getuid()) + " "
build_cmd += "--build-arg GROUP_ID=" + str(os.getgid()) + " "
build_cmd += container_build_args + " "
build_cmd += build_image_path
if os.system(build_cmd):
raise OpenCEError(Error.BUILD_IMAGE, image_name)
return image_name
def build_image(local_conda_channel,
conda_env_file,
container_tool,
image_version,
container_build_args=""):
"""
Build a container image from the Dockerfile in RUNTIME_IMAGE_PATH.
Returns a result code and the name of the new image.
"""
variant = os.path.splitext(conda_env_file)[0].replace(
utils.CONDA_ENV_FILENAME_PREFIX, "", 1)
variant = variant.replace("-runtime", "")
image_name = REPO_NAME + ":" + image_version + "-" + variant
# Docker version on ppc64le rhel7 doesn't allow Dockerfiles to be out of build context.
# Hence, copying it in temp_dir inside the build context. This isn't needed with newer
# docker versions or podman but to be consistent, doing this in all cases.
dockerfile_path = os.path.join(local_conda_channel, TEMP_FILES,
"Dockerfile")
runtime_img_file = _get_runtime_image_file(container_tool)
create_copy(runtime_img_file, dockerfile_path)
build_cmd = container_tool + " build "
build_cmd += "-f " + dockerfile_path + " "
build_cmd += "-t " + image_name + " "
build_cmd += "--build-arg OPENCE_USER="******" "
build_cmd += "--build-arg LOCAL_CONDA_CHANNEL=" + "./ "
build_cmd += "--build-arg CONDA_ENV_FILE=" + os.path.join(
TEMP_FILES, conda_env_file) + " "
build_cmd += "--build-arg TARGET_DIR=" + TARGET_DIR + " "
build_cmd += container_build_args + " "
build_cmd += local_conda_channel
print("Container build command: ", build_cmd)
if os.system(build_cmd):
raise OpenCEError(Error.BUILD_IMAGE, image_name)
return image_name
def _run_tests(build_tree, test_labels, conda_env_files):
"""
Run through all of the tests within a build tree for the given conda environment files.
Args:
build_tree (BuildTree): The build tree containing the tests
conda_env_files (dict): A dictionary where the key is a variant string and the value
is the name of a conda environment file.
"""
failed_tests = []
# Run test commands for each conda environment that was generated
for variant_string, conda_env_file in conda_env_files.items():
test_feedstocks = build_tree.get_test_feedstocks(variant_string)
if test_feedstocks:
print("\n*** Running tests within the " + os.path.basename(conda_env_file) + " conda environment ***\n")
for feedstock in test_feedstocks:
print("Running tests for " + feedstock)
failed_tests += test_feedstock.test_feedstock(conda_env_file,
test_labels=test_labels,
working_directory=feedstock)
test_feedstock.display_failed_tests(failed_tests)
if failed_tests:
raise OpenCEError(Error.FAILED_TESTS, len(failed_tests))
def _set_local_src_dir(local_src_dir_arg, recipe, recipe_config_file):
"""
Set the LOCAL_SRC_DIR environment variable if local_src_dir is specified.
"""
# Local source directory provided as command line argument has higher priority
# than what is specified in build-config.yaml
if local_src_dir_arg:
local_src_dir = os.path.expanduser(local_src_dir_arg)
elif 'local_src_dir' in recipe:
local_src_dir = os.path.expanduser(recipe.get('local_src_dir'))
# If a relative path is specified, it should be in relation to the config file
if not os.path.isabs(local_src_dir):
local_src_dir = os.path.join(os.path.dirname(os.path.abspath(recipe_config_file)),
local_src_dir)
else:
local_src_dir = None
if local_src_dir:
if not os.path.exists(local_src_dir):
raise OpenCEError(Error.LOCAL_SRC_DIR, local_src_dir)
os.environ["LOCAL_SRC_DIR"] = local_src_dir
else:
if 'LOCAL_SRC_DIR' in os.environ:
del os.environ['LOCAL_SRC_DIR']
def git_clone(git_url, git_tag, location, up_to_date=False):
'''
Clone a git repository and checkout a certain branch.
'''
clone_cmd = "git clone " + git_url + " " + location
print("Clone cmd: ", clone_cmd)
clone_result = os.system(clone_cmd)
cur_dir = os.getcwd()
clone_successful = clone_result == 0
if clone_successful:
if git_tag:
os.chdir(location)
if up_to_date:
git_tag = get_branch_of_tag(git_tag)
checkout_cmd = "git checkout " + git_tag
print("Checkout branch/tag command: ", checkout_cmd)
checkout_res = os.system(checkout_cmd)
os.chdir(cur_dir)
clone_successful = checkout_res == 0
else:
raise OpenCEError(Error.CLONE_REPO, git_url)
return clone_successful
def _start_container(container_name, container_tool):
if os.system(container_tool + " start " + container_name):
raise OpenCEError(Error.START_CONTAINER, container_name)
def _copy_to_container(src, dest, container_name, container_tool):
if os.system(container_tool + " cp " + src + " " + container_name + ":" +
dest):
raise OpenCEError(Error.COPY_DIR_TO_CONTAINER, src, container_name)
def __init__(self,
env_config_files,
python_versions,
build_types,
mpi_types,
cuda_versions,
repository_folder="./",
channels=None,
git_location=utils.DEFAULT_GIT_LOCATION,
git_tag_for_env=utils.DEFAULT_GIT_TAG,
conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG,
packages=None):
self._env_config_files = env_config_files
self._repository_folder = repository_folder
self._channels = channels if channels else []
self._git_location = git_location
self._git_tag_for_env = git_tag_for_env
self._conda_build_config = conda_build_config
self._external_dependencies = dict()
self._conda_env_files = dict()
self._test_feedstocks = dict()
self._initial_package_indices = []
# Create a dependency tree that includes recipes for every combination
# of variants.
self._possible_variants = utils.make_variants(python_versions,
build_types, mpi_types,
cuda_versions)
self.build_commands = []
for variant in self._possible_variants:
try:
build_commands, external_deps = self._create_all_commands(
variant)
except OpenCEError as exc:
raise OpenCEError(Error.CREATE_BUILD_TREE, exc.msg) from exc
variant_string = utils.variant_string(variant["python"],
variant["build_type"],
variant["mpi_type"],
variant["cudatoolkit"])
self._external_dependencies[variant_string] = external_deps
# Add dependency tree information to the packages list and
# remove build commands from build_commands that are already in self.build_commands
build_commands, package_indices = _add_build_command_dependencies(
build_commands, self.build_commands, len(self.build_commands))
self.build_commands += build_commands
self._detect_cycle()
# If the packages argument is provided, find the indices into the build_commands for all
# of the packages that were requested.
variant_package_indices = []
if packages:
for package in packages:
if package in package_indices:
variant_package_indices += package_indices[package]
else:
print("INFO: No recipes were found for " + package +
" for variant " + variant_string)
else:
for package in package_indices:
variant_package_indices += package_indices[package]
self._initial_package_indices += variant_package_indices
validate_config.validate_build_tree(self.build_commands,
external_deps,
variant_package_indices)
installable_packages = get_installable_packages(
self.build_commands, external_deps, variant_package_indices)
filtered_packages = [
package for package in installable_packages
if utils.remove_version(package) in package_indices or
utils.remove_version(package) in utils.KNOWN_VARIANT_PACKAGES
]
self._conda_env_files[variant_string] = CondaEnvFileGenerator(
filtered_packages)
self._test_feedstocks[variant_string] = []
for build_command in traverse_build_commands(
self.build_commands, variant_package_indices):
self._test_feedstocks[variant_string].append(
build_command.repository)