def parse_args(parser, arg_strings=None):
'''
Parses input arguments and handles more complex defaults.
- conda_build_config: If not passed in the default is with the env file,
if is passed in, otherwise it is expected to be in
the local path.
'''
args = parser.parse_args(arg_strings)
_create_env_config_paths(args)
if "conda_build_config" in vars(args).keys():
if args.conda_build_config is None:
if "env_config_file" in vars(args).keys() and args.env_config_file:
args.conda_build_config = os.path.join(
os.path.dirname(args.env_config_file[0]),
utils.CONDA_BUILD_CONFIG_FILE)
else:
args.conda_build_config = utils.DEFAULT_CONDA_BUILD_CONFIG
if utils.is_url(args.conda_build_config):
args.conda_build_config = utils.download_file(
args.conda_build_config,
filename=utils.CONDA_BUILD_CONFIG_FILE)
elif os.path.exists(args.conda_build_config):
args.conda_build_config = os.path.abspath(args.conda_build_config)
else:
print(
"WARNING: No valid conda_build_config.yaml file was found. Some recipes may fail to build."
)
return args
def _create_commands(repository, runtime_package, recipe_path, recipes,
variant_config_files, variants, channels):
"""
Returns:
A tree of nodes containing BuildCommands for each recipe within a repository.
"""
retval = graph.OpenCEGraph()
saved_working_directory = os.getcwd()
os.chdir(repository)
config_data, _ = build_feedstock.load_package_config(
variants=variants, recipe_path=recipe_path)
combined_config_files = [
utils.download_file(config) if utils.is_url(config) else config
for config in variant_config_files
]
feedstock_conda_build_config_file = build_feedstock.get_conda_build_config(
)
if feedstock_conda_build_config_file:
combined_config_files.append(feedstock_conda_build_config_file)
recipes_from_config = config_data.get('recipes', [])
if recipes_from_config is None:
recipes_from_config = []
channels_from_config = config_data.get('channels', [])
if channels_from_config is not None:
channels += channels_from_config
for recipe in recipes_from_config:
if recipes and not recipe.get('name') in recipes:
continue
packages, version, run_deps, host_deps, build_deps, test_deps, output_files = _get_package_dependencies(
recipe.get('path'), combined_config_files, variants)
build_command = BuildCommand(recipe=recipe.get('name', None),
repository=repository,
packages=packages,
version=version,
recipe_path=recipe_path,
runtime_package=runtime_package,
output_files=output_files,
python=variants.get('python'),
build_type=variants.get('build_type'),
mpi_type=variants.get('mpi_type'),
cudatoolkit=variants.get('cudatoolkit'),
run_dependencies=run_deps,
host_dependencies=host_deps,
build_dependencies=build_deps,
test_dependencies=test_deps,
channels=channels,
resources=recipe.get('resources'),
conda_build_configs=variant_config_files)
package_node = DependencyNode(set(packages), build_command)
retval.add_node(package_node)
os.chdir(saved_working_directory)
return retval
def _clone_repo(self, git_url, repo_dir, env_config_data, package):
"""
Clone the git repo at repository.
"""
# Priority is given to command line specified tag, if it is not
# specified then package specific tag, and when even that is not specified
# then top level git tag specified for env in the env file. And if nothing is
# at all specified then fall back to default branch of the repo.
git_tag = self._git_tag_for_env
git_tag_for_package = None
if git_tag is None:
git_tag_for_package = package.get(env_config.Key.git_tag.name,
None) if package else None
if git_tag_for_package:
git_tag = git_tag_for_package
else:
git_tag = env_config_data.get(
env_config.Key.git_tag_for_env.name,
None) if env_config_data else None
clone_successful = utils.git_clone(
git_url, git_tag, repo_dir, self._git_up_to_date
and not git_tag_for_package)
if clone_successful:
patches = package.get(env_config.Key.patches.name,
[]) if package else []
if len(patches) > 0:
cur_dir = os.getcwd()
os.chdir(repo_dir)
for patch in patches:
if os.path.isabs(patch) and os.path.exists(patch):
patch_file = patch
else:
# Look for patch relative to where the Open-CE environment file is
patch_file = os.path.join(
os.path.dirname(
env_config_data.get(
env_config.Key.opence_env_file_path.name)),
patch)
if utils.is_url(patch_file):
patch_file = utils.download_file(patch_file)
patch_apply_cmd = "git apply {}".format(patch_file)
log.info("Patch apply command: %s", patch_apply_cmd)
patch_apply_res = os.system(patch_apply_cmd)
if patch_apply_res != 0:
os.chdir(cur_dir)
shutil.rmtree(repo_dir)
raise OpenCEError(
Error.PATCH_APPLICATION, patch,
package[env_config.Key.feedstock.name])
os.chdir(cur_dir)
def _validate_config_file(env_file, variants):
'''Perform some validation on the environment file after loading it.'''
# pylint: disable=import-outside-toplevel
from open_ce import conda_utils
try:
if utils.is_url(env_file):
env_file = utils.download_file(env_file)
meta_obj = conda_utils.render_yaml(env_file,
variants=variants,
schema=_ENV_CONFIG_SCHEMA)
if not (Key.packages.name in meta_obj.keys()
or Key.imported_envs.name in meta_obj.keys()):
raise OpenCEError(Error.CONFIG_CONTENT)
meta_obj[Key.opence_env_file_path.name] = env_file
return meta_obj
except (Exception, SystemExit) as exc: #pylint: disable=broad-except
raise OpenCEError(Error.ERROR,
"Error in {}:\n {}".format(env_file,
str(exc))) from exc
def _validate_config_file(env_file, variants):
'''Perform some validation on the environment file after loading it.'''
# pylint: disable=import-outside-toplevel
from open_ce import conda_utils
try:
if utils.is_url(env_file):
env_file = utils.download_file(env_file)
# First, partially render yaml to validate builder version number.
version_check_obj = conda_utils.render_yaml(
env_file, permit_undefined_jinja=True)
if Key.builder_version.name in version_check_obj.keys():
if not conda_utils.version_matches_spec(
version_check_obj.get(Key.builder_version.name)):
raise OpenCEError(
Error.SCHEMA_VERSION_MISMATCH, env_file,
version_check_obj.get(Key.builder_version.name),
open_ce_version)
meta_obj = None
try:
meta_obj = conda_utils.render_yaml(env_file,
variants=variants,
schema=_ENV_CONFIG_SCHEMA)
if not (Key.packages.name in meta_obj.keys()
or Key.imported_envs.name in meta_obj.keys()):
raise OpenCEError(Error.CONFIG_CONTENT)
meta_obj[Key.opence_env_file_path.name] = env_file
except OpenCEError as exc:
if Key.builder_version.name not in version_check_obj.keys():
show_warning(Error.SCHEMA_VERSION_NOT_FOUND, env_file,
Key.builder_version.name)
raise exc
return meta_obj
except (Exception, SystemExit) as exc: #pylint: disable=broad-except
raise OpenCEError(Error.ERROR,
"Error in {}:\n {}".format(env_file,
str(exc))) from exc
def build_feedstock_from_command(
command, # pylint: disable=too-many-arguments, too-many-locals
recipe_config_file=None,
output_folder=utils.DEFAULT_OUTPUT_FOLDER,
local_src_dir=None,
pkg_format=utils.DEFAULT_PKG_FORMAT,
debug=None,
debug_output_id=None):
'''
Build a feedstock from a build_command object.
'''
utils.check_if_package_exists('conda-build')
# pylint: disable=import-outside-toplevel
import conda_build.api
from conda_build.config import get_or_merge_config
saved_working_directory = None
if command.repository:
saved_working_directory = os.getcwd()
os.chdir(os.path.abspath(command.repository))
recipes_to_build = inputs.parse_arg_list(command.recipe)
for variant in utils.make_variants(command.python, command.build_type,
command.mpi_type, command.cudatoolkit):
build_config_data, recipe_config_file = load_package_config(
recipe_config_file, variant, command.recipe_path)
# Build each recipe
if build_config_data['recipes'] is None:
build_config_data['recipes'] = []
log.info("No recipe to build for given configuration.")
for recipe in build_config_data['recipes']:
if recipes_to_build and recipe['name'] not in recipes_to_build:
continue
config = get_or_merge_config(None, variant=variant)
config.skip_existing = False
config.prefix_length = 225
config.output_folder = output_folder
conda_build_configs = [
utils.download_file(conda_build_config)
if utils.is_url(conda_build_config) else conda_build_config
for conda_build_config in command.conda_build_configs
]
config.variant_config_files = [
config for config in conda_build_configs
if os.path.exists(config)
]
if pkg_format == "conda":
config.conda_pkg_format = "2" # set to .conda format
recipe_conda_build_config = get_conda_build_config()
if recipe_conda_build_config:
config.variant_config_files.append(recipe_conda_build_config)
config.channel_urls = [os.path.abspath(output_folder)]
config.channel_urls += command.channels
config.channel_urls += build_config_data.get('channels', [])
_set_local_src_dir(local_src_dir, recipe, recipe_config_file)
try:
if debug:
activation_string = conda_build.api.debug(
os.path.join(os.getcwd(), recipe['path']),
output_id=debug_output_id,
config=config)
if activation_string:
log.info("#" * 80)
log.info(
"Build and/or host environments created for debug output id %s."
"To enter a debugging environment:\n",
debug_output_id)
log.info(activation_string)
log.info("#" * 80)
else:
conda_build.api.build(os.path.join(os.getcwd(),
recipe['path']),
config=config)
except Exception as exc: # pylint: disable=broad-except
traceback.print_exc()
raise OpenCEError(
Error.BUILD_RECIPE,
recipe['name'] if 'name' in recipe else os.getcwd,
str(exc)) from exc
if saved_working_directory:
os.chdir(saved_working_directory)