def parse_args(parser, arg_strings=None): ''' Parses input arguments and handles more complex defaults. - conda_build_config: If not passed in the default is with the env file, if is passed in, otherwise it is expected to be in the local path. ''' args = parser.parse_args(arg_strings) _create_env_config_paths(args) if "conda_build_config" in vars(args).keys(): if args.conda_build_config is None: if "env_config_file" in vars(args).keys() and args.env_config_file: args.conda_build_config = os.path.join( os.path.dirname(args.env_config_file[0]), utils.CONDA_BUILD_CONFIG_FILE) else: args.conda_build_config = utils.DEFAULT_CONDA_BUILD_CONFIG if utils.is_url(args.conda_build_config): args.conda_build_config = utils.download_file( args.conda_build_config, filename=utils.CONDA_BUILD_CONFIG_FILE) elif os.path.exists(args.conda_build_config): args.conda_build_config = os.path.abspath(args.conda_build_config) else: print( "WARNING: No valid conda_build_config.yaml file was found. Some recipes may fail to build." ) return args
def _create_commands(repository, runtime_package, recipe_path, recipes, variant_config_files, variants, channels): """ Returns: A tree of nodes containing BuildCommands for each recipe within a repository. """ retval = graph.OpenCEGraph() saved_working_directory = os.getcwd() os.chdir(repository) config_data, _ = build_feedstock.load_package_config( variants=variants, recipe_path=recipe_path) combined_config_files = [ utils.download_file(config) if utils.is_url(config) else config for config in variant_config_files ] feedstock_conda_build_config_file = build_feedstock.get_conda_build_config( ) if feedstock_conda_build_config_file: combined_config_files.append(feedstock_conda_build_config_file) recipes_from_config = config_data.get('recipes', []) if recipes_from_config is None: recipes_from_config = [] channels_from_config = config_data.get('channels', []) if channels_from_config is not None: channels += channels_from_config for recipe in recipes_from_config: if recipes and not recipe.get('name') in recipes: continue packages, version, run_deps, host_deps, build_deps, test_deps, output_files = _get_package_dependencies( recipe.get('path'), combined_config_files, variants) build_command = BuildCommand(recipe=recipe.get('name', None), repository=repository, packages=packages, version=version, recipe_path=recipe_path, runtime_package=runtime_package, output_files=output_files, python=variants.get('python'), build_type=variants.get('build_type'), mpi_type=variants.get('mpi_type'), cudatoolkit=variants.get('cudatoolkit'), run_dependencies=run_deps, host_dependencies=host_deps, build_dependencies=build_deps, test_dependencies=test_deps, channels=channels, resources=recipe.get('resources'), conda_build_configs=variant_config_files) package_node = DependencyNode(set(packages), build_command) retval.add_node(package_node) os.chdir(saved_working_directory) return retval
def _clone_repo(self, git_url, repo_dir, env_config_data, package): """ Clone the git repo at repository. """ # Priority is given to command line specified tag, if it is not # specified then package specific tag, and when even that is not specified # then top level git tag specified for env in the env file. And if nothing is # at all specified then fall back to default branch of the repo. git_tag = self._git_tag_for_env git_tag_for_package = None if git_tag is None: git_tag_for_package = package.get(env_config.Key.git_tag.name, None) if package else None if git_tag_for_package: git_tag = git_tag_for_package else: git_tag = env_config_data.get( env_config.Key.git_tag_for_env.name, None) if env_config_data else None clone_successful = utils.git_clone( git_url, git_tag, repo_dir, self._git_up_to_date and not git_tag_for_package) if clone_successful: patches = package.get(env_config.Key.patches.name, []) if package else [] if len(patches) > 0: cur_dir = os.getcwd() os.chdir(repo_dir) for patch in patches: if os.path.isabs(patch) and os.path.exists(patch): patch_file = patch else: # Look for patch relative to where the Open-CE environment file is patch_file = os.path.join( os.path.dirname( env_config_data.get( env_config.Key.opence_env_file_path.name)), patch) if utils.is_url(patch_file): patch_file = utils.download_file(patch_file) patch_apply_cmd = "git apply {}".format(patch_file) log.info("Patch apply command: %s", patch_apply_cmd) patch_apply_res = os.system(patch_apply_cmd) if patch_apply_res != 0: os.chdir(cur_dir) shutil.rmtree(repo_dir) raise OpenCEError( Error.PATCH_APPLICATION, patch, package[env_config.Key.feedstock.name]) os.chdir(cur_dir)
def _validate_config_file(env_file, variants): '''Perform some validation on the environment file after loading it.''' # pylint: disable=import-outside-toplevel from open_ce import conda_utils try: if utils.is_url(env_file): env_file = utils.download_file(env_file) meta_obj = conda_utils.render_yaml(env_file, variants=variants, schema=_ENV_CONFIG_SCHEMA) if not (Key.packages.name in meta_obj.keys() or Key.imported_envs.name in meta_obj.keys()): raise OpenCEError(Error.CONFIG_CONTENT) meta_obj[Key.opence_env_file_path.name] = env_file return meta_obj except (Exception, SystemExit) as exc: #pylint: disable=broad-except raise OpenCEError(Error.ERROR, "Error in {}:\n {}".format(env_file, str(exc))) from exc
def _validate_config_file(env_file, variants): '''Perform some validation on the environment file after loading it.''' # pylint: disable=import-outside-toplevel from open_ce import conda_utils try: if utils.is_url(env_file): env_file = utils.download_file(env_file) # First, partially render yaml to validate builder version number. version_check_obj = conda_utils.render_yaml( env_file, permit_undefined_jinja=True) if Key.builder_version.name in version_check_obj.keys(): if not conda_utils.version_matches_spec( version_check_obj.get(Key.builder_version.name)): raise OpenCEError( Error.SCHEMA_VERSION_MISMATCH, env_file, version_check_obj.get(Key.builder_version.name), open_ce_version) meta_obj = None try: meta_obj = conda_utils.render_yaml(env_file, variants=variants, schema=_ENV_CONFIG_SCHEMA) if not (Key.packages.name in meta_obj.keys() or Key.imported_envs.name in meta_obj.keys()): raise OpenCEError(Error.CONFIG_CONTENT) meta_obj[Key.opence_env_file_path.name] = env_file except OpenCEError as exc: if Key.builder_version.name not in version_check_obj.keys(): show_warning(Error.SCHEMA_VERSION_NOT_FOUND, env_file, Key.builder_version.name) raise exc return meta_obj except (Exception, SystemExit) as exc: #pylint: disable=broad-except raise OpenCEError(Error.ERROR, "Error in {}:\n {}".format(env_file, str(exc))) from exc
def build_feedstock_from_command( command, # pylint: disable=too-many-arguments, too-many-locals recipe_config_file=None, output_folder=utils.DEFAULT_OUTPUT_FOLDER, local_src_dir=None, pkg_format=utils.DEFAULT_PKG_FORMAT, debug=None, debug_output_id=None): ''' Build a feedstock from a build_command object. ''' utils.check_if_package_exists('conda-build') # pylint: disable=import-outside-toplevel import conda_build.api from conda_build.config import get_or_merge_config saved_working_directory = None if command.repository: saved_working_directory = os.getcwd() os.chdir(os.path.abspath(command.repository)) recipes_to_build = inputs.parse_arg_list(command.recipe) for variant in utils.make_variants(command.python, command.build_type, command.mpi_type, command.cudatoolkit): build_config_data, recipe_config_file = load_package_config( recipe_config_file, variant, command.recipe_path) # Build each recipe if build_config_data['recipes'] is None: build_config_data['recipes'] = [] log.info("No recipe to build for given configuration.") for recipe in build_config_data['recipes']: if recipes_to_build and recipe['name'] not in recipes_to_build: continue config = get_or_merge_config(None, variant=variant) config.skip_existing = False config.prefix_length = 225 config.output_folder = output_folder conda_build_configs = [ utils.download_file(conda_build_config) if utils.is_url(conda_build_config) else conda_build_config for conda_build_config in command.conda_build_configs ] config.variant_config_files = [ config for config in conda_build_configs if os.path.exists(config) ] if pkg_format == "conda": config.conda_pkg_format = "2" # set to .conda format recipe_conda_build_config = get_conda_build_config() if recipe_conda_build_config: config.variant_config_files.append(recipe_conda_build_config) config.channel_urls = [os.path.abspath(output_folder)] config.channel_urls += command.channels config.channel_urls += build_config_data.get('channels', []) _set_local_src_dir(local_src_dir, recipe, recipe_config_file) try: if debug: activation_string = conda_build.api.debug( os.path.join(os.getcwd(), recipe['path']), output_id=debug_output_id, config=config) if activation_string: log.info("#" * 80) log.info( "Build and/or host environments created for debug output id %s." "To enter a debugging environment:\n", debug_output_id) log.info(activation_string) log.info("#" * 80) else: conda_build.api.build(os.path.join(os.getcwd(), recipe['path']), config=config) except Exception as exc: # pylint: disable=broad-except traceback.print_exc() raise OpenCEError( Error.BUILD_RECIPE, recipe['name'] if 'name' in recipe else os.getcwd, str(exc)) from exc if saved_working_directory: os.chdir(saved_working_directory)