def __init__(self, env_config_files, python_versions, build_types, repository_folder="./", git_location=DEFAULT_GIT_LOCATION, git_tag_for_env="master", conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG): self._env_config_files = env_config_files self._repository_folder = repository_folder self._git_location = git_location self._git_tag_for_env = git_tag_for_env self._conda_build_config = conda_build_config # Create a dependency tree that includes recipes for every combination # of variants. variants = { 'python': utils.parse_arg_list(python_versions), 'build_type': utils.parse_arg_list(build_types) } variant_cart_product = [ dict(zip(variants, y)) for y in product(*variants.values()) ] self.build_commands = [] for variant in variant_cart_product: result, variant_recipes = self._create_all_recipes(variant) if result != 0: raise OpenCEError("Error creating Build Tree") # Add dependency tree information to the packages list _add_build_command_dependencies(variant_recipes, len(self.build_commands)) self.build_commands += variant_recipes
def build_env(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) if args.docker_build: return docker_build.build_with_docker(args.output_folder, sys.argv) result = 0 common_package_build_args = [] common_package_build_args += [ "--output_folder", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--channel", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--conda_build_config", os.path.abspath(args.conda_build_config) ] for channel in args.channels_list: common_package_build_args += ["--channels", channel] # If repository_folder doesn't exist, create it if args.repository_folder and not os.path.exists(args.repository_folder): os.mkdir(args.repository_folder) # Create the build tree try: build_tree = BuildTree( env_config_files=args.env_config_file, python_versions=utils.parse_arg_list(args.python_versions), build_types=utils.parse_arg_list(args.build_types), repository_folder=args.repository_folder, git_location=args.git_location, git_tag_for_env=args.git_tag_for_env, conda_build_config=args.conda_build_config) except OpenCEError as err: print(err.msg) return 1 # Build each package in the packages list for build_command in build_tree: build_args = common_package_build_args + build_command.feedstock_args() result = build_feedstock.build_feedstock(build_args) if result != 0: print("Unable to build recipe: " + build_command.repository) return result return result
def validate_env(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) variants = { 'python': utils.parse_arg_list(args.python_versions), 'build_type': utils.parse_arg_list(args.build_types) } retval, _ = env_config.load_env_config_files(args.env_config_file, variants) return retval
def test_parse_arg_list_large_string_input(): ''' Test parse_arg_list with a more complicated input, including spaces. ''' string_input = "this,is a, big , test ," list_output = ["this", "is a", " big ", " test ", ""] assert list_output == utils.parse_arg_list(string_input)
def test_parse_arg_list_small_string_input(): ''' Tests that parse_arg_list works for a simple case. ''' string_input = "a,b,c" list_output = ["a", "b", "c"] assert list_output == utils.parse_arg_list(string_input)
def __init__( self, python_versions=utils.DEFAULT_PYTHON_VERS, build_types=utils.DEFAULT_BUILD_TYPES, mpi_types=utils.DEFAULT_MPI_TYPES, channels=None, output_folder=None, env_file_prefix=utils.CONDA_ENV_FILENAME_PREFIX, ): self.python_versions = utils.parse_arg_list(python_versions) self.build_types = utils.parse_arg_list(build_types) self.mpi_types = utils.parse_arg_list(mpi_types) self.env_file_prefix = env_file_prefix self.dependency_dict = {} self.channels = [] self._initialize_dependency_dict() self._initialize_channels(channels, output_folder)
def validate_env(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) variants = [{ 'python': py_vers, 'build_type': build_type } for py_vers in utils.parse_arg_list(args.python_versions) for build_type in utils.parse_arg_list(args.build_types)] retval = 0 for variant in variants: result, _ = env_config.load_env_config_files(args.env_config_file, variant) retval += result return retval
def build_feedstock(args_string=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(args_string) saved_working_directory = None if args.working_directory: saved_working_directory = os.getcwd() os.chdir(os.path.abspath(args.working_directory)) build_config_data, recipe_config_file = load_package_config( args.recipe_config_file) args.recipes = utils.parse_arg_list(args.recipe_list) # Build each recipe for recipe in build_config_data['recipes']: if args.recipes and recipe['name'] not in args.recipes: continue config = get_or_merge_config(None) config.skip_existing = True config.output_folder = args.output_folder config.variant_config_files = [args.conda_build_config] recipe_conda_build_config = os.path.join(os.getcwd(), "config", "conda_build_config.yaml") if os.path.exists(recipe_conda_build_config): config.variant_config_files.append(recipe_conda_build_config) config.channel_urls = args.channels_list + build_config_data.get( 'channels', []) _set_local_src_dir(args.local_src_dir, recipe, recipe_config_file) try: for variant in utils.make_variants(args.python_versions, args.build_types, args.mpi_types): conda_build.api.build(os.path.join(os.getcwd(), recipe['path']), config=config, variants=variant) except Exception as exc: # pylint: disable=broad-except traceback.print_exc() raise OpenCEError( Error.BUILD_RECIPE, recipe['name'] if 'name' in recipe else os.getcwd, str(exc)) from exc if saved_working_directory: os.chdir(saved_working_directory)
def tag_all_repos(github_org, tag, tag_msg, branch, repo_dir, pat, skipped_repos): # pylint: disable=too-many-arguments ''' Clones, then tags all repos with a given tag, and pushes back to remote. These steps are performed in separate loops to make debugging easier. ''' skipped_repos = utils.parse_arg_list(skipped_repos) repos = git_utils.get_all_repos(github_org, pat) repos = [repo for repo in repos if repo["name"] not in skipped_repos] print("---------------------------Cloning all Repos") for repo in repos: repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"])) print("--->Making clone location: " + repo_path) os.makedirs(repo_path, exist_ok=True) print("--->Cloning {}".format(repo["name"])) git_utils.clone_repo(repo["ssh_url"], repo_path, branch) print("---------------------------Tagging all Repos") for repo in repos: repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"])) print("--->Tagging {}".format(repo["name"])) git_utils.create_tag(repo_path, tag, tag_msg) push = git_utils.ask_for_input( "Would you like to push all tags to remote?") if not push.startswith("y"): return print("---------------------------Pushing all Repos") for repo in repos: try: repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"])) print("--->Pushing {}".format(repo["name"])) git_utils.push_branch(repo_path, tag) except Exception as exc: # pylint: disable=broad-except print("Error encountered when trying to push {}".format( repo["name"])) print(exc) cont = git_utils.ask_for_input( "Would you like to continue with the other repos?") if cont.startswith("y"): continue raise
def build_env(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) if args.docker_build: return docker_build.build_with_docker(args.output_folder, sys.argv) # Checking conda-build existence if --docker_build is not specified utils.check_if_conda_build_exists() # Here, importing BuildTree is intentionally done after checking # existence of conda-build as BuildTree uses conda_build APIs. from build_tree import BuildTree # pylint: disable=import-outside-toplevel result = 0 common_package_build_args = [] common_package_build_args += [ "--output_folder", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--channel", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--conda_build_config", os.path.abspath(args.conda_build_config) ] for channel in args.channels_list: common_package_build_args += ["--channels", channel] # If repository_folder doesn't exist, create it if args.repository_folder and not os.path.exists(args.repository_folder): os.mkdir(args.repository_folder) # Create the build tree try: build_tree = BuildTree( env_config_files=args.env_config_file, python_versions=utils.parse_arg_list(args.python_versions), build_types=utils.parse_arg_list(args.build_types), repository_folder=args.repository_folder, git_location=args.git_location, git_tag_for_env=args.git_tag_for_env, conda_build_config=args.conda_build_config) except OpenCEError as err: print(err.msg) return 1 # Build each package in the packages list for build_command in build_tree: build_args = common_package_build_args + build_command.feedstock_args() result = build_feedstock.build_feedstock(build_args) if result != 0: print("Unable to build recipe: " + build_command.repository) return result return result
def validate_config(arg_strings=None): ''' Entry function. ''' args = make_parser().parse_args(arg_strings) variants = [{ 'python': py_vers, 'build_type': build_type } for py_vers in utils.parse_arg_list(args.python_versions) for build_type in utils.parse_arg_list(args.build_types)] for variant in variants: print('Validating {} for {}'.format(args.conda_build_config, variant)) for env_file in args.env_config_file: print('Validating {} for {} : {}'.format(args.conda_build_config, env_file, variant)) try: recipes = build_tree.BuildTree( [env_file], variant['python'], variant['build_type'], repository_folder=args.repository_folder, conda_build_config=args.conda_build_config) except OpenCEError as err: print(err.msg) print('Error while validating {} for {} : {}'.format( args.conda_build_config, env_file, variant)) return 1 packages = [ package for recipe in recipes for package in recipe.packages ] channels = { channel for recipe in recipes for channel in recipe.channels } deps = { dep for recipe in recipes for dep in recipe.run_dependencies } pkg_args = " ".join([ "\"{}\"".format(generalize_version(dep)) for dep in deps if not utils.remove_version(dep) in packages ]) channel_args = " ".join( {"-c \"{}\"".format(channel) for channel in channels}) cli = "conda create --dry-run -n test_conda_dependencies {} {}".format( channel_args, pkg_args) retval = run_and_log(cli) if retval != 0: print('Error while validating {} for {} : {}'.format( args.conda_build_config, env_file, variant)) return 1 print('Successfully validated {} for {} : {}'.format( args.conda_build_config, env_file, variant)) print('Successfully validated {} for {}'.format( args.conda_build_config, variant)) print("{} Successfully validated!".format(args.conda_build_config)) return 0
def build_env(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) if args.docker_build: docker_build.build_with_docker(args.output_folder, sys.argv) return # Checking conda-build existence if --docker_build is not specified utils.check_if_conda_build_exists() # Here, importing BuildTree is intentionally done after checking # existence of conda-build as BuildTree uses conda_build APIs. from build_tree import BuildTree # pylint: disable=import-outside-toplevel common_package_build_args = [] common_package_build_args += [ "--output_folder", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--channel", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--conda_build_config", os.path.abspath(args.conda_build_config) ] for channel in args.channels_list: common_package_build_args += ["--channels", channel] # If repository_folder doesn't exist, create it if args.repository_folder and not os.path.exists(args.repository_folder): os.mkdir(args.repository_folder) # Create the build tree build_tree = BuildTree(env_config_files=args.env_config_file, python_versions=utils.parse_arg_list( args.python_versions), build_types=utils.parse_arg_list(args.build_types), mpi_types=utils.parse_arg_list(args.mpi_types), repository_folder=args.repository_folder, git_location=args.git_location, git_tag_for_env=args.git_tag_for_env, conda_build_config=args.conda_build_config) conda_env_data = CondaEnvFileGenerator( python_versions=args.python_versions, build_types=args.build_types, mpi_types=args.mpi_types, channels=args.channels_list, output_folder=os.path.abspath(args.output_folder), ) # Build each package in the packages list for build_command in build_tree: build_args = common_package_build_args + build_command.feedstock_args() try: build_feedstock.build_feedstock(build_args) except OpenCEError as exc: raise OpenCEError(Error.BUILD_RECIPE, build_command.repository, exc.msg) from exc conda_env_data.update_conda_env_file_content(build_command, build_tree) conda_env_files = conda_env_data.write_conda_env_files() print( "Generated conda environment files from the selected build arguments:", conda_env_files) print("INFO: One can use these environment files to create a conda" \ " environment using \"conda env create -f <conda_env_file_name>.\"")
def test_parse_arg_list_list_input(): ''' Parse arg list should return the input argument if it's already a list. ''' list_input = ["a", "b", "c"] assert list_input == utils.parse_arg_list(list_input)
def build_feedstock(args_string=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(args_string) saved_working_directory = None if args.working_directory: saved_working_directory = os.getcwd() os.chdir(os.path.abspath(args.working_directory)) build_config_data, recipe_config_file = load_package_config( args.recipe_config_file) if build_config_data is None: return 1 args.recipes = utils.parse_arg_list(args.recipe_list) result = 0 # Build each recipe for recipe in build_config_data['recipes']: if args.recipes and recipe['name'] not in args.recipes: continue conda_build_args = "conda-build " conda_build_args += "--skip-existing " conda_build_args += "--output-folder " + args.output_folder + " " conda_build_args += "-m " + args.conda_build_config + " " recipe_conda_build_config = os.path.join(os.getcwd(), "config", "conda_build_config.yaml") if os.path.exists(recipe_conda_build_config): conda_build_args += " -m " + recipe_conda_build_config + " " conda_build_args += " ".join( ["-c " + c + " " for c in args.channels_list]) conda_build_args += " ".join( ["-c " + c + " " for c in build_config_data.get('channels', [])]) variants = dict() if args.python_versions: variants['python'] = utils.parse_arg_list(args.python_versions) if args.build_types: variants['build_type'] = utils.parse_arg_list(args.build_types) if variants: conda_build_args += "--variants \"" + str(variants) + "\" " conda_build_args += recipe['path'] result = _set_local_src_dir(args.local_src_dir, recipe, recipe_config_file) if result != 0: break print(conda_build_args) result = os.system(conda_build_args) if result != 0: print("Failure building recipe: " + (recipe['name'] if 'name' in recipe else os.getcwd)) result = 1 break if saved_working_directory: os.chdir(saved_working_directory) return result
def build_env(arg_strings=None): ''' Entry function. ''' parser = make_parser() args = parser.parse_args(arg_strings) result = 0 python_build_args = [] common_package_build_args = [] common_package_build_args += [ "--output_folder", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--channel", os.path.abspath(args.output_folder) ] common_package_build_args += [ "--conda_build_config", os.path.abspath(args.conda_build_config) ] if args.build_types: common_package_build_args += ["--build_types", args.build_types] for channel in args.channels_list: common_package_build_args += ["--channels", channel] # If repository_folder doesn't exist, create it if args.repository_folder and not os.path.exists(args.repository_folder): os.mkdir(args.repository_folder) for py_vers in utils.parse_arg_list(args.python_versions): print("Builds for python version: " + py_vers) python_build_args = ["--python_versions", py_vers] variants = { 'python': py_vers, 'build_type': utils.parse_arg_list(args.build_types) } result, recipes = create_all_recipes( args.env_config_file, variants, repository_folder=args.repository_folder, git_location=args.git_location, git_tag_for_env=args.git_tag_for_env, conda_build_config=args.conda_build_config) if result != 0: return result # Add dependency tree information to the packages list dep_tree = _create_dep_tree(recipes) # Build each package in the packages list for recipe in _traverse_recipes(recipes, dep_tree): package_build_args = ["--working_directory", recipe['repository']] for channel in recipe.get('channels', []): package_build_args += ["--channels", channel] if 'recipe' in recipe: package_build_args += ["--recipes", recipe['recipe']] build_args = common_package_build_args + python_build_args + package_build_args result = build_feedstock.build_feedstock(build_args) if result != 0: print("Unable to build recipe: " + recipe['repository']) return result return result