def __init__(self,
env_config_files,
python_versions,
build_types,
repository_folder="./",
git_location=DEFAULT_GIT_LOCATION,
git_tag_for_env="master",
conda_build_config=utils.DEFAULT_CONDA_BUILD_CONFIG):
self._env_config_files = env_config_files
self._repository_folder = repository_folder
self._git_location = git_location
self._git_tag_for_env = git_tag_for_env
self._conda_build_config = conda_build_config
# Create a dependency tree that includes recipes for every combination
# of variants.
variants = {
'python': utils.parse_arg_list(python_versions),
'build_type': utils.parse_arg_list(build_types)
}
variant_cart_product = [
dict(zip(variants, y)) for y in product(*variants.values())
]
self.build_commands = []
for variant in variant_cart_product:
result, variant_recipes = self._create_all_recipes(variant)
if result != 0:
raise OpenCEError("Error creating Build Tree")
# Add dependency tree information to the packages list
_add_build_command_dependencies(variant_recipes,
len(self.build_commands))
self.build_commands += variant_recipes
def build_env(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
if args.docker_build:
return docker_build.build_with_docker(args.output_folder, sys.argv)
result = 0
common_package_build_args = []
common_package_build_args += [
"--output_folder",
os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--channel", os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--conda_build_config",
os.path.abspath(args.conda_build_config)
]
for channel in args.channels_list:
common_package_build_args += ["--channels", channel]
# If repository_folder doesn't exist, create it
if args.repository_folder and not os.path.exists(args.repository_folder):
os.mkdir(args.repository_folder)
# Create the build tree
try:
build_tree = BuildTree(
env_config_files=args.env_config_file,
python_versions=utils.parse_arg_list(args.python_versions),
build_types=utils.parse_arg_list(args.build_types),
repository_folder=args.repository_folder,
git_location=args.git_location,
git_tag_for_env=args.git_tag_for_env,
conda_build_config=args.conda_build_config)
except OpenCEError as err:
print(err.msg)
return 1
# Build each package in the packages list
for build_command in build_tree:
build_args = common_package_build_args + build_command.feedstock_args()
result = build_feedstock.build_feedstock(build_args)
if result != 0:
print("Unable to build recipe: " + build_command.repository)
return result
return result
def validate_env(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
variants = {
'python': utils.parse_arg_list(args.python_versions),
'build_type': utils.parse_arg_list(args.build_types)
}
retval, _ = env_config.load_env_config_files(args.env_config_file,
variants)
return retval
def test_parse_arg_list_large_string_input():
'''
Test parse_arg_list with a more complicated input, including spaces.
'''
string_input = "this,is a, big , test ,"
list_output = ["this", "is a", " big ", " test ", ""]
assert list_output == utils.parse_arg_list(string_input)
def test_parse_arg_list_small_string_input():
'''
Tests that parse_arg_list works for a simple case.
'''
string_input = "a,b,c"
list_output = ["a", "b", "c"]
assert list_output == utils.parse_arg_list(string_input)
def __init__(
self,
python_versions=utils.DEFAULT_PYTHON_VERS,
build_types=utils.DEFAULT_BUILD_TYPES,
mpi_types=utils.DEFAULT_MPI_TYPES,
channels=None,
output_folder=None,
env_file_prefix=utils.CONDA_ENV_FILENAME_PREFIX,
):
self.python_versions = utils.parse_arg_list(python_versions)
self.build_types = utils.parse_arg_list(build_types)
self.mpi_types = utils.parse_arg_list(mpi_types)
self.env_file_prefix = env_file_prefix
self.dependency_dict = {}
self.channels = []
self._initialize_dependency_dict()
self._initialize_channels(channels, output_folder)
def validate_env(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
variants = [{
'python': py_vers,
'build_type': build_type
} for py_vers in utils.parse_arg_list(args.python_versions)
for build_type in utils.parse_arg_list(args.build_types)]
retval = 0
for variant in variants:
result, _ = env_config.load_env_config_files(args.env_config_file,
variant)
retval += result
return retval
def build_feedstock(args_string=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(args_string)
saved_working_directory = None
if args.working_directory:
saved_working_directory = os.getcwd()
os.chdir(os.path.abspath(args.working_directory))
build_config_data, recipe_config_file = load_package_config(
args.recipe_config_file)
args.recipes = utils.parse_arg_list(args.recipe_list)
# Build each recipe
for recipe in build_config_data['recipes']:
if args.recipes and recipe['name'] not in args.recipes:
continue
config = get_or_merge_config(None)
config.skip_existing = True
config.output_folder = args.output_folder
config.variant_config_files = [args.conda_build_config]
recipe_conda_build_config = os.path.join(os.getcwd(), "config",
"conda_build_config.yaml")
if os.path.exists(recipe_conda_build_config):
config.variant_config_files.append(recipe_conda_build_config)
config.channel_urls = args.channels_list + build_config_data.get(
'channels', [])
_set_local_src_dir(args.local_src_dir, recipe, recipe_config_file)
try:
for variant in utils.make_variants(args.python_versions,
args.build_types,
args.mpi_types):
conda_build.api.build(os.path.join(os.getcwd(),
recipe['path']),
config=config,
variants=variant)
except Exception as exc: # pylint: disable=broad-except
traceback.print_exc()
raise OpenCEError(
Error.BUILD_RECIPE,
recipe['name'] if 'name' in recipe else os.getcwd,
str(exc)) from exc
if saved_working_directory:
os.chdir(saved_working_directory)
def tag_all_repos(github_org, tag, tag_msg, branch, repo_dir, pat,
skipped_repos): # pylint: disable=too-many-arguments
'''
Clones, then tags all repos with a given tag, and pushes back to remote.
These steps are performed in separate loops to make debugging easier.
'''
skipped_repos = utils.parse_arg_list(skipped_repos)
repos = git_utils.get_all_repos(github_org, pat)
repos = [repo for repo in repos if repo["name"] not in skipped_repos]
print("---------------------------Cloning all Repos")
for repo in repos:
repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"]))
print("--->Making clone location: " + repo_path)
os.makedirs(repo_path, exist_ok=True)
print("--->Cloning {}".format(repo["name"]))
git_utils.clone_repo(repo["ssh_url"], repo_path, branch)
print("---------------------------Tagging all Repos")
for repo in repos:
repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"]))
print("--->Tagging {}".format(repo["name"]))
git_utils.create_tag(repo_path, tag, tag_msg)
push = git_utils.ask_for_input(
"Would you like to push all tags to remote?")
if not push.startswith("y"):
return
print("---------------------------Pushing all Repos")
for repo in repos:
try:
repo_path = os.path.abspath(os.path.join(repo_dir, repo["name"]))
print("--->Pushing {}".format(repo["name"]))
git_utils.push_branch(repo_path, tag)
except Exception as exc: # pylint: disable=broad-except
print("Error encountered when trying to push {}".format(
repo["name"]))
print(exc)
cont = git_utils.ask_for_input(
"Would you like to continue with the other repos?")
if cont.startswith("y"):
continue
raise
def build_env(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
if args.docker_build:
return docker_build.build_with_docker(args.output_folder, sys.argv)
# Checking conda-build existence if --docker_build is not specified
utils.check_if_conda_build_exists()
# Here, importing BuildTree is intentionally done after checking
# existence of conda-build as BuildTree uses conda_build APIs.
from build_tree import BuildTree # pylint: disable=import-outside-toplevel
result = 0
common_package_build_args = []
common_package_build_args += [
"--output_folder",
os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--channel", os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--conda_build_config",
os.path.abspath(args.conda_build_config)
]
for channel in args.channels_list:
common_package_build_args += ["--channels", channel]
# If repository_folder doesn't exist, create it
if args.repository_folder and not os.path.exists(args.repository_folder):
os.mkdir(args.repository_folder)
# Create the build tree
try:
build_tree = BuildTree(
env_config_files=args.env_config_file,
python_versions=utils.parse_arg_list(args.python_versions),
build_types=utils.parse_arg_list(args.build_types),
repository_folder=args.repository_folder,
git_location=args.git_location,
git_tag_for_env=args.git_tag_for_env,
conda_build_config=args.conda_build_config)
except OpenCEError as err:
print(err.msg)
return 1
# Build each package in the packages list
for build_command in build_tree:
build_args = common_package_build_args + build_command.feedstock_args()
result = build_feedstock.build_feedstock(build_args)
if result != 0:
print("Unable to build recipe: " + build_command.repository)
return result
return result
def validate_config(arg_strings=None):
'''
Entry function.
'''
args = make_parser().parse_args(arg_strings)
variants = [{
'python': py_vers,
'build_type': build_type
} for py_vers in utils.parse_arg_list(args.python_versions)
for build_type in utils.parse_arg_list(args.build_types)]
for variant in variants:
print('Validating {} for {}'.format(args.conda_build_config, variant))
for env_file in args.env_config_file:
print('Validating {} for {} : {}'.format(args.conda_build_config,
env_file, variant))
try:
recipes = build_tree.BuildTree(
[env_file],
variant['python'],
variant['build_type'],
repository_folder=args.repository_folder,
conda_build_config=args.conda_build_config)
except OpenCEError as err:
print(err.msg)
print('Error while validating {} for {} : {}'.format(
args.conda_build_config, env_file, variant))
return 1
packages = [
package for recipe in recipes for package in recipe.packages
]
channels = {
channel
for recipe in recipes for channel in recipe.channels
}
deps = {
dep
for recipe in recipes for dep in recipe.run_dependencies
}
pkg_args = " ".join([
"\"{}\"".format(generalize_version(dep)) for dep in deps
if not utils.remove_version(dep) in packages
])
channel_args = " ".join(
{"-c \"{}\"".format(channel)
for channel in channels})
cli = "conda create --dry-run -n test_conda_dependencies {} {}".format(
channel_args, pkg_args)
retval = run_and_log(cli)
if retval != 0:
print('Error while validating {} for {} : {}'.format(
args.conda_build_config, env_file, variant))
return 1
print('Successfully validated {} for {} : {}'.format(
args.conda_build_config, env_file, variant))
print('Successfully validated {} for {}'.format(
args.conda_build_config, variant))
print("{} Successfully validated!".format(args.conda_build_config))
return 0
def build_env(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
if args.docker_build:
docker_build.build_with_docker(args.output_folder, sys.argv)
return
# Checking conda-build existence if --docker_build is not specified
utils.check_if_conda_build_exists()
# Here, importing BuildTree is intentionally done after checking
# existence of conda-build as BuildTree uses conda_build APIs.
from build_tree import BuildTree # pylint: disable=import-outside-toplevel
common_package_build_args = []
common_package_build_args += [
"--output_folder",
os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--channel", os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--conda_build_config",
os.path.abspath(args.conda_build_config)
]
for channel in args.channels_list:
common_package_build_args += ["--channels", channel]
# If repository_folder doesn't exist, create it
if args.repository_folder and not os.path.exists(args.repository_folder):
os.mkdir(args.repository_folder)
# Create the build tree
build_tree = BuildTree(env_config_files=args.env_config_file,
python_versions=utils.parse_arg_list(
args.python_versions),
build_types=utils.parse_arg_list(args.build_types),
mpi_types=utils.parse_arg_list(args.mpi_types),
repository_folder=args.repository_folder,
git_location=args.git_location,
git_tag_for_env=args.git_tag_for_env,
conda_build_config=args.conda_build_config)
conda_env_data = CondaEnvFileGenerator(
python_versions=args.python_versions,
build_types=args.build_types,
mpi_types=args.mpi_types,
channels=args.channels_list,
output_folder=os.path.abspath(args.output_folder),
)
# Build each package in the packages list
for build_command in build_tree:
build_args = common_package_build_args + build_command.feedstock_args()
try:
build_feedstock.build_feedstock(build_args)
except OpenCEError as exc:
raise OpenCEError(Error.BUILD_RECIPE, build_command.repository,
exc.msg) from exc
conda_env_data.update_conda_env_file_content(build_command, build_tree)
conda_env_files = conda_env_data.write_conda_env_files()
print(
"Generated conda environment files from the selected build arguments:",
conda_env_files)
print("INFO: One can use these environment files to create a conda" \
" environment using \"conda env create -f <conda_env_file_name>.\"")
def test_parse_arg_list_list_input():
'''
Parse arg list should return the input argument if it's already a list.
'''
list_input = ["a", "b", "c"]
assert list_input == utils.parse_arg_list(list_input)
def build_feedstock(args_string=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(args_string)
saved_working_directory = None
if args.working_directory:
saved_working_directory = os.getcwd()
os.chdir(os.path.abspath(args.working_directory))
build_config_data, recipe_config_file = load_package_config(
args.recipe_config_file)
if build_config_data is None:
return 1
args.recipes = utils.parse_arg_list(args.recipe_list)
result = 0
# Build each recipe
for recipe in build_config_data['recipes']:
if args.recipes and recipe['name'] not in args.recipes:
continue
conda_build_args = "conda-build "
conda_build_args += "--skip-existing "
conda_build_args += "--output-folder " + args.output_folder + " "
conda_build_args += "-m " + args.conda_build_config + " "
recipe_conda_build_config = os.path.join(os.getcwd(), "config",
"conda_build_config.yaml")
if os.path.exists(recipe_conda_build_config):
conda_build_args += " -m " + recipe_conda_build_config + " "
conda_build_args += " ".join(
["-c " + c + " " for c in args.channels_list])
conda_build_args += " ".join(
["-c " + c + " " for c in build_config_data.get('channels', [])])
variants = dict()
if args.python_versions:
variants['python'] = utils.parse_arg_list(args.python_versions)
if args.build_types:
variants['build_type'] = utils.parse_arg_list(args.build_types)
if variants:
conda_build_args += "--variants \"" + str(variants) + "\" "
conda_build_args += recipe['path']
result = _set_local_src_dir(args.local_src_dir, recipe,
recipe_config_file)
if result != 0:
break
print(conda_build_args)
result = os.system(conda_build_args)
if result != 0:
print("Failure building recipe: " +
(recipe['name'] if 'name' in recipe else os.getcwd))
result = 1
break
if saved_working_directory:
os.chdir(saved_working_directory)
return result
def build_env(arg_strings=None):
'''
Entry function.
'''
parser = make_parser()
args = parser.parse_args(arg_strings)
result = 0
python_build_args = []
common_package_build_args = []
common_package_build_args += [
"--output_folder",
os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--channel", os.path.abspath(args.output_folder)
]
common_package_build_args += [
"--conda_build_config",
os.path.abspath(args.conda_build_config)
]
if args.build_types:
common_package_build_args += ["--build_types", args.build_types]
for channel in args.channels_list:
common_package_build_args += ["--channels", channel]
# If repository_folder doesn't exist, create it
if args.repository_folder and not os.path.exists(args.repository_folder):
os.mkdir(args.repository_folder)
for py_vers in utils.parse_arg_list(args.python_versions):
print("Builds for python version: " + py_vers)
python_build_args = ["--python_versions", py_vers]
variants = {
'python': py_vers,
'build_type': utils.parse_arg_list(args.build_types)
}
result, recipes = create_all_recipes(
args.env_config_file,
variants,
repository_folder=args.repository_folder,
git_location=args.git_location,
git_tag_for_env=args.git_tag_for_env,
conda_build_config=args.conda_build_config)
if result != 0:
return result
# Add dependency tree information to the packages list
dep_tree = _create_dep_tree(recipes)
# Build each package in the packages list
for recipe in _traverse_recipes(recipes, dep_tree):
package_build_args = ["--working_directory", recipe['repository']]
for channel in recipe.get('channels', []):
package_build_args += ["--channels", channel]
if 'recipe' in recipe:
package_build_args += ["--recipes", recipe['recipe']]
build_args = common_package_build_args + python_build_args + package_build_args
result = build_feedstock.build_feedstock(build_args)
if result != 0:
print("Unable to build recipe: " + recipe['repository'])
return result
return result
