def cli_list(cmd, raw_args):
"""List all kipoi-induced conda environments
"""
entries = get_model_env_db().get_all(only_valid=True)
if len(entries) != 0:
print("# Functional kipoi environments:")
print_valid_env_names(entries)
invalid_entries = get_model_env_db().get_all_unfinished()
if len(invalid_entries) != 0:
print("# Non-Functional kipoi environments:")
print_invalid_env_names(invalid_entries)
print("# Conda environments starting with kipoi:")
subprocess.call("conda env list | grep ^kipoi | cut -f 1 -d ' '",
shell=True)
def delete_envs(to_delete):
db = get_model_env_db()
for e in to_delete:
try:
kipoi_conda.remove_env(e.create_args.env)
db.remove(e)
db.save()
except Exception as err:
logger.warning(
"Conda delete of environment {0} failed with error: {1}. This environment entry was not "
"removed from the database.".format(e.create_args.env,
str(err)))
def rm_env(env_name):
"""Alias for remove_env
"""
from kipoi.env_db import get_model_env_db
if env_exists(env_name):
logger.info("Removing environment: {0}".format(env_name))
remove_env(env_name)
# remove from db
db = get_model_env_db()
db_entries = [e for e in db.get_all() if e.create_args.env == env_name]
[db.remove(e) for e in db_entries]
db.save()
def get_envs_by_model(models, source, only_most_recent=True, only_valid=False):
if isinstance(models, str):
models = [models]
source_path = kipoi.get_source(source).local_path
entries = []
db = env_db.get_model_env_db()
for m in models:
res = db.get_entry_by_model(_env_db_model_name(source, m),
only_most_recent=only_most_recent,
only_valid=only_valid)
if only_most_recent:
entries.append(res)
else:
entries.extend(res)
entries = [e for e in entries if e is not None]
return entries
def ask_and_delete_envs(to_delete, args):
db = get_model_env_db()
del_only_db = False
if "db" in dir(args) and args.db:
del_only_db = True
if not del_only_db:
warn_msg = "Are you sure you want to delete the following environments:\n"
else:
warn_msg = "Are you sure you want to delete the following environments from the database ONLY:\n"
warn_msg += "\n".join(
["{0} ({1})".format(e.create_args.env, e.cli_path) for e in to_delete])
warn_msg += "\n"
if args.yes or confirm(warn_msg):
if not del_only_db:
delete_envs(to_delete)
else:
for e in to_delete:
db.remove(e)
db.save()
def cli_cleanup(cmd, raw_args):
"""Remove all environments that have failed during setup. Or remove all environments
"""
parser = argparse.ArgumentParser(
'kipoi env {}'.format(cmd), description='Clean-up conda environments.')
parser.add_argument('-a',
'--all',
action='store_true',
help="If set all environments will be removed!")
parser.add_argument(
'-d',
'--db',
action='store_true',
help=
"Clean-up DB only. Make sure that you run this only if `kipoi env cleanup` has been "
"executed first. Running this will remove environments only from the kipoi database and "
"will NOT attempt to remove conda environments")
parser.add_argument(
'-y',
'--yes',
action='store_true',
help="If set then do NOT ask before deleting environments.")
args = parser.parse_args(raw_args)
db = get_model_env_db()
if not args.all:
to_delete = db.get_all_unfinished()
else:
to_delete = db.get_all()
if len(to_delete) != 0:
ask_and_delete_envs(to_delete, args)
else:
logger.info("Nothing to clean up!")
logger.info("Done!")
def cli_create(cmd, raw_args):
"""Create a conda environment for a model
"""
from kipoi_conda import get_kipoi_bin
import uuid
parser = argparse.ArgumentParser(
'kipoi env {}'.format(cmd),
description='Create a conda environment for a specific model.')
add_env_args(parser)
parser.add_argument(
'-e',
'--env',
default=None,
help="Special environment name. default: kipoi-<model>[-<dataloader>]")
parser.add_argument('--dry-run',
action='store_true',
help="Don't actually create the environment")
parser.add_argument(
'-t',
'--tmpdir',
default=None,
help=
("Temporary directory path where to create the conda environment file"
"Defaults to /tmp/kipoi/envfiles/<uuid>/"))
parser.add_argument('-v',
'--verbose',
action='store_true',
help="Increase output verbosity. Show conda stdout.")
args = parser.parse_args(raw_args)
# create the tmp dir
if args.tmpdir is None:
tmpdir = "/tmp/kipoi/envfiles/" + str(uuid.uuid4())[:8]
else:
tmpdir = args.tmpdir
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
# write the env file
logger.info("Writing environment file: {0}".format(tmpdir))
if args.model == ['all']:
from kipoi.cli.source_test import get_common_env
src = kipoi.get_source(args.source)
model_envs = yaml.safe_load(
open(
os.path.join(src.local_path, SPECIAL_ENV_PREFIX,
"models.yaml")))
# TODO - test this by mocking up the CLI command execution
# setup the args for all the models
df = kipoi.list_models()
dfg = list_models_by_group(df, "")
for model_group in dfg.group.unique().tolist():
existing_envs = get_envs_by_model(model_group,
args.source,
only_valid=True)
if existing_envs or existing_envs is None:
# No need to create the environment
existing_envs_str = "\n".join(
[e.create_args.env for e in existing_envs])
logger.info(
"Environment for {} already exists ({}). Skipping installation"
.format(model_group, existing_envs_str))
continue
logger.info(
"Environment doesn't exists for model group {}. Installing it".
format(model_group))
# Figure out which <model> to use for installation
common_env = get_common_env(model_group, model_envs)
if common_env is not None:
# common environment exists for the model. Use it
logger.info("Using common environment: {}".format(common_env))
model_group = os.path.join(SPECIAL_ENV_PREFIX, common_env)
# Run cli_create
def optional_replace(x, ref, alt):
if x == ref:
return alt
else:
return x
new_raw_args = [
optional_replace(x, 'all', model_group) for x in raw_args
if x is not None
]
cli_create(cmd, new_raw_args)
logger.info("Done installing all environments!")
return None
env, env_file = export_env(args.model,
args.dataloader,
args.source,
env_file=None,
env_dir=tmpdir,
env=args.env,
vep=args.vep,
interpret=args.interpret,
gpu=args.gpu)
if not args.dry_run:
env_db_entry = generate_env_db_entry(args, args_env_overload=env)
envdb = get_model_env_db()
envdb.append(env_db_entry)
envdb.save()
# setup the conda env from file
logger.info("Creating conda env from file: {0}".format(env_file))
kipoi_conda.create_env_from_file(env_file, use_stdout=args.verbose)
env_db_entry.successful = True
# env is environment name
env_db_entry.cli_path = get_kipoi_bin(env)
get_model_env_db().save()
logger.info("Done!")
print("\nActivate the environment via:")
print("conda activate {0}".format(env))
else:
print("Dry run. Conda file path: {}".format(env_file))