def cli_list(cmd, raw_args): """List all kipoi-induced conda environments """ entries = get_model_env_db().get_all(only_valid=True) if len(entries) != 0: print("# Functional kipoi environments:") print_valid_env_names(entries) invalid_entries = get_model_env_db().get_all_unfinished() if len(invalid_entries) != 0: print("# Non-Functional kipoi environments:") print_invalid_env_names(invalid_entries) print("# Conda environments starting with kipoi:") subprocess.call("conda env list | grep ^kipoi | cut -f 1 -d ' '", shell=True)
def delete_envs(to_delete): db = get_model_env_db() for e in to_delete: try: kipoi_conda.remove_env(e.create_args.env) db.remove(e) db.save() except Exception as err: logger.warning( "Conda delete of environment {0} failed with error: {1}. This environment entry was not " "removed from the database.".format(e.create_args.env, str(err)))
def rm_env(env_name): """Alias for remove_env """ from kipoi.env_db import get_model_env_db if env_exists(env_name): logger.info("Removing environment: {0}".format(env_name)) remove_env(env_name) # remove from db db = get_model_env_db() db_entries = [e for e in db.get_all() if e.create_args.env == env_name] [db.remove(e) for e in db_entries] db.save()
def get_envs_by_model(models, source, only_most_recent=True, only_valid=False): if isinstance(models, str): models = [models] source_path = kipoi.get_source(source).local_path entries = [] db = env_db.get_model_env_db() for m in models: res = db.get_entry_by_model(_env_db_model_name(source, m), only_most_recent=only_most_recent, only_valid=only_valid) if only_most_recent: entries.append(res) else: entries.extend(res) entries = [e for e in entries if e is not None] return entries
def ask_and_delete_envs(to_delete, args): db = get_model_env_db() del_only_db = False if "db" in dir(args) and args.db: del_only_db = True if not del_only_db: warn_msg = "Are you sure you want to delete the following environments:\n" else: warn_msg = "Are you sure you want to delete the following environments from the database ONLY:\n" warn_msg += "\n".join( ["{0} ({1})".format(e.create_args.env, e.cli_path) for e in to_delete]) warn_msg += "\n" if args.yes or confirm(warn_msg): if not del_only_db: delete_envs(to_delete) else: for e in to_delete: db.remove(e) db.save()
def cli_cleanup(cmd, raw_args): """Remove all environments that have failed during setup. Or remove all environments """ parser = argparse.ArgumentParser( 'kipoi env {}'.format(cmd), description='Clean-up conda environments.') parser.add_argument('-a', '--all', action='store_true', help="If set all environments will be removed!") parser.add_argument( '-d', '--db', action='store_true', help= "Clean-up DB only. Make sure that you run this only if `kipoi env cleanup` has been " "executed first. Running this will remove environments only from the kipoi database and " "will NOT attempt to remove conda environments") parser.add_argument( '-y', '--yes', action='store_true', help="If set then do NOT ask before deleting environments.") args = parser.parse_args(raw_args) db = get_model_env_db() if not args.all: to_delete = db.get_all_unfinished() else: to_delete = db.get_all() if len(to_delete) != 0: ask_and_delete_envs(to_delete, args) else: logger.info("Nothing to clean up!") logger.info("Done!")
def cli_create(cmd, raw_args): """Create a conda environment for a model """ from kipoi_conda import get_kipoi_bin import uuid parser = argparse.ArgumentParser( 'kipoi env {}'.format(cmd), description='Create a conda environment for a specific model.') add_env_args(parser) parser.add_argument( '-e', '--env', default=None, help="Special environment name. default: kipoi-<model>[-<dataloader>]") parser.add_argument('--dry-run', action='store_true', help="Don't actually create the environment") parser.add_argument( '-t', '--tmpdir', default=None, help= ("Temporary directory path where to create the conda environment file" "Defaults to /tmp/kipoi/envfiles/<uuid>/")) parser.add_argument('-v', '--verbose', action='store_true', help="Increase output verbosity. Show conda stdout.") args = parser.parse_args(raw_args) # create the tmp dir if args.tmpdir is None: tmpdir = "/tmp/kipoi/envfiles/" + str(uuid.uuid4())[:8] else: tmpdir = args.tmpdir if not os.path.exists(tmpdir): os.makedirs(tmpdir) # write the env file logger.info("Writing environment file: {0}".format(tmpdir)) if args.model == ['all']: from kipoi.cli.source_test import get_common_env src = kipoi.get_source(args.source) model_envs = yaml.safe_load( open( os.path.join(src.local_path, SPECIAL_ENV_PREFIX, "models.yaml"))) # TODO - test this by mocking up the CLI command execution # setup the args for all the models df = kipoi.list_models() dfg = list_models_by_group(df, "") for model_group in dfg.group.unique().tolist(): existing_envs = get_envs_by_model(model_group, args.source, only_valid=True) if existing_envs or existing_envs is None: # No need to create the environment existing_envs_str = "\n".join( [e.create_args.env for e in existing_envs]) logger.info( "Environment for {} already exists ({}). Skipping installation" .format(model_group, existing_envs_str)) continue logger.info( "Environment doesn't exists for model group {}. Installing it". format(model_group)) # Figure out which <model> to use for installation common_env = get_common_env(model_group, model_envs) if common_env is not None: # common environment exists for the model. Use it logger.info("Using common environment: {}".format(common_env)) model_group = os.path.join(SPECIAL_ENV_PREFIX, common_env) # Run cli_create def optional_replace(x, ref, alt): if x == ref: return alt else: return x new_raw_args = [ optional_replace(x, 'all', model_group) for x in raw_args if x is not None ] cli_create(cmd, new_raw_args) logger.info("Done installing all environments!") return None env, env_file = export_env(args.model, args.dataloader, args.source, env_file=None, env_dir=tmpdir, env=args.env, vep=args.vep, interpret=args.interpret, gpu=args.gpu) if not args.dry_run: env_db_entry = generate_env_db_entry(args, args_env_overload=env) envdb = get_model_env_db() envdb.append(env_db_entry) envdb.save() # setup the conda env from file logger.info("Creating conda env from file: {0}".format(env_file)) kipoi_conda.create_env_from_file(env_file, use_stdout=args.verbose) env_db_entry.successful = True # env is environment name env_db_entry.cli_path = get_kipoi_bin(env) get_model_env_db().save() logger.info("Done!") print("\nActivate the environment via:") print("conda activate {0}".format(env)) else: print("Dry run. Conda file path: {}".format(env_file))