def merge_deps(models, dataloaders=None, source="kipoi", vep=False, gpu=False): deps = Dependencies() for model in models: logger.info("Loading model: {0} description".format(model)) parsed_source, parsed_model = parse_source_name(source, model) model_descr = kipoi.get_model_descr(parsed_model, parsed_source) deps = deps.merge(model_descr.dependencies) # handle the dataloader=None case if dataloaders is None or not dataloaders: dataloader = os.path.normpath(os.path.join(parsed_model, model_descr.default_dataloader)) logger.info("Inferred dataloader name: {0} from".format(dataloader) + " the model.") dataloader_descr = kipoi.get_dataloader_descr(dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) if dataloaders is not None or dataloaders: for dataloader in dataloaders: parsed_source, parsed_dataloader = parse_source_name(source, dataloader) dataloader_descr = kipoi.get_dataloader_descr(parsed_dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) # add Kipoi to the dependencies deps = KIPOI_DEPS.merge(deps) if vep: # add vep dependencies logger.info("Adding the vep dependencies") deps = VEP_DEPS.merge(deps) if gpu: logger.info("Using gpu-compatible dependencies") deps = deps.gpu() if platform == "darwin": logger.info("Using osx-type dependencies") deps = deps.osx() return deps
def test_generate_env_db_entry(): # test in general and test whether the automatic generation of sub-models works, also in combination # with a clearly defined model import yaml import kipoi import time from kipoi.cli.parser_utils import parse_source_name kwargs = { "dataloader": [], "env": "test_env", "gpu": True, "model": None, "source": "dir", "tmpdir": "something", "vep": True } source_path = kipoi.get_source("dir").local_path kipoi_path = kipoi.get_source("kipoi").local_path for model in [["example/models/pyt"], [ "example/models/shared/envs/kipoi-py3-keras1.2", "example/models/pyt" ]]: kwargs['model'] = model db_entry = generate_env_db_entry(get_args(kwargs)()) assert all( [kwargs[k] == getattr(db_entry.create_args, k) for k in kwargs]) # generate the reference output special_envs, only_models = split_models_special_envs(model) sub_models = [] for model in only_models: parsed_source, parsed_model = parse_source_name( kwargs["source"], model) sub_models.extend([ os.path.join(source_path, e) for e in list_subcomponents( parsed_model, parsed_source, "model") ]) if len(special_envs) != 0: with open("example/models/shared/envs/models.yaml", "r") as fh: special_env_models = yaml.load(fh) for special_env in special_envs: for model_group_name in special_env_models[os.path.basename( special_env)]: sub_models.extend([ os.path.join(kipoi_path, e) for e in list_subcomponents( model_group_name, "kipoi", "model") ]) assert set(db_entry.compatible_models) == set(sub_models) assert db_entry.cli_path is None assert db_entry.successful == False assert db_entry.kipoi_version == kipoi.__version__ assert db_entry.timestamp < time.time()
def generate_env_db_entry(args, args_env_overload=None): from collections import OrderedDict from kipoi.env_db import EnvDbEntry from kipoi_conda import get_conda_version special_envs, only_models = split_models_special_envs(args.model) sub_models = [] for model in only_models: parsed_source, parsed_model = parse_source_name(args.source, model) models = list_subcomponents(parsed_model, parsed_source, "model") sub_models.extend( [_env_db_model_name(parsed_source, m) for m in models]) if len(special_envs) != 0: # for the special envs load the corresponding models: for special_env in special_envs: special_env_folder = "/".join( special_env.rstrip("/").split("/")[:-1]) source_path = kipoi.get_source(args.source).local_path with open(os.path.join(source_path, special_env_folder, "models.yaml"), "r", encoding="utf-8") as fh: special_env_models = yaml.safe_load(fh) # extend the sub_models by all the submodels covered by the handcrafted environments (special_envs) # Those models **always** refer to the kipoi source for model_group_name in special_env_models[os.path.basename( special_env)]: models = list_subcomponents(model_group_name, "kipoi", "model") sub_models.extend( [_env_db_model_name("kipoi", m) for m in models]) entry = EnvDbEntry(conda_version=get_conda_version(), kipoi_version=kipoi.__version__, timestamp=time.time(), compatible_models=sub_models, create_args=OrderedDict(args._get_kwargs())) if args_env_overload is not None: entry.create_args.env = args_env_overload return entry
def merge_deps(models, dataloaders=None, source="kipoi", vep=False, interpret=False, gpu=False): """Setup the dependencies """ special_envs, only_models = split_models_special_envs(models) deps = Dependencies() # Treat the handcrafted environments differently for special_env in special_envs: from related import from_yaml logger.info("Loading environment definition: {0}".format(special_env)) # Load and merge the handcrafted deps. yaml_path = os.path.join( kipoi.get_source(source).local_path, special_env + ".yaml") if not os.path.exists(yaml_path): raise ValueError( "Environment definition file {0} not found in source {1}". format(yaml_path, source)) with open(yaml_path, "r", encoding="utf-8") as fh: special_env_deps = Dependencies.from_env_dict(from_yaml(fh)) deps = deps.merge(special_env_deps) for model in only_models: logger.info("Loading model: {0} description".format(model)) parsed_source, parsed_model = parse_source_name(source, model) sub_models = list_subcomponents(parsed_model, parsed_source, "model") if len(sub_models) == 0: raise ValueError("Model {0} not found in source {1}".format( parsed_model, parsed_source)) if len(sub_models) > 1: logger.info( "Found {0} models under the model name: {1}. Merging dependencies for all" .format(len(sub_models), parsed_model)) for sub_model in sub_models: model_descr = kipoi.get_model_descr(sub_model, parsed_source) model_dir = kipoi.get_source(parsed_source).get_model_dir( sub_model) deps = deps.merge(model_descr.dependencies) # handle the dataloader=None case if dataloaders is None or not dataloaders: if isinstance(model_descr.default_dataloader, DataLoaderImport): # dataloader specified by the import deps = deps.merge( model_descr.default_dataloader.dependencies) if model_descr.default_dataloader.parse_dependencies: # add dependencies specified in the yaml file # load from the dataloader description if you can try: with cd(model_dir): dataloader_descr = model_descr.default_dataloader.get( ) deps = deps.merge(dataloader_descr.dependencies) except ImportError as e: # package providing the dataloader is not installed yet if model_descr.default_dataloader.defined_as.startswith( "kipoiseq."): logger.info( "kipoiseq not installed. Using default kipoiseq dependencies for the dataloader: {}" .format(model_descr.default_dataloader. defined_as)) deps = deps.merge(KIPOISEQ_DEPS) else: logger.warning( "Unable to extract dataloader description. " "Make sure the package containing the dataloader `{}` is installed" .format(model_descr.default_dataloader. defined_as)) else: dataloader = os.path.normpath( os.path.join(sub_model, str(model_descr.default_dataloader))) logger.info("Inferred dataloader name: {0} from".format( dataloader) + " the model.") dataloader_descr = kipoi.get_dataloader_descr( dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) if dataloaders is not None or dataloaders: for dataloader in dataloaders: parsed_source, parsed_dataloader = parse_source_name( source, dataloader) sub_dataloaders = list_subcomponents(parsed_dataloader, parsed_source, "dataloader") if len(sub_dataloaders) == 0: raise ValueError( "Dataloader: {0} not found in source {1}".format( parsed_dataloader, parsed_source)) if len(sub_dataloaders) > 1: logger.info( "Found {0} dataloaders under the dataloader name: {1}. Merging dependencies for all" .format(len(sub_dataloaders), parsed_dataloader)) for sub_dataloader in sub_dataloaders: dataloader_descr = kipoi.get_dataloader_descr( sub_dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) # add Kipoi to the dependencies deps = KIPOI_DEPS.merge(deps) if vep: # add vep dependencies logger.info("Adding the vep dependencies") deps = VEP_DEPS.merge(deps) if interpret: # add vep dependencies logger.info("Adding the interpret dependencies") deps = INTERPRET_DEPS.merge(deps) if gpu: logger.info("Using gpu-compatible dependencies") deps = deps.gpu() if platform == "darwin": logger.info("Using osx-type dependencies") deps = deps.osx() return deps
def merge_deps(models, dataloaders=None, source="kipoi", vep=False, gpu=False): """Setup the dependencies """ deps = Dependencies() for model in models: logger.info("Loading model: {0} description".format(model)) parsed_source, parsed_model = parse_source_name(source, model) sub_models = list_subcomponents(parsed_model, parsed_source, "model") if len(sub_models) == 0: raise ValueError("Model {0} not found in source {1}".format( parsed_model, parsed_source)) if len(sub_models) > 1: logger.info( "Found {0} models under the model name: {1}. Merging dependencies for all" .format(len(sub_models), parsed_model)) for sub_model in sub_models: model_descr = kipoi.get_model_descr(sub_model, parsed_source) model_dir = kipoi.get_source(parsed_source).get_model_dir( sub_model) deps = deps.merge(model_descr.dependencies) # handle the dataloader=None case if dataloaders is None or not dataloaders: if isinstance(model_descr.default_dataloader, DataLoaderImport): # dataloader specified by the import deps = deps.merge( model_descr.default_dataloader.dependencies) if model_descr.default_dataloader.parse_dependencies: # add dependencies specified in the yaml file # load from the dataloader description if you can try: with cd(model_dir): dataloader_descr = model_descr.default_dataloader.get( ) deps = deps.merge(dataloader_descr.dependencies) except ImportError as e: # package providing the dataloader is not installed yet if model_descr.default_dataloader.defined_as.startswith( "kipoiseq."): logger.info( "kipoiseq not installed. Using default kipoiseq dependencies for the dataloader: {}" .format(model_descr.default_dataloader. defined_as)) deps = deps.merge(KIPOISEQ_DEPS) else: logger.warn( "Unable to extract dataloader description. " "Make sure the package containing the dataloader `{}` is installed" .format(model_descr.default_dataloader. defined_as)) else: dataloader = os.path.normpath( os.path.join(sub_model, str(model_descr.default_dataloader))) logger.info("Inferred dataloader name: {0} from".format( dataloader) + " the model.") dataloader_descr = kipoi.get_dataloader_descr( dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) if dataloaders is not None or dataloaders: for dataloader in dataloaders: parsed_source, parsed_dataloader = parse_source_name( source, dataloader) sub_dataloaders = list_subcomponents(parsed_dataloader, parsed_source, "dataloader") if len(sub_dataloaders) == 0: raise ValueError( "Dataloader: {0} not found in source {1}".format( parsed_dataloader, parsed_source)) if len(sub_dataloaders) > 1: logger.info( "Found {0} dataloaders under the dataloader name: {1}. Merging dependencies for all" .format(len(sub_dataloaders), parsed_dataloader)) for sub_dataloader in sub_dataloaders: dataloader_descr = kipoi.get_dataloader_descr( sub_dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) # add Kipoi to the dependencies deps = KIPOI_DEPS.merge(deps) if vep: # add vep dependencies logger.info("Adding the vep dependencies") deps = VEP_DEPS.merge(deps) if gpu: logger.info("Using gpu-compatible dependencies") deps = deps.gpu() if platform == "darwin": logger.info("Using osx-type dependencies") deps = deps.osx() return deps
def merge_deps(models, dataloaders=None, source="kipoi", vep=False, gpu=False): """Setup the dependencies """ deps = Dependencies() for model in models: logger.info("Loading model: {0} description".format(model)) parsed_source, parsed_model = parse_source_name(source, model) sub_models = list_subcomponents(parsed_model, parsed_source, "model") if len(sub_models) == 0: raise ValueError("Model {0} not found in source {1}".format( parsed_model, parsed_source)) if len(sub_models) > 1: logger.info( "Found {0} models under the model name: {1}. Merging dependencies for all" .format(len(sub_models), parsed_model)) for sub_model in sub_models: model_descr = kipoi.get_model_descr(sub_model, parsed_source) deps = deps.merge(model_descr.dependencies) # handle the dataloader=None case if dataloaders is None or not dataloaders: dataloader = os.path.normpath( os.path.join(sub_model, model_descr.default_dataloader)) logger.info( "Inferred dataloader name: {0} from".format(dataloader) + " the model.") dataloader_descr = kipoi.get_dataloader_descr( dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) if dataloaders is not None or dataloaders: for dataloader in dataloaders: parsed_source, parsed_dataloader = parse_source_name( source, dataloader) sub_dataloaders = list_subcomponents(parsed_dataloader, parsed_source, "dataloader") if len(sub_dataloaders) == 0: raise ValueError( "Dataloader: {0} not found in source {1}".format( parsed_dataloader, parsed_source)) if len(sub_dataloaders) > 1: logger.info( "Found {0} dataloaders under the dataloader name: {1}. Merging dependencies for all" .format(len(sub_dataloaders), parsed_dataloader)) for sub_dataloader in sub_dataloaders: dataloader_descr = kipoi.get_dataloader_descr( sub_dataloader, parsed_source) deps = deps.merge(dataloader_descr.dependencies) # add Kipoi to the dependencies deps = KIPOI_DEPS.merge(deps) if vep: # add vep dependencies logger.info("Adding the vep dependencies") deps = VEP_DEPS.merge(deps) if gpu: logger.info("Using gpu-compatible dependencies") deps = deps.gpu() if platform == "darwin": logger.info("Using osx-type dependencies") deps = deps.osx() return deps