def test_generate_env_db_entry():
# test in general and test whether the automatic generation of sub-models works, also in combination
# with a clearly defined model
import yaml
import kipoi
import time
from kipoi.cli.parser_utils import parse_source_name
kwargs = {
"dataloader": [],
"env": "test_env",
"gpu": True,
"model": None,
"source": "dir",
"tmpdir": "something",
"vep": True
}
source_path = kipoi.get_source("dir").local_path
kipoi_path = kipoi.get_source("kipoi").local_path
for model in [["example/models/pyt"],
[
"example/models/shared/envs/kipoi-py3-keras1.2",
"example/models/pyt"
]]:
kwargs['model'] = model
db_entry = generate_env_db_entry(get_args(kwargs)())
assert all(
[kwargs[k] == getattr(db_entry.create_args, k) for k in kwargs])
# generate the reference output
special_envs, only_models = split_models_special_envs(model)
sub_models = []
for model in only_models:
parsed_source, parsed_model = parse_source_name(
kwargs["source"], model)
sub_models.extend([
os.path.join(source_path, e) for e in list_subcomponents(
parsed_model, parsed_source, "model")
])
if len(special_envs) != 0:
with open("example/models/shared/envs/models.yaml", "r") as fh:
special_env_models = yaml.load(fh)
for special_env in special_envs:
for model_group_name in special_env_models[os.path.basename(
special_env)]:
sub_models.extend([
os.path.join(kipoi_path,
e) for e in list_subcomponents(
model_group_name, "kipoi", "model")
])
assert set(db_entry.compatible_models) == set(sub_models)
assert db_entry.cli_path is None
assert db_entry.successful == False
assert db_entry.kipoi_version == kipoi.__version__
assert db_entry.timestamp < time.time()
def cli_pull(command, raw_args):
"""Pull the repository
"""
assert command == "pull"
parser = argparse.ArgumentParser('kipoi {}'.format(command),
description="Downloads the directory" +
" associated with the model.")
parser.add_argument(
'model',
help='Model name. '
'<model> can also refer to a model-group - e.g. if you '
'specify MaxEntScan then the dependencies\n'
'for MaxEntScan/5prime and MaxEntScan/3prime will be installed')
add_source(parser)
args = parser.parse_args(raw_args)
src = kipoi.config.get_source(args.source)
sub_models = list_subcomponents(args.model, args.source, which='model')
if len(sub_models) == 0:
logger.error("Model {0} not found in source {1}".format(
args.model, args.source))
sys.exit(1)
if len(sub_models) > 1:
logger.info(
"Found {0} models under the model name: {1}. Pulling all of them".
format(len(sub_models), args.model))
for sub_model in sub_models:
src.pull_model(sub_model)
def generate_env_db_entry(args, args_env_overload=None):
from collections import OrderedDict
from kipoi.env_db import EnvDbEntry
from kipoi_conda import get_conda_version
special_envs, only_models = split_models_special_envs(args.model)
sub_models = []
for model in only_models:
parsed_source, parsed_model = parse_source_name(args.source, model)
models = list_subcomponents(parsed_model, parsed_source, "model")
sub_models.extend(
[_env_db_model_name(parsed_source, m) for m in models])
if len(special_envs) != 0:
# for the special envs load the corresponding models:
for special_env in special_envs:
special_env_folder = "/".join(
special_env.rstrip("/").split("/")[:-1])
source_path = kipoi.get_source(args.source).local_path
with open(os.path.join(source_path, special_env_folder,
"models.yaml"),
"r",
encoding="utf-8") as fh:
special_env_models = yaml.safe_load(fh)
# extend the sub_models by all the submodels covered by the handcrafted environments (special_envs)
# Those models **always** refer to the kipoi source
for model_group_name in special_env_models[os.path.basename(
special_env)]:
models = list_subcomponents(model_group_name, "kipoi", "model")
sub_models.extend(
[_env_db_model_name("kipoi", m) for m in models])
entry = EnvDbEntry(conda_version=get_conda_version(),
kipoi_version=kipoi.__version__,
timestamp=time.time(),
compatible_models=sub_models,
create_args=OrderedDict(args._get_kwargs()))
if args_env_overload is not None:
entry.create_args.env = args_env_overload
return entry
def test_list_submodules():
assert set(list_subcomponents("MaxEntScan", "kipoi", "model")) == {
"MaxEntScan/3prime", "MaxEntScan/5prime"
}
assert set(list_subcomponents("Basenji", "kipoi", "model")) == {"Basenji"}
def merge_deps(models,
dataloaders=None,
source="kipoi",
vep=False,
interpret=False,
gpu=False):
"""Setup the dependencies
"""
special_envs, only_models = split_models_special_envs(models)
deps = Dependencies()
# Treat the handcrafted environments differently
for special_env in special_envs:
from related import from_yaml
logger.info("Loading environment definition: {0}".format(special_env))
# Load and merge the handcrafted deps.
yaml_path = os.path.join(
kipoi.get_source(source).local_path, special_env + ".yaml")
if not os.path.exists(yaml_path):
raise ValueError(
"Environment definition file {0} not found in source {1}".
format(yaml_path, source))
with open(yaml_path, "r", encoding="utf-8") as fh:
special_env_deps = Dependencies.from_env_dict(from_yaml(fh))
deps = deps.merge(special_env_deps)
for model in only_models:
logger.info("Loading model: {0} description".format(model))
parsed_source, parsed_model = parse_source_name(source, model)
sub_models = list_subcomponents(parsed_model, parsed_source, "model")
if len(sub_models) == 0:
raise ValueError("Model {0} not found in source {1}".format(
parsed_model, parsed_source))
if len(sub_models) > 1:
logger.info(
"Found {0} models under the model name: {1}. Merging dependencies for all"
.format(len(sub_models), parsed_model))
for sub_model in sub_models:
model_descr = kipoi.get_model_descr(sub_model, parsed_source)
model_dir = kipoi.get_source(parsed_source).get_model_dir(
sub_model)
deps = deps.merge(model_descr.dependencies)
# handle the dataloader=None case
if dataloaders is None or not dataloaders:
if isinstance(model_descr.default_dataloader,
DataLoaderImport):
# dataloader specified by the import
deps = deps.merge(
model_descr.default_dataloader.dependencies)
if model_descr.default_dataloader.parse_dependencies:
# add dependencies specified in the yaml file
# load from the dataloader description if you can
try:
with cd(model_dir):
dataloader_descr = model_descr.default_dataloader.get(
)
deps = deps.merge(dataloader_descr.dependencies)
except ImportError as e:
# package providing the dataloader is not installed yet
if model_descr.default_dataloader.defined_as.startswith(
"kipoiseq."):
logger.info(
"kipoiseq not installed. Using default kipoiseq dependencies for the dataloader: {}"
.format(model_descr.default_dataloader.
defined_as))
deps = deps.merge(KIPOISEQ_DEPS)
else:
logger.warning(
"Unable to extract dataloader description. "
"Make sure the package containing the dataloader `{}` is installed"
.format(model_descr.default_dataloader.
defined_as))
else:
dataloader = os.path.normpath(
os.path.join(sub_model,
str(model_descr.default_dataloader)))
logger.info("Inferred dataloader name: {0} from".format(
dataloader) + " the model.")
dataloader_descr = kipoi.get_dataloader_descr(
dataloader, parsed_source)
deps = deps.merge(dataloader_descr.dependencies)
if dataloaders is not None or dataloaders:
for dataloader in dataloaders:
parsed_source, parsed_dataloader = parse_source_name(
source, dataloader)
sub_dataloaders = list_subcomponents(parsed_dataloader,
parsed_source, "dataloader")
if len(sub_dataloaders) == 0:
raise ValueError(
"Dataloader: {0} not found in source {1}".format(
parsed_dataloader, parsed_source))
if len(sub_dataloaders) > 1:
logger.info(
"Found {0} dataloaders under the dataloader name: {1}. Merging dependencies for all"
.format(len(sub_dataloaders), parsed_dataloader))
for sub_dataloader in sub_dataloaders:
dataloader_descr = kipoi.get_dataloader_descr(
sub_dataloader, parsed_source)
deps = deps.merge(dataloader_descr.dependencies)
# add Kipoi to the dependencies
deps = KIPOI_DEPS.merge(deps)
if vep:
# add vep dependencies
logger.info("Adding the vep dependencies")
deps = VEP_DEPS.merge(deps)
if interpret:
# add vep dependencies
logger.info("Adding the interpret dependencies")
deps = INTERPRET_DEPS.merge(deps)
if gpu:
logger.info("Using gpu-compatible dependencies")
deps = deps.gpu()
if platform == "darwin":
logger.info("Using osx-type dependencies")
deps = deps.osx()
return deps
def merge_deps(models, dataloaders=None, source="kipoi", vep=False, gpu=False):
"""Setup the dependencies
"""
deps = Dependencies()
for model in models:
logger.info("Loading model: {0} description".format(model))
parsed_source, parsed_model = parse_source_name(source, model)
sub_models = list_subcomponents(parsed_model, parsed_source, "model")
if len(sub_models) == 0:
raise ValueError("Model {0} not found in source {1}".format(
parsed_model, parsed_source))
if len(sub_models) > 1:
logger.info(
"Found {0} models under the model name: {1}. Merging dependencies for all"
.format(len(sub_models), parsed_model))
for sub_model in sub_models:
model_descr = kipoi.get_model_descr(sub_model, parsed_source)
model_dir = kipoi.get_source(parsed_source).get_model_dir(
sub_model)
deps = deps.merge(model_descr.dependencies)
# handle the dataloader=None case
if dataloaders is None or not dataloaders:
if isinstance(model_descr.default_dataloader,
DataLoaderImport):
# dataloader specified by the import
deps = deps.merge(
model_descr.default_dataloader.dependencies)
if model_descr.default_dataloader.parse_dependencies:
# add dependencies specified in the yaml file
# load from the dataloader description if you can
try:
with cd(model_dir):
dataloader_descr = model_descr.default_dataloader.get(
)
deps = deps.merge(dataloader_descr.dependencies)
except ImportError as e:
# package providing the dataloader is not installed yet
if model_descr.default_dataloader.defined_as.startswith(
"kipoiseq."):
logger.info(
"kipoiseq not installed. Using default kipoiseq dependencies for the dataloader: {}"
.format(model_descr.default_dataloader.
defined_as))
deps = deps.merge(KIPOISEQ_DEPS)
else:
logger.warn(
"Unable to extract dataloader description. "
"Make sure the package containing the dataloader `{}` is installed"
.format(model_descr.default_dataloader.
defined_as))
else:
dataloader = os.path.normpath(
os.path.join(sub_model,
str(model_descr.default_dataloader)))
logger.info("Inferred dataloader name: {0} from".format(
dataloader) + " the model.")
dataloader_descr = kipoi.get_dataloader_descr(
dataloader, parsed_source)
deps = deps.merge(dataloader_descr.dependencies)
if dataloaders is not None or dataloaders:
for dataloader in dataloaders:
parsed_source, parsed_dataloader = parse_source_name(
source, dataloader)
sub_dataloaders = list_subcomponents(parsed_dataloader,
parsed_source, "dataloader")
if len(sub_dataloaders) == 0:
raise ValueError(
"Dataloader: {0} not found in source {1}".format(
parsed_dataloader, parsed_source))
if len(sub_dataloaders) > 1:
logger.info(
"Found {0} dataloaders under the dataloader name: {1}. Merging dependencies for all"
.format(len(sub_dataloaders), parsed_dataloader))
for sub_dataloader in sub_dataloaders:
dataloader_descr = kipoi.get_dataloader_descr(
sub_dataloader, parsed_source)
deps = deps.merge(dataloader_descr.dependencies)
# add Kipoi to the dependencies
deps = KIPOI_DEPS.merge(deps)
if vep:
# add vep dependencies
logger.info("Adding the vep dependencies")
deps = VEP_DEPS.merge(deps)
if gpu:
logger.info("Using gpu-compatible dependencies")
deps = deps.gpu()
if platform == "darwin":
logger.info("Using osx-type dependencies")
deps = deps.osx()
return deps
def merge_deps(models, dataloaders=None, source="kipoi", vep=False, gpu=False):
"""Setup the dependencies
"""
deps = Dependencies()
for model in models:
logger.info("Loading model: {0} description".format(model))
parsed_source, parsed_model = parse_source_name(source, model)
sub_models = list_subcomponents(parsed_model, parsed_source, "model")
if len(sub_models) == 0:
raise ValueError("Model {0} not found in source {1}".format(
parsed_model, parsed_source))
if len(sub_models) > 1:
logger.info(
"Found {0} models under the model name: {1}. Merging dependencies for all"
.format(len(sub_models), parsed_model))
for sub_model in sub_models:
model_descr = kipoi.get_model_descr(sub_model, parsed_source)
deps = deps.merge(model_descr.dependencies)
# handle the dataloader=None case
if dataloaders is None or not dataloaders:
dataloader = os.path.normpath(
os.path.join(sub_model, model_descr.default_dataloader))
logger.info(
"Inferred dataloader name: {0} from".format(dataloader) +
" the model.")
dataloader_descr = kipoi.get_dataloader_descr(
dataloader, parsed_source)
deps = deps.merge(dataloader_descr.dependencies)
if dataloaders is not None or dataloaders:
for dataloader in dataloaders:
parsed_source, parsed_dataloader = parse_source_name(
source, dataloader)
sub_dataloaders = list_subcomponents(parsed_dataloader,
parsed_source, "dataloader")
if len(sub_dataloaders) == 0:
raise ValueError(
"Dataloader: {0} not found in source {1}".format(
parsed_dataloader, parsed_source))
if len(sub_dataloaders) > 1:
logger.info(
"Found {0} dataloaders under the dataloader name: {1}. Merging dependencies for all"
.format(len(sub_dataloaders), parsed_dataloader))
for sub_dataloader in sub_dataloaders:
dataloader_descr = kipoi.get_dataloader_descr(
sub_dataloader, parsed_source)
deps = deps.merge(dataloader_descr.dependencies)
# add Kipoi to the dependencies
deps = KIPOI_DEPS.merge(deps)
if vep:
# add vep dependencies
logger.info("Adding the vep dependencies")
deps = VEP_DEPS.merge(deps)
if gpu:
logger.info("Using gpu-compatible dependencies")
deps = deps.gpu()
if platform == "darwin":
logger.info("Using osx-type dependencies")
deps = deps.osx()
return deps
