def test_model(model_name, source_name, env_name, batch_size, vep=False):
"""kipoi test ...
Args:
model_name (str)
source_name: source name
"""
if env_exists(env_name):
logger.info("Environment {0} exists. Removing it.".format(env_name))
remove_env(env_name)
# TODO - if the model is a Keras model, print the Keras config file
# and note which config file got used
# create the model test environment
cmd = "kipoi"
args = ["env", "create",
"--source", source_name,
"--env", env_name,
model_name]
if vep:
# Add --vep to environment installation
args.insert(-1, "--vep")
returncode = _call_command(cmd, args, use_stdout=True)
assert returncode == 0
# run the tests in the environment
cmd = get_kipoi_bin(env_name)
args = ["test",
"--batch_size", str(batch_size),
"--source", source_name,
model_name]
# New, modified path for conda. Source activate namely does the following:
# - CONDA_DEFAULT_ENV=${env_name}
# - CONDA_PREFIX=${env_path}
# - PATH=$conda_bin:$PATH
new_env = os.environ.copy()
new_env['PATH'] = os.path.dirname(cmd) + os.pathsep + new_env['PATH']
returncode, logs = _call_command(cmd, args, use_stdout=True,
return_logs_with_stdout=True,
env=new_env
)
assert returncode == 0
# detect WARNING in the output log
warn = 0
for line in logs:
warn_start = escape_codes[default_log_colors['WARNING']] + \
'WARNING' + escape_codes['reset']
if line.startswith(warn_start):
logger.error("Warning present: {0}".format(line))
warn += 1
if warn > 0:
raise ValueError("{0} warnings were observed for model {1}".
format(warn, model_name))
def test_model(model_name, source_name, env_name, batch_size):
"""kipoi test ...
Args:
model_name (str)
source_name: source name
"""
if env_exists(env_name):
logger.info("Environment {0} exists. Removing it.".format(env_name))
remove_env(env_name)
# TODO - if the model is a Keras model, print the Keras config file
# and note which config file got used
# create the model test environment
cmd = "kipoi"
args = ["env", "create",
"--source", source_name,
"--env", env_name,
model_name]
returncode = _call_command(cmd, args, use_stdout=True)
assert returncode == 0
# run the tests in the environment
cmd = get_kipoi_bin(env_name)
args = ["test",
"--batch_size", str(batch_size),
"--source", source_name,
model_name]
returncode, logs = _call_command(cmd, args, use_stdout=True,
return_logs_with_stdout=True)
assert returncode == 0
# detect WARNING in the output log
warn = 0
for line in logs:
warn_start = escape_codes[default_log_colors['WARNING']] + \
'WARNING' + escape_codes['reset']
if line.startswith(warn_start):
logger.error("Warning present: {0}".format(line))
warn += 1
if warn > 0:
raise ValueError("{0} warnings were observed for model {1}".
format(warn, model_name))
def test_model(model_name,
source_name,
env_name,
batch_size,
vep=False,
create_env=True):
"""kipoi test ...
Args:
model_name (str)
source_name: source name
"""
if create_env:
create_model_env(model_name, source_name, env_name, vep)
# run the tests in the environment
cmd = get_kipoi_bin(env_name)
args = [
"test", "--batch_size",
str(batch_size), "--source", source_name, model_name
]
# New, modified path for conda. Source activate namely does the following:
# - CONDA_DEFAULT_ENV=${env_name}
# - CONDA_PREFIX=${env_path}
# - PATH=$conda_bin:$PATH
new_env = os.environ.copy()
new_env['PATH'] = os.path.dirname(cmd) + os.pathsep + new_env['PATH']
returncode, logs = _call_command(cmd,
args,
use_stdout=True,
return_logs_with_stdout=True,
env=new_env)
assert returncode == 0
# detect WARNING in the output log
warn = 0
for line in logs:
warn_start = escape_codes[default_log_colors['WARNING']] + \
'WARNING' + escape_codes['reset']
if line.startswith(warn_start):
logger.error("Warning present: {0}".format(line))
warn += 1
if warn > 0:
raise ValueError("{0} warnings were observed for model {1}".format(
warn, model_name))
def modified_files(git_range, source_folder, relative=True):
"""
Returns files under the models dir that have been modified within the git
range. Filenames are returned with the `source_folder` included.
Args:
git_range : list or tuple of length 1 or 2
For example, ['00232ffe', '10fab113'], or commonly ['master', 'HEAD']
or ['master']. If length 2, then the commits are provided to `git diff`
using the triple-dot syntax, `commit1...commit2`. If length 1, the
comparison is any changes in the working tree relative to the commit.
source_folder : str
Root of the model source/git repo
relative=True: return the relative path
"""
assert isinstance(git_range, list)
cmds = ['diff', '--name-only'] + git_range
with cd(source_folder):
code, lines = _call_command("git", cmds, use_stdout=True,
return_logs_with_stdout=True)
assert code == 0
modified = [os.path.join(source_folder, line)
for line in lines]
# exclude files that were deleted in the git-range
existing = list(filter(os.path.exists, modified))
# if the only diff is that files were deleted, we can have ['model/'], so
# filter on existing *files*
existing = list(filter(os.path.isfile, existing))
if relative:
return [os.path.relpath(f, source_folder)
for f in existing]
else:
return existing
def create_model_env(model_name, source_name, env_name, vep=False):
"""kipoi test ...
Args:
model_name (str)
source_name: source name
"""
if env_exists(env_name):
logger.info("Environment {0} exists. Removing it.".format(env_name))
remove_env(env_name)
# TODO - if the model is a Keras model, print the Keras config file
# and note which config file got used
# create the model test environment
cmd = "kipoi"
args = [
"env", "create", "--source", source_name, "--env", env_name, model_name
]
if vep:
# Add --vep to environment installation
args.insert(-1, "--vep")
returncode = _call_command(cmd, args, use_stdout=True)
assert returncode == 0
def singularity_command(kipoi_cmd,
model,
dataloader_kwargs,
output_files=[],
source='kipoi',
dry_run=False):
remote_path = container_remote_url(source)
local_path = container_local_path(remote_path)
singularity_pull(remote_path, local_path)
assert kipoi_cmd[0] == 'kipoi'
# remove all spaces within each command
kipoi_cmd = [
x.replace(" ", "").replace("\n", "").replace("\t", "")
for x in kipoi_cmd
]
# figure out the right environment name
stdout, stderr = _call_command(
'singularity', ['exec', local_path, 'kipoi', 'env', 'get', model],
stdin=subprocess.PIPE)
env_name = stdout.decode().strip()
# create/get the `conda_run` command
conda_run = create_conda_run()
singularity_exec(
local_path,
[conda_run, env_name] + kipoi_cmd,
# kipoi_cmd_conda,
bind_directories=involved_directories(dataloader_kwargs,
output_files,
exclude_dirs=['/tmp', '~']),
dry_run=dry_run)
