def evaluate(self,
dataset,
eval_metric=None,
num_workers=8,
batch_size=256):
lpreds = []
llabels = []
for inputs, targets in tqdm(dataset.batch_train_iter(
cycle=False, num_workers=num_workers, batch_size=batch_size),
total=len(dataset) // batch_size):
assert isinstance(targets, dict)
target_keys = list(targets)
llabels.append(deepcopy(targets))
bpreds = {
k: v
for k, v in self.predict(inputs, batch_size=None).items()
if k in target_keys
} # keep only the target key predictions
lpreds.append(bpreds)
del inputs
del targets
preds = numpy_collate_concat(lpreds)
labels = numpy_collate_concat(llabels)
del lpreds
del llabels
if eval_metric is not None:
return eval_metric(labels, preds)
else:
task_avg_tape = defaultdict(list)
out = {}
for task, heads in self.all_heads.items():
for head_i, head in enumerate(heads):
target_name = head.get_target(task)
if target_name not in labels:
print(
f"Target {target_name} not found. Skipping evaluation"
)
continue
res = head.metric(labels[target_name], preds[target_name])
out[target_name] = res
metrics_dict = flatten(res, separator='/')
for k, v in metrics_dict.items():
task_avg_tape[
head.target_name.replace("{task}", "avg") + "/" +
k].append(v)
for k, v in task_avg_tape.items():
# get the average
out[k] = mean(v)
# flatten everything
out = flatten(out, separator='/')
return out
def load_all(self, **kwargs):
"""Loads and returns the whole dataset
Arguments:
**kwargs: passed to batch_iter()
"""
return numpy_collate_concat([x for x in tqdm(self.batch_iter(**kwargs))])
def batch_write(self, batch):
fbatch = flatten(batch, separator="/")
batch_sizes = [fbatch[k].shape[0] for k in fbatch]
# assert all shapes are the same
assert len(pd.Series(batch_sizes).unique()) == 1
batch_size = batch_sizes[0]
if self.first_pass:
# have a dictionary holding
for k in fbatch:
if fbatch[k].dtype.type in [np.string_, np.str_, np.unicode_]:
dtype = self.string_type
else:
dtype = fbatch[k].dtype
self.f.create_dataset(k,
shape=(0, ) + fbatch[k].shape[1:],
dtype=dtype,
maxshape=(None, ) + fbatch[k].shape[1:],
compression=self.compression,
chunks=(self.chunk_size, ) +
fbatch[k].shape[1:])
self.first_pass = False
# add data to the buffer
if self.write_buffer is None:
self.write_buffer = fbatch
self.write_buffer_size = batch_size
else:
self.write_buffer = numpy_collate_concat(
[self.write_buffer, fbatch])
self.write_buffer_size += batch_size
if self.write_buffer is not None and self.write_buffer_size >= self.chunk_size:
self._flush_buffer()
def feature_importance(model,
dataloader,
importance_score,
importance_score_kwargs={},
batch_size=32,
num_workers=0):
"""Return feature importance scores
"""
ImpScore = get_importance_score(importance_score)
if not ImpScore.is_compatible(model):
raise ValueError(
"model not compatible with score: {0}".format(importance_score))
impscore = ImpScore(model, **importance_score_kwargs)
def append_key(d, k, v):
d[k] = v
return d
# TODO - handle the reference-based importance scores...
return numpy_collate_concat([
append_key(batch, "importance_scores", impscore.score(batch['inputs']))
for batch in tqdm(
dataloader.batch_iter(batch_size=batch_size,
num_workers=num_workers))
])
def load_all(self, batch_size=32, **kwargs):
"""Load the whole dataset into memory
Arguments:
batch_size (int, optional): how many samples per batch to load
(default: 1).
"""
return numpy_collate_concat([x for x in tqdm(self.batch_iter(batch_size, **kwargs))])
def __add__(self, stacked_seqlets):
s = stacked_seqlets
# tasks = self.tasks()
from kipoi.data_utils import numpy_collate_concat
return StackedSeqletImp(
name=self.name,
seq=np.concatenate([self.seq, s.seq]),
contrib=numpy_collate_concat([self.contrib, s.contrib]),
hyp_contrib=numpy_collate_concat([self.hyp_contrib, s.hyp_contrib]),
profile=numpy_collate_concat([self.profile, s.profile]),
dfi=(pd.concat([self.dfi, s.dfi])
if self.dfi is not None and s.dfi is not None
else None),
attrs=self.attrs
)
def load_all_with_metadata(self,
batch_size=64,
num_workers=64,
shuffle=True,
drop_last=False):
it = self.batch_iter(batch_size=batch_size,
num_workers=num_workers,
shuffle=shuffle,
drop_last=drop_last)
return numpy_collate_concat([x for x in tqdm(it)])
def load_all(self, batch_size=32, num_workers=0, **kwargs):
"""Load the whole dataset into memory
Arguments:
batch_size (int, optional): how many samples per batch to load
(default: 1).
num_workers (int, optional): how many subprocesses to use for data
loading. 0 means that the data will be loaded in the main process
(default: 0)
"""
return numpy_collate_concat([x for x in tqdm(self.batch_iter(batch_size,
num_workers=num_workers))])
def load_all(self,
batch_size=64,
num_workers=64,
shuffle=True,
drop_last=False):
it = self.batch_iter(batch_size=batch_size,
num_workers=num_workers,
shuffle=shuffle,
drop_last=drop_last)
dataset = numpy_collate_concat([x for x in tqdm(it)])
return dataset['inputs'], dataset['targets']
def load_all(self, batch_size=32, **kwargs):
"""Load the whole dataset into memory
Arguments:
batch_size (int, optional): how many samples per batch to load
(default: 1).
"""
from copy import deepcopy
return numpy_collate_concat([deepcopy(x)
for x in tqdm(self.batch_iter(batch_size,
**kwargs),
total=len(self) // batch_size)])
def _flush_buffer(self):
"""Write buffer
"""
wb = numpy_collate_concat(self.write_buffer) # merge the buffer
for k in wb:
if sys.version_info[0] == 2 and wb[k].dtype.type in [
np.string_, np.str_, np.unicode_
]:
self.root[k].append(wb[k].astype(unicode))
else:
self.root[k].append(wb[k])
self.write_buffer = None
self.write_buffer_size = 0
def _flush_buffer(self):
"""Write buffer
"""
wb = numpy_collate_concat(self.write_buffer)
for k in wb:
dset = self.f[k]
clen = dset.shape[0]
# resize
dset.resize(clen + self.write_buffer_size, axis=0)
# write
dset[clen:] = wb[k]
self.f.flush()
self.write_buffer = None
self.write_buffer_size = 0
def imp_score(self,
x,
name,
method='deeplift',
batch_size=512,
preact_only=False):
"""Compute the importance score
Args:
x: one-hot encoded DNA sequence
name: which interepretation method to compute
method: which importance score to use. Available: grad, ism, deeplift
"""
# Do we need bias?
if not isinstance(x, dict) and not isinstance(x, list):
seqlen = x.shape[1]
x = {'seq': x, **self.neutral_bias_inputs(len(x), seqlen=seqlen)}
if method == "deeplift":
fn = self._imp_deeplift_fn(x, name, preact_only=preact_only)
else:
raise ValueError(
"Please provide a valid importance scoring method: grad, ism or deeplift"
)
def input_to_list(input_names, x):
if isinstance(x, list):
return x
elif isinstance(x, dict):
return [x[k] for k in input_names]
else:
return [x]
input_names = self.model.input_names
assert input_names[0] == "seq"
if batch_size is None:
return fn(input_to_list(input_names, x))[0]
else:
return numpy_collate_concat([
fn(input_to_list(input_names, batch))[0]
for batch in nested_numpy_minibatch(x, batch_size=batch_size)
])
def concat(cls, objects):
return cls(data=numpy_collate_concat(objects), attrs=None)
def append(self, datax):
"""Append two datasets
"""
return super().__init__(data=numpy_collate_concat([self.data, datax.data]),
attrs=deepcopy(self.attrs))
def imp_score(self,
x,
task,
strand='both',
method='grad',
pred_summary='weighted',
batch_size=512):
"""Compute the importance score
Args:
x: one-hot encoded DNA sequence
method: which importance score to use. Available: grad, ism, deeplift
strand: for which strand to run it ('pos', 'neg' or 'both'). If None, the average of both strands is returned
task_id: id of the task as an int. See `self.tasks` for available tasks
"""
assert task in self.tasks
# figure out the task id
task_id = [i for i, t in enumerate(self.tasks) if t == task][0]
# task_id
if strand == 'both':
# average across strands
return mean([
self.imp_score(x,
task,
strand=strand,
method=method,
pred_summary=pred_summary,
batch_size=batch_size)
for strand in ['pos', 'neg']
])
def input_to_list(input_names, x):
if isinstance(x, list):
return x
elif isinstance(x, dict):
return [x[k] for k in input_names]
else:
return [x]
input_names = self.model.input_names
assert input_names[0] == "seq"
# get the importance scoring function
# if method == "grad":
# fn = self._imp_grad_fn(strand, task_id, pred_summary) #returns fxn
# fn_applied = fn(input_to_list(input_names, x))[0]
# elif method == "ism":
# fn = self._imp_ism_fn(strand, task_id, pred_summary) #returns fxn
# fn_applied = fn(input_to_list(input_names, x))[0]
# elif method == "deeplift":
# fn = self._imp_deeplift_fn(x, strand, task_id, pred_summary) #returns numpy.ndarray
# fn_applied = fn
# else:
# raise ValueError("Please provide a valid importance scoring method: grad, ism or deeplift")
# if batch_size is None:
# return fn_applied
# else:
# return numpy_collate_concat([fn_applied for batch in nested_numpy_minibatch(x, batch_size=batch_size)])
if method == "grad":
fn = self._imp_grad_fn(strand, task_id, pred_summary)
elif method == "ism":
fn = self._imp_ism_fn(strand, task_id, pred_summary)
elif method == "deeplift":
fn = self._imp_deeplift_fn(x, strand, task_id, pred_summary)
else:
raise ValueError(
"Please provide a valid importance scoring method: grad, ism or deeplift"
)
if batch_size is None:
return fn(input_to_list(input_names, x))[0]
else:
return numpy_collate_concat([
fn(input_to_list(input_names, batch))[0]
for batch in nested_numpy_minibatch(x, batch_size=batch_size)
])
def evaluate(self,
metric,
batch_size=256,
num_workers=8,
eval_train=False,
eval_skip=(),
save=True,
**kwargs):
"""Evaluate the model on the validation set
Args:
metrics: a list or a dictionary of metrics
batch_size:
num_workers:
eval_train: if True, also compute the evaluation metrics on the training set
save: save the json file to the output directory
"""
if len(kwargs) > 0:
logger.warning(
f"Extra kwargs were provided to trainer.evaluate: {kwargs}")
# contruct a list of dataset to evaluate
if eval_train:
eval_datasets = [('train', self.train_dataset)
] + self.valid_dataset
else:
eval_datasets = self.valid_dataset
try:
if len(eval_skip) > 0:
eval_datasets = [(k, v) for k, v in eval_datasets
if k not in eval_skip]
except:
logger.warning(
f"eval datasets don't contain tuples. Unable to skip them using {eval_skip}"
)
metric_res = OrderedDict()
for d in eval_datasets:
if len(d) == 2:
dataset_name, dataset = d
eval_metric = metric # use the default eval metric
elif len(d) == 3:
# specialized evaluation metric was passed
dataset_name, dataset, eval_metric = d
else:
# TODO - this should be made more explicit with classes
raise ValueError(
"Valid dataset needs to be a list of tuples of 2 or 3 elements"
"(name, dataset) or (name, dataset, metric)")
logger.info(f"Evaluating dataset: {dataset_name}")
lpreds = []
llabels = []
from copy import deepcopy
for inputs, targets in tqdm(
dataset.batch_train_iter(cycle=False,
num_workers=num_workers,
batch_size=batch_size),
total=len(dataset) // batch_size):
lpreds.append(self.model.predict_on_batch(inputs))
llabels.append(deepcopy(targets))
del inputs
del targets
preds = numpy_collate_concat(lpreds)
labels = numpy_collate_concat(llabels)
del lpreds
del llabels
metric_res[dataset_name] = eval_metric(labels, preds)
if save:
write_json(metric_res, self.evaluation_path, indent=2)
logger.info("Saved metrics to {}".format(self.evaluation_path))
if self.cometml_experiment is not None:
self.cometml_experiment.log_multiple_metrics(flatten(
metric_res, separator='/'),
prefix="eval/")
if self.wandb_run is not None:
self.wandb_run.summary.update(
flatten(prefix_dict(metric_res, prefix="eval/"),
separator='/'))
metric_res = {**self.metrics, **metric_res}
return metric_res
else:
gpu = args.gpu
create_tf_session(gpu)
model = load_model(args.model)
# Get the dataloader
# Dl = kipoi.get_dataloader_factory(args.dataloader, args.dataloader_source)
Dl = kipoi.get_model(args.dataloader,
args.dataloader_source).default_dataloader
dl = Dl(intervals_file=args.intervals_file,
fasta_file=args.fasta_file,
ignore_targets=False,
num_chr_fasta=True)
metric_fns = {"auprc": auprc, "auc": auc, "accuracy": accuracy}
y_true = dl.seq_dl.bed.df[3].values
y_pred = numpy_collate_concat([
model.predict_on_batch(x['inputs']) for x in tqdm(
dl.batch_iter(batch_size=args.batch_size,
num_workers=args.num_workers))
])
# ---------------
metrics = {
k: m(np.ravel(y_true), np.ravel(y_pred))
for k, m in metric_fns.items()
}
output.write_text(json.dumps(metrics))
