def on_metric_call(self, epoch, iter, logs={}):
""" compute metrics on several predictions """
with timer.Timer('predict metrics callback', self.verbose):
# prepare metric
met = np.zeros(
(self.nb_samples, self.nb_labels, len(self.metrics)))
# generate predictions
# the idea is to predict either a full volume or just a slice,
# depending on what we need
gen = _generate_predictions(self.model, self.data_generator,
self.batch_size, self.nb_samples,
self.vol_params)
batch_idx = 0
for (vol_true, vol_pred) in gen:
for idx, metric in enumerate(self.metrics):
met[batch_idx, :, idx] = metric(vol_true, vol_pred)
batch_idx += 1
# write metric to csv file
if self.filepath is not None:
for idx, metric in enumerate(self.metrics):
filen = self.filepath.format(epoch=epoch,
iter=iter,
metric=metric.__name__)
np.savetxt(filen, met[:, :, idx], fmt='%f', delimiter=',')
else:
meanmet = np.nanmean(met, axis=0)
for midx, metric in enumerate(self.metrics):
for idx in range(self.nb_labels):
varname = '%s_label_%d' % (metric.__name__,
self.label_ids[idx])
logs[varname] = meanmet[idx, midx]
def next_pred_label(model, data_generator, verbose=False):
"""
predict the next sample batch from the generator, and compute max labels
return sample, prediction, max_labels
"""
sample = next(data_generator)
with timer.Timer('prediction', verbose):
pred = model.predict(sample[0])
sample_input = sample[0] if not isinstance(sample[0],
(list, tuple)) else sample[0][0]
max_labels = pred_to_label(sample_input, pred)
return (sample, pred) + max_labels
def on_model_save(self, epoch, iter, logs=None):
""" save the model to hdf5. Code mostly from keras core """
with timer.Timer('model save callback', self.verbose):
logs = logs or {}
num_outputs = len(self.model.outputs)
self.epochs_since_last_save += 1
if self.epochs_since_last_save >= self.period:
self.epochs_since_last_save = 0
filepath = self.filepath.format(epoch=epoch, iter=iter, **logs)
if self.save_best_only:
current = logs.get(self.monitor)
if current is None:
warnings.warn(
'Can save best model only with %s available, '
'skipping.' % (self.monitor), RuntimeWarning)
else:
if self.monitor_op(current, self.best):
if self.verbose > 0:
print(
'Epoch %05d: Iter%05d: %s improved from %0.5f to %0.5f,'
' saving model to %s' %
(epoch, iter, self.monitor, self.best,
current, filepath))
self.best = current
if self.save_weights_only:
self.model.layers[-(num_outputs +
1)].save_weights(
filepath,
overwrite=True)
else:
self.model.layers[-(num_outputs + 1)].save(
filepath, overwrite=True)
else:
if self.verbose > 0:
print(
'Epoch %05d Iter%05d: %s did not improve' %
(epoch, iter, self.monitor))
else:
if self.verbose > 0:
print('Epoch %05d: saving model to %s' %
(epoch, filepath))
if self.save_weights_only:
self.model.layers[-(num_outputs + 1)].save_weights(
filepath, overwrite=True)
else:
self.model.layers[-(num_outputs + 1)].save(
filepath, overwrite=True)
def next_vol_pred(model, data_generator, verbose=False):
"""
get the next batch, predict model output
returns (input_vol, y_true, y_pred, <prior>)
"""
# batch to input, output and prediction
sample = next(data_generator)
with timer.Timer('prediction', verbose):
pred = model.predict(sample[0])
data = (sample[0], sample[1], pred)
if isinstance(sample[0], (list, tuple)): # if given prior, might be a list
data = (sample[0][0], sample[1], pred, sample[0][1])
return data
def on_plot_save(self, epoch, iter, logs={}):
# import neuron sandbox
# has to be here, can't be at the top, due to cyclical imports (??)
# TODO: should just pass the function to compute the figures given the model and generator
import neuron.sandbox as nrn_sandbox
reload(nrn_sandbox)
with timer.Timer('plot callback', self.verbose):
if len(self.run.grid_size) == 3:
collapse_2d = [0, 1, 2]
else:
collapse_2d = [2]
exampl = nrn_sandbox.show_example_prediction_result(
self.model,
self.generator,
self.run,
self.data,
test_batch_size=1,
test_model_names=None,
test_grid_size=self.run.grid_size,
ccmap=None,
collapse_2d=collapse_2d,
slice_nr=None,
plt_width=17,
verbose=self.verbose)
# save, then close
figs = exampl[1:]
for idx, fig in enumerate(figs):
dirn = "dirn_%d" % idx
slice_nr = 0
filename = self.savefilepath.format(epoch=epoch,
iter=iter,
axis=dirn,
slice_nr=slice_nr)
fig.savefig(filename)
plt.close()
def predict_volumes(
models,
data_generator,
batch_size,
patch_size,
patch_stride,
grid_size,
nan_func=np.nanmedian,
do_extra_vol=False, # should compute vols beyond label
do_prob_of_true=False, # should compute prob_of_true vols
verbose=False):
"""
Note: we allow models to be a list or a single model.
Normally, if you'd like to run a function over a list for some param,
you can simply loop outside of the function. here, however, we are dealing with a generator,
and want the output of that generator to be consistent for each model.
Returns:
if models isa list of more than one model:
a tuple of model entried, each entry is a tuple of:
true_label, pred_label, <vol>, <prior_label>, <pred_prob_of_true>, <prior_prob_of_true>
if models is just one model:
a tuple of
(true_label, pred_label, <vol>, <prior_label>, <pred_prob_of_true>, <prior_prob_of_true>)
TODO: could add prior
"""
if not isinstance(models, (list, tuple)):
models = (models, )
# get the input and prediction stack
with timer.Timer('predict_volume_stack', verbose):
vol_stack = predict_volume_stack(models, data_generator, batch_size,
grid_size, verbose)
if len(models) == 1:
do_prior = len(vol_stack) == 4
else:
do_prior = len(vol_stack[0]) == 4
# go through models and volumes
ret = ()
for midx, _ in enumerate(models):
stack = vol_stack if len(models) == 1 else vol_stack[midx]
if do_prior:
all_true, all_pred, all_vol, all_prior = stack
else:
all_true, all_pred, all_vol = stack
# get max labels
all_true_label, all_pred_label = pred_to_label(all_true, all_pred)
# quilt volumes and aggregate overlapping patches, if any
args = [patch_size, grid_size, patch_stride]
label_kwargs = {
'nan_func_layers': nan_func,
'nan_func_K': nan_func,
'verbose': verbose
}
vol_true_label = _quilt(all_true_label, *args,
**label_kwargs).astype('int')
vol_pred_label = _quilt(all_pred_label, *args,
**label_kwargs).astype('int')
ret_set = (vol_true_label, vol_pred_label)
if do_extra_vol:
vol_input = _quilt(all_vol, *args)
ret_set += (vol_input, )
if do_prior:
all_prior_label, = pred_to_label(all_prior)
vol_prior_label = _quilt(all_prior_label, *args,
**label_kwargs).astype('int')
ret_set += (vol_prior_label, )
# compute the probability of prediction and prior
# instead of quilting the probabilistic volumes and then computing the probability
# of true label, which takes a long time, we'll first compute the probability of label,
# and then quilt. This is faster, but we'll need to take median votes
if do_extra_vol and do_prob_of_true:
all_pp = prob_of_label(all_pred, all_true_label)
pred_prob_of_true = _quilt(all_pp, *args, **label_kwargs)
ret_set += (pred_prob_of_true, )
if do_prior:
all_pp = prob_of_label(all_prior, all_true_label)
prior_prob_of_true = _quilt(all_pp, *args, **label_kwargs)
ret_set += (prior_prob_of_true, )
ret += (ret_set, )
if len(models) == 1:
ret = ret[0]
# return
return ret