def evaluate(self, X, y):
"""
Evaluate, i.e., compute metrics on given inputs and targets.
Parameters
----------
X: ndarray
Input data.
y: 1darray
Targets.
Returns
-------
result: dict
Dictionary with result metrics.
"""
X = _ensure_float32(X)
stop_criterion = MaxEpochs(0)
train_set = SignalAndTarget(X, y)
model_constraint = None
valid_set = None
test_set = None
loss_function = self.loss
if self.cropped:
loss_function = lambda outputs, targets: self.loss(
th.mean(outputs, dim=2), targets
)
# reset runtime monitor if exists...
for monitor in self.monitors:
if hasattr(monitor, "last_call_time"):
monitor.last_call_time = time.time()
exp = Experiment(
self.network,
train_set,
valid_set,
test_set,
iterator=self.iterator,
loss_function=loss_function,
optimizer=self.optimizer,
model_constraint=model_constraint,
monitors=self.monitors,
stop_criterion=stop_criterion,
remember_best_column=None,
run_after_early_stop=False,
cuda=self.is_cuda,
log_0_epoch=True,
do_early_stop=False,
)
exp.monitor_epoch({"train": train_set})
result_dict = dict(
[
(key.replace("train_", ""), val)
for key, val in dict(exp.epochs_df.iloc[0]).items()
]
)
return result_dict
def evaluate(self, X, y, batch_size=32):
# Create a dummy experiment for the evaluation
iterator = BalancedBatchSizeIterator(batch_size=batch_size,
seed=0) # seed irrelevant
stop_criterion = MaxEpochs(0)
train_set = SignalAndTarget(X, y)
model_constraint = None
valid_set = None
test_set = None
loss_function = self.loss
if self.cropped:
loss_function = lambda outputs, targets: \
self.loss(th.mean(outputs, dim=2), targets)
exp = Experiment(self.network,
train_set,
valid_set,
test_set,
iterator=iterator,
loss_function=loss_function,
optimizer=self.optimizer,
model_constraint=model_constraint,
monitors=self.monitors,
stop_criterion=stop_criterion,
remember_best_column=None,
run_after_early_stop=False,
cuda=self.cuda,
print_0_epoch=False,
do_early_stop=False)
exp.monitor_epoch({'train': train_set})
result_dict = dict([
(key.replace('train_', ''), val)
for key, val in dict(exp.epochs_df.iloc[0]).items()
])
return result_dict
def run_exp(max_recording_mins,
n_recordings,
sec_to_cut_at_start,
sec_to_cut_at_end,
duration_recording_mins,
max_abs_val,
clip_before_resample,
sampling_freq,
divisor,
n_folds,
i_test_fold,
shuffle,
merge_train_valid,
model,
input_time_length,
optimizer,
learning_rate,
weight_decay,
scheduler,
model_constraint,
batch_size,
max_epochs,
only_return_exp,
time_cut_off_sec,
start_time,
test_on_eval,
test_recording_mins,
sensor_types,
log_dir,
np_th_seed,
cuda=True):
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
if optimizer == 'adam':
assert merge_train_valid == False
else:
assert optimizer == 'adamw'
assert merge_train_valid == True
preproc_functions = create_preproc_functions(
sec_to_cut_at_start=sec_to_cut_at_start,
sec_to_cut_at_end=sec_to_cut_at_end,
duration_recording_mins=duration_recording_mins,
max_abs_val=max_abs_val,
clip_before_resample=clip_before_resample,
sampling_freq=sampling_freq,
divisor=divisor)
dataset = DiagnosisSet(n_recordings=n_recordings,
max_recording_mins=max_recording_mins,
preproc_functions=preproc_functions,
train_or_eval='train',
sensor_types=sensor_types)
if test_on_eval:
if test_recording_mins is None:
test_recording_mins = duration_recording_mins
test_preproc_functions = create_preproc_functions(
sec_to_cut_at_start=sec_to_cut_at_start,
sec_to_cut_at_end=sec_to_cut_at_end,
duration_recording_mins=test_recording_mins,
max_abs_val=max_abs_val,
clip_before_resample=clip_before_resample,
sampling_freq=sampling_freq,
divisor=divisor)
test_dataset = DiagnosisSet(n_recordings=n_recordings,
max_recording_mins=None,
preproc_functions=test_preproc_functions,
train_or_eval='eval',
sensor_types=sensor_types)
if not only_return_exp:
X, y = dataset.load()
max_shape = np.max([list(x.shape) for x in X], axis=0)
assert max_shape[1] == int(duration_recording_mins * sampling_freq *
60)
if test_on_eval:
test_X, test_y = test_dataset.load()
max_shape = np.max([list(x.shape) for x in test_X], axis=0)
assert max_shape[1] == int(test_recording_mins * sampling_freq *
60)
if not test_on_eval:
splitter = TrainValidTestSplitter(n_folds,
i_test_fold,
shuffle=shuffle)
else:
splitter = TrainValidSplitter(n_folds,
i_valid_fold=i_test_fold,
shuffle=shuffle)
if not only_return_exp:
if not test_on_eval:
train_set, valid_set, test_set = splitter.split(X, y)
else:
train_set, valid_set = splitter.split(X, y)
test_set = SignalAndTarget(test_X, test_y)
del test_X, test_y
del X, y # shouldn't be necessary, but just to make sure
if merge_train_valid:
train_set = concatenate_sets([train_set, valid_set])
# just reduce valid for faster computations
valid_set.X = valid_set.X[:8]
valid_set.y = valid_set.y[:8]
# np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/train_X.npy', train_set.X)
# np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/train_y.npy', train_set.y)
# np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/valid_X.npy', valid_set.X)
# np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/valid_y.npy', valid_set.y)
# np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/test_X.npy', test_set.X)
# np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/test_y.npy', test_set.y)
else:
train_set = None
valid_set = None
test_set = None
log.info("Model:\n{:s}".format(str(model)))
if cuda:
model.cuda()
model.eval()
in_chans = 21
# determine output size
test_input = np_to_var(
np.ones((2, in_chans, input_time_length, 1), dtype=np.float32))
if cuda:
test_input = test_input.cuda()
out = model(test_input)
n_preds_per_input = out.cpu().data.numpy().shape[2]
log.info("{:d} predictions per input/trial".format(n_preds_per_input))
iterator = CropsFromTrialsIterator(batch_size=batch_size,
input_time_length=input_time_length,
n_preds_per_input=n_preds_per_input,
seed=np_th_seed)
assert optimizer in ['adam', 'adamw'], ("Expect optimizer to be either "
"adam or adamw")
schedule_weight_decay = optimizer == 'adamw'
if optimizer == 'adam':
optim_class = optim.Adam
assert schedule_weight_decay == False
assert merge_train_valid == False
else:
optim_class = AdamW
assert schedule_weight_decay == True
assert merge_train_valid == True
optimizer = optim_class(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
if scheduler is not None:
assert scheduler == 'cosine'
n_updates_per_epoch = sum(
[1 for _ in iterator.get_batches(train_set, shuffle=True)])
# Adapt if you have a different number of epochs
n_updates_per_period = n_updates_per_epoch * max_epochs
scheduler = CosineAnnealing(n_updates_per_period)
optimizer = ScheduledOptimizer(
scheduler, optimizer, schedule_weight_decay=schedule_weight_decay)
loss_function = nll_loss_on_mean
if model_constraint is not None:
assert model_constraint == 'defaultnorm'
model_constraint = MaxNormDefaultConstraint()
monitors = [
LossMonitor(),
MisclassMonitor(col_suffix='sample_misclass'),
CroppedDiagnosisMonitor(input_time_length, n_preds_per_input),
RuntimeMonitor(),
]
stop_criterion = MaxEpochs(max_epochs)
loggers = [Printer(), TensorboardWriter(log_dir)]
batch_modifier = None
exp = Experiment(model,
train_set,
valid_set,
test_set,
iterator,
loss_function,
optimizer,
model_constraint,
monitors,
stop_criterion,
remember_best_column='valid_misclass',
run_after_early_stop=True,
batch_modifier=batch_modifier,
cuda=cuda,
loggers=loggers)
if not only_return_exp:
# Until first stop
exp.setup_training()
exp.monitor_epoch(exp.datasets)
exp.log_epoch()
exp.rememberer.remember_epoch(exp.epochs_df, exp.model, exp.optimizer)
exp.iterator.reset_rng()
while not exp.stop_criterion.should_stop(exp.epochs_df):
if (time.time() - start_time) > time_cut_off_sec:
log.info(
"Ran out of time after {:.2f} sec.".format(time.time() -
start_time))
return exp
log.info("Still in time after {:.2f} sec.".format(time.time() -
start_time))
exp.run_one_epoch(exp.datasets, remember_best=True)
if (time.time() - start_time) > time_cut_off_sec:
log.info("Ran out of time after {:.2f} sec.".format(time.time() -
start_time))
return exp
if not merge_train_valid:
exp.setup_after_stop_training()
# Run until second stop
datasets = exp.datasets
datasets['train'] = concatenate_sets(
[datasets['train'], datasets['valid']])
exp.monitor_epoch(datasets)
exp.log_epoch()
exp.iterator.reset_rng()
while not exp.stop_criterion.should_stop(exp.epochs_df):
if (time.time() - start_time) > time_cut_off_sec:
log.info("Ran out of time after {:.2f} sec.".format(
time.time() - start_time))
return exp
log.info("Still in time after {:.2f} sec.".format(time.time() -
start_time))
exp.run_one_epoch(datasets, remember_best=False)
else:
exp.dataset = dataset
exp.splitter = splitter
if test_on_eval:
exp.test_dataset = test_dataset
return exp
