def evaluate(self, X, y): """ Evaluate, i.e., compute metrics on given inputs and targets. Parameters ---------- X: ndarray Input data. y: 1darray Targets. Returns ------- result: dict Dictionary with result metrics. """ X = _ensure_float32(X) stop_criterion = MaxEpochs(0) train_set = SignalAndTarget(X, y) model_constraint = None valid_set = None test_set = None loss_function = self.loss if self.cropped: loss_function = lambda outputs, targets: self.loss( th.mean(outputs, dim=2), targets ) # reset runtime monitor if exists... for monitor in self.monitors: if hasattr(monitor, "last_call_time"): monitor.last_call_time = time.time() exp = Experiment( self.network, train_set, valid_set, test_set, iterator=self.iterator, loss_function=loss_function, optimizer=self.optimizer, model_constraint=model_constraint, monitors=self.monitors, stop_criterion=stop_criterion, remember_best_column=None, run_after_early_stop=False, cuda=self.is_cuda, log_0_epoch=True, do_early_stop=False, ) exp.monitor_epoch({"train": train_set}) result_dict = dict( [ (key.replace("train_", ""), val) for key, val in dict(exp.epochs_df.iloc[0]).items() ] ) return result_dict
def evaluate(self, X, y, batch_size=32): # Create a dummy experiment for the evaluation iterator = BalancedBatchSizeIterator(batch_size=batch_size, seed=0) # seed irrelevant stop_criterion = MaxEpochs(0) train_set = SignalAndTarget(X, y) model_constraint = None valid_set = None test_set = None loss_function = self.loss if self.cropped: loss_function = lambda outputs, targets: \ self.loss(th.mean(outputs, dim=2), targets) exp = Experiment(self.network, train_set, valid_set, test_set, iterator=iterator, loss_function=loss_function, optimizer=self.optimizer, model_constraint=model_constraint, monitors=self.monitors, stop_criterion=stop_criterion, remember_best_column=None, run_after_early_stop=False, cuda=self.cuda, print_0_epoch=False, do_early_stop=False) exp.monitor_epoch({'train': train_set}) result_dict = dict([ (key.replace('train_', ''), val) for key, val in dict(exp.epochs_df.iloc[0]).items() ]) return result_dict
def run_exp(max_recording_mins, n_recordings, sec_to_cut_at_start, sec_to_cut_at_end, duration_recording_mins, max_abs_val, clip_before_resample, sampling_freq, divisor, n_folds, i_test_fold, shuffle, merge_train_valid, model, input_time_length, optimizer, learning_rate, weight_decay, scheduler, model_constraint, batch_size, max_epochs, only_return_exp, time_cut_off_sec, start_time, test_on_eval, test_recording_mins, sensor_types, log_dir, np_th_seed, cuda=True): import torch.backends.cudnn as cudnn cudnn.benchmark = True if optimizer == 'adam': assert merge_train_valid == False else: assert optimizer == 'adamw' assert merge_train_valid == True preproc_functions = create_preproc_functions( sec_to_cut_at_start=sec_to_cut_at_start, sec_to_cut_at_end=sec_to_cut_at_end, duration_recording_mins=duration_recording_mins, max_abs_val=max_abs_val, clip_before_resample=clip_before_resample, sampling_freq=sampling_freq, divisor=divisor) dataset = DiagnosisSet(n_recordings=n_recordings, max_recording_mins=max_recording_mins, preproc_functions=preproc_functions, train_or_eval='train', sensor_types=sensor_types) if test_on_eval: if test_recording_mins is None: test_recording_mins = duration_recording_mins test_preproc_functions = create_preproc_functions( sec_to_cut_at_start=sec_to_cut_at_start, sec_to_cut_at_end=sec_to_cut_at_end, duration_recording_mins=test_recording_mins, max_abs_val=max_abs_val, clip_before_resample=clip_before_resample, sampling_freq=sampling_freq, divisor=divisor) test_dataset = DiagnosisSet(n_recordings=n_recordings, max_recording_mins=None, preproc_functions=test_preproc_functions, train_or_eval='eval', sensor_types=sensor_types) if not only_return_exp: X, y = dataset.load() max_shape = np.max([list(x.shape) for x in X], axis=0) assert max_shape[1] == int(duration_recording_mins * sampling_freq * 60) if test_on_eval: test_X, test_y = test_dataset.load() max_shape = np.max([list(x.shape) for x in test_X], axis=0) assert max_shape[1] == int(test_recording_mins * sampling_freq * 60) if not test_on_eval: splitter = TrainValidTestSplitter(n_folds, i_test_fold, shuffle=shuffle) else: splitter = TrainValidSplitter(n_folds, i_valid_fold=i_test_fold, shuffle=shuffle) if not only_return_exp: if not test_on_eval: train_set, valid_set, test_set = splitter.split(X, y) else: train_set, valid_set = splitter.split(X, y) test_set = SignalAndTarget(test_X, test_y) del test_X, test_y del X, y # shouldn't be necessary, but just to make sure if merge_train_valid: train_set = concatenate_sets([train_set, valid_set]) # just reduce valid for faster computations valid_set.X = valid_set.X[:8] valid_set.y = valid_set.y[:8] # np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/train_X.npy', train_set.X) # np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/train_y.npy', train_set.y) # np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/valid_X.npy', valid_set.X) # np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/valid_y.npy', valid_set.y) # np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/test_X.npy', test_set.X) # np.save('/data/schirrmr/schirrmr/auto-diag/lukasrepr/compare/mne-0-16-2/test_y.npy', test_set.y) else: train_set = None valid_set = None test_set = None log.info("Model:\n{:s}".format(str(model))) if cuda: model.cuda() model.eval() in_chans = 21 # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] log.info("{:d} predictions per input/trial".format(n_preds_per_input)) iterator = CropsFromTrialsIterator(batch_size=batch_size, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input, seed=np_th_seed) assert optimizer in ['adam', 'adamw'], ("Expect optimizer to be either " "adam or adamw") schedule_weight_decay = optimizer == 'adamw' if optimizer == 'adam': optim_class = optim.Adam assert schedule_weight_decay == False assert merge_train_valid == False else: optim_class = AdamW assert schedule_weight_decay == True assert merge_train_valid == True optimizer = optim_class(model.parameters(), lr=learning_rate, weight_decay=weight_decay) if scheduler is not None: assert scheduler == 'cosine' n_updates_per_epoch = sum( [1 for _ in iterator.get_batches(train_set, shuffle=True)]) # Adapt if you have a different number of epochs n_updates_per_period = n_updates_per_epoch * max_epochs scheduler = CosineAnnealing(n_updates_per_period) optimizer = ScheduledOptimizer( scheduler, optimizer, schedule_weight_decay=schedule_weight_decay) loss_function = nll_loss_on_mean if model_constraint is not None: assert model_constraint == 'defaultnorm' model_constraint = MaxNormDefaultConstraint() monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedDiagnosisMonitor(input_time_length, n_preds_per_input), RuntimeMonitor(), ] stop_criterion = MaxEpochs(max_epochs) loggers = [Printer(), TensorboardWriter(log_dir)] batch_modifier = None exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, batch_modifier=batch_modifier, cuda=cuda, loggers=loggers) if not only_return_exp: # Until first stop exp.setup_training() exp.monitor_epoch(exp.datasets) exp.log_epoch() exp.rememberer.remember_epoch(exp.epochs_df, exp.model, exp.optimizer) exp.iterator.reset_rng() while not exp.stop_criterion.should_stop(exp.epochs_df): if (time.time() - start_time) > time_cut_off_sec: log.info( "Ran out of time after {:.2f} sec.".format(time.time() - start_time)) return exp log.info("Still in time after {:.2f} sec.".format(time.time() - start_time)) exp.run_one_epoch(exp.datasets, remember_best=True) if (time.time() - start_time) > time_cut_off_sec: log.info("Ran out of time after {:.2f} sec.".format(time.time() - start_time)) return exp if not merge_train_valid: exp.setup_after_stop_training() # Run until second stop datasets = exp.datasets datasets['train'] = concatenate_sets( [datasets['train'], datasets['valid']]) exp.monitor_epoch(datasets) exp.log_epoch() exp.iterator.reset_rng() while not exp.stop_criterion.should_stop(exp.epochs_df): if (time.time() - start_time) > time_cut_off_sec: log.info("Ran out of time after {:.2f} sec.".format( time.time() - start_time)) return exp log.info("Still in time after {:.2f} sec.".format(time.time() - start_time)) exp.run_one_epoch(datasets, remember_best=False) else: exp.dataset = dataset exp.splitter = splitter if test_on_eval: exp.test_dataset = test_dataset return exp