def run(self): epochs_tested = self.get_epochs_to_test() folds_to_test = self.get_folds_to_test() for fold in folds_to_test: for epoch in epochs_tested[fold]: pretrained_path = self.args.pretrained_path or \ os.path.join(self.args.checkpoint_dir, get_chk_name(self.args.exp_name, fold, epoch)) for t in ["", "Intra_"]: if self.args.outfile_name is None: outfile = "%s%s" % ("Test_" + t, self.args.exp_name) else: outfile = "%s%s" % (t, self.args.outfile_name) exp_name = outfile + "_fold{}_epoch{}".format(fold, epoch) model = Base(model=self.net, loss=self.loss, metrics=self.args.metrics, pretrained=pretrained_path, use_cuda=self.args.cuda) model.testing( self.manager.get_dataloader(test=(t != "Intra_"), test_intra=(t == "Intra_"), fold_index=fold), with_visuals=False, with_logit=self.args.with_logit, predict=self.args.predict, saving_dir=self.args.checkpoint_dir, exp_name=exp_name, standard_optim=getattr(self.net, 'std_optim', True), **self.kwargs_test)
def run(self, nb_rep=10): if self.args.pretrained_path is not None: raise ValueError( 'Unset <pretrained_path> to use the EnsemblingTester') epochs_tested = self.get_epochs_to_test() folds_to_test = self.get_folds_to_test() for fold in folds_to_test: for epoch in epochs_tested[fold]: Y, Y_true = [], [] for i in range(nb_rep): pretrained_path = os.path.join( self.args.checkpoint_dir, get_chk_name( self.args.exp_name + '_ensemble_%i' % (i + 1), fold, epoch)) outfile = self.args.outfile_name or ("EnsembleTest_" + self.args.exp_name) exp_name = outfile + "_fold{}_epoch{}.pkl".format( fold, epoch) model = Base(model=self.net, loss=self.loss, metrics=self.args.metrics, pretrained=pretrained_path, use_cuda=self.args.cuda) y, y_true, _, _, _ = model.test( self.manager.get_dataloader(test=True).test) Y.append(y) Y_true.append(y_true) with open(os.path.join(self.args.checkpoint_dir, exp_name), 'wb') as f: pickle.dump( { "y": np.array(Y).swapaxes(0, 1), "y_true": np.array(Y_true).swapaxes(0, 1) }, f)
def run(self, MC=10): epochs_tested = self.get_epochs_to_test() folds_to_test = self.get_folds_to_test() if self.args.cv: self.logger.warning( "CROSS-VALIDATION USED DURING TESTING, EVENTUAL TESTING SET IS OMIT" ) for fold in folds_to_test: for epoch in epochs_tested[fold]: pretrained_path = self.args.pretrained_path or \ os.path.join(self.args.checkpoint_dir, get_chk_name(self.args.exp_name, fold, epoch)) outfile = self.args.outfile_name or ("MCTest_" + self.args.exp_name) exp_name = outfile + "_fold{}_epoch{}.pkl".format(fold, epoch) model = Base(model=self.net, loss=self.loss, metrics=self.args.metrics, pretrained=pretrained_path, use_cuda=self.args.cuda) if self.args.cv: y, y_true = model.MC_test(self.manager.get_dataloader( validation=True, fold_index=fold).validation, MC=MC) else: y, y_true = model.MC_test( self.manager.get_dataloader(test=True).test, MC=MC) with open(os.path.join(self.args.checkpoint_dir, exp_name), 'wb') as f: pickle.dump({"y": y, "y_true": y_true}, f)
def run(self): epochs_tested = self.get_epochs_to_test() folds_to_test = self.get_folds_to_test() for fold in folds_to_test: for epoch in epochs_tested[fold]: pretrained_path = self.args.pretrained_path or \ os.path.join(self.args.checkpoint_dir, get_chk_name(self.args.exp_name, fold, epoch)) outfile = self.args.outfile_name or ("Test_CV_" + self.args.exp_name) exp_name = outfile + "_fold{}_epoch{}".format(fold, epoch) model = Base(model=self.net, loss=self.loss, metrics=self.args.metrics, pretrained=pretrained_path, use_cuda=self.args.cuda) loader = self.manager.get_dataloader(validation=True, fold_index=fold) res = model.test(loader.validation, with_visuals=False, with_logit=self.args.with_logit, predict=self.args.predict, standard_optim=getattr( self.net, 'std_optim', True)) with open( os.path.join(self.args.checkpoint_dir, exp_name + '.pkl'), 'wb') as f: pickle.dump( { 'y_pred': res[0], 'y_true': res[1], 'loss': res[3], 'metrics': res[4] }, f)
def run(self): epochs_tested = [[self.args.nb_epochs - 1] for _ in range(self.manager.number_of_folds)] folds_to_test = self.get_folds_to_test() std_noise = [0, 0.05, 0.1, 0.15, 0.20] nb_repetitions = 5 # nb of repetitions per Gaussian Noise results = {std: [] for std in std_noise} for sigma in std_noise: self.manager = BaseTrainer.build_data_manager( self.args, input_transforms=[ Crop((1, 121, 128, 121)), Padding([1, 128, 128, 128], mode='constant'), Normalize(), GaussianNoise(sigma) ]) for _ in range(nb_repetitions): for fold in folds_to_test: for epoch in epochs_tested[fold]: pretrained_path = self.args.pretrained_path or \ os.path.join(self.args.checkpoint_dir, get_chk_name(self.args.exp_name, fold, epoch)) outfile = self.args.outfile_name or ( "Test_" + self.args.exp_name) exp_name = outfile + "_fold{}_epoch{}".format( fold, epoch) model = Base(model=self.net, loss=self.loss, metrics=self.args.metrics, pretrained=pretrained_path, use_cuda=self.args.cuda) y, X, y_true, l, metrics = model.testing( self.manager.get_dataloader(test=True), with_visuals=False, with_logit=self.args.with_logit, predict=self.args.predict, saving_dir=None, exp_name=exp_name, standard_optim=getattr(self.net, 'std_optim', True)) results[sigma].append([y, y_true]) with open( os.path.join(self.args.checkpoint_dir, 'Robustness_' + self.args.exp_name + '.pkl'), 'wb') as f: pickle.dump(results, f)
num_workers=5) # Train only linear layer at the end of encoder nets[mod] = MultiModal_MNIST_SVHN([mod], latent_dim=32, mode="linear_classifier") # Freeze all the parameters except the last layer freeze_until(nets[mod], "%s_encoder.head" % mod) if training: for pth in models_pth: optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, nets[mod].parameters()), lr=1e-3, weight_decay=1e-5) models[mod][pth] = Base(model=nets[mod], pretrained=pth, metrics=["balanced_accuracy"], use_cuda=True, loss=nn.CrossEntropyLoss(), optimizer=optimizer) models[mod][pth].training(managers[mod], nb_epochs, checkpointdir=os.path.dirname(pth), exp_name="Linear_%s_Pretrained" % mod.upper(), nb_epochs_per_saving=nb_epochs) _, _, _, _, metrics = models[mod][pth].testing( managers[mod].get_dataloader(test=True)) print("{} with {} --> {}".format(mod, pth, metrics), flush=True) ## 2) GradCAM on Pre-trained networks for mod in modalities: if mod == "mnist":