def val_epoch(self, dataloader, device, step=0, plot=False): ''' Epoch operation in evaluation phase ''' if device == 'cuda': assert self.CUDA_AVAILABLE # Set model and classifier training mode self.model.eval() self.classifier.eval() # use evaluator to calculate the average performance evaluator = Evaluator() pred_list = [] real_list = [] with torch.no_grad(): for batch in tqdm( dataloader, mininterval=5, desc=' - (Evaluation) ', leave=False): # training_data should be a iterable # get data from dataloader feature_1, feature_2, y = parse_data(batch, device) batch_size = len(feature_1) # get logits logits, attn = self.model(feature_1, feature_2) logits = logits.view(batch_size, -1) logits = self.classifier(logits) if self.d_output == 1: pred = logits.sigmoid() loss = mse_loss(pred, y) else: pred = logits loss = cross_entropy_loss(pred, y, smoothing=False) acc = accuracy(pred, y, threshold=self.threshold) precision, recall, _, _ = precision_recall( pred, y, self.d_output, threshold=self.threshold) # feed the metrics in the evaluator evaluator(loss.item(), acc.item(), precision[1].item(), recall[1].item()) '''append the results to the predict / real list for drawing ROC or PR curve.''' if plot: pred_list += pred.tolist() real_list += y.tolist() if plot: area, precisions, recalls, thresholds = pr( pred_list, real_list) plot_pr_curve(recalls, precisions, auc=area) # get evaluation results from the evaluator loss_avg, acc_avg, pre_avg, rec_avg = evaluator.avg_results() self.eval_logger.info( '[EVALUATION] - step: %5d, loss: %3.4f, acc: %1.4f, pre: %1.4f, rec: %1.4f' % (step, loss_avg, acc_avg, pre_avg, rec_avg)) self.summary_writer.add_scalar('loss/eval', loss_avg, step) self.summary_writer.add_scalar('acc/eval', acc_avg, step) self.summary_writer.add_scalar('precision/eval', pre_avg, step) self.summary_writer.add_scalar('recall/eval', rec_avg, step) state_dict = self.early_stopping(loss_avg) if state_dict['save']: checkpoint = self.checkpoint(step) self.save_model( checkpoint, self.save_path + '-step-%d_loss-%.5f' % (step, loss_avg)) return state_dict['break']
d_features=30, d_meta=6, d_classifier=256, d_output=1, threshold=0.5, mode='1d', n_layers=6, n_head=8, dropout=0.1, use_bottleneck=True, d_bottleneck=256) # model.resume_checkpoint('models/ckpt-loss-0.0008903071458891007') # model.train(20, dataloader.train_dataloader(), dataloader.val_dataloader(), device='cuda', save_mode='best', smoothing=False) model.load_model('models/ckpt-11d-loss-0.00106') pred, real = model.get_predictions(dataloader.test_dataloader(), 'cuda', activation=nn.functional.sigmoid) from utils.plot_curves import precision_recall, plot_pr_curve area, precisions, recalls, thresholds = precision_recall(pred, real) plot_pr_curve(recalls, precisions, auc=area) from utils.plot_curves import auc_roc, plot_roc_curve auc, fprs, tprs, thresholds = auc_roc(pred, real) plot_roc_curve(fprs, tprs, auc)