def val_epoch(self, dataloader, device, step=0, plot=False):
''' Epoch operation in evaluation phase '''
if device == 'cuda':
assert self.CUDA_AVAILABLE
# Set model and classifier training mode
self.model.eval()
self.classifier.eval()
# use evaluator to calculate the average performance
evaluator = Evaluator()
pred_list = []
real_list = []
with torch.no_grad():
for batch in tqdm(
dataloader,
mininterval=5,
desc=' - (Evaluation) ',
leave=False): # training_data should be a iterable
# get data from dataloader
feature_1, feature_2, y = parse_data(batch, device)
batch_size = len(feature_1)
# get logits
logits, attn = self.model(feature_1, feature_2)
logits = logits.view(batch_size, -1)
logits = self.classifier(logits)
if self.d_output == 1:
pred = logits.sigmoid()
loss = mse_loss(pred, y)
else:
pred = logits
loss = cross_entropy_loss(pred, y, smoothing=False)
acc = accuracy(pred, y, threshold=self.threshold)
precision, recall, _, _ = precision_recall(
pred, y, self.d_output, threshold=self.threshold)
# feed the metrics in the evaluator
evaluator(loss.item(), acc.item(), precision[1].item(),
recall[1].item())
'''append the results to the predict / real list for drawing ROC or PR curve.'''
if plot:
pred_list += pred.tolist()
real_list += y.tolist()
if plot:
area, precisions, recalls, thresholds = pr(
pred_list, real_list)
plot_pr_curve(recalls, precisions, auc=area)
# get evaluation results from the evaluator
loss_avg, acc_avg, pre_avg, rec_avg = evaluator.avg_results()
self.eval_logger.info(
'[EVALUATION] - step: %5d, loss: %3.4f, acc: %1.4f, pre: %1.4f, rec: %1.4f'
% (step, loss_avg, acc_avg, pre_avg, rec_avg))
self.summary_writer.add_scalar('loss/eval', loss_avg, step)
self.summary_writer.add_scalar('acc/eval', acc_avg, step)
self.summary_writer.add_scalar('precision/eval', pre_avg, step)
self.summary_writer.add_scalar('recall/eval', rec_avg, step)
state_dict = self.early_stopping(loss_avg)
if state_dict['save']:
checkpoint = self.checkpoint(step)
self.save_model(
checkpoint,
self.save_path + '-step-%d_loss-%.5f' % (step, loss_avg))
return state_dict['break']
d_features=30,
d_meta=6,
d_classifier=256,
d_output=1,
threshold=0.5,
mode='1d',
n_layers=6,
n_head=8,
dropout=0.1,
use_bottleneck=True,
d_bottleneck=256)
# model.resume_checkpoint('models/ckpt-loss-0.0008903071458891007')
# model.train(20, dataloader.train_dataloader(), dataloader.val_dataloader(), device='cuda', save_mode='best', smoothing=False)
model.load_model('models/ckpt-11d-loss-0.00106')
pred, real = model.get_predictions(dataloader.test_dataloader(),
'cuda',
activation=nn.functional.sigmoid)
from utils.plot_curves import precision_recall, plot_pr_curve
area, precisions, recalls, thresholds = precision_recall(pred, real)
plot_pr_curve(recalls, precisions, auc=area)
from utils.plot_curves import auc_roc, plot_roc_curve
auc, fprs, tprs, thresholds = auc_roc(pred, real)
plot_roc_curve(fprs, tprs, auc)