def test_step(self, batch, batch_nb):
sender_input, receiver_input, labels = batch
test_result = self(sender_input, receiver_input, labels)
result = pl.EvalResult()
result.log('test_loss', test_result['log']['loss'])
result.log('test_acc', test_result['log']['acc'])
argmax_sample = test_result['log']['distr'].argmax(dim=-1)
z_one_hot = \
torch.zeros(
len(argmax_sample),
test_result['log']['distr'].size(-1)).to(argmax_sample.device)
z_one_hot.scatter_(1, argmax_sample.view(-1, 1), 1)
z_one_hot = z_one_hot.view(len(argmax_sample),
test_result['log']['distr'].size(-1))
if not hasattr(self, 'usage'):
self.usage = z_one_hot.sum(dim=0).cpu().numpy()
else:
self.usage += z_one_hot.sum(dim=0).cpu().numpy()
if 'support' in test_result['log'].keys():
result.log('test_support',
test_result['log']['support'],
prog_bar=True)
return result
def validation_step(self, batch, batch_idx):
x = self.get_input(batch, self.image_key)
xrec, qloss = self(x)
aeloss, log_dict_ae = self.loss(qloss,
x,
xrec,
self.global_step,
split="val")
rec_loss = log_dict_ae["val/rec_loss"]
output = pl.EvalResult(checkpoint_on=rec_loss)
output.log("val/rec_loss",
rec_loss,
prog_bar=True,
logger=True,
on_step=True,
on_epoch=True)
output.log("val/aeloss",
aeloss,
prog_bar=True,
logger=True,
on_step=True,
on_epoch=True)
output.log_dict(log_dict_ae)
return output
def test_step(self, batch, batch_idx):
txt, segment, mask, img, y = batch
if self.hparams.model == "bert":
y_hat = self(txt, mask, segment)
elif self.hparams.model == "mmbt":
y_hat = self(txt, mask, segment, img)
else:
raise ValueError(
f'Specified model ({hparams.model}) not implemented')
loss = self.criterion(y_hat, y)
if self.hparams.task_type == "multilabel":
#preds = torch.sigmoid(y_hat).cpu().detach().numpy() > 0.5
preds = (torch.sigmoid(y_hat) > 0.5).float().detach()
else:
preds = torch.nn.functional.softmax(
y_hat, dim=1).argmax(dim=1).cpu().detach().numpy()
if self.hparams.task_type == "multilabel":
macro_f1 = f1_score(preds, y, class_reduction='macro')
micro_f1 = f1_score(preds, y, class_reduction='micro')
result = pl.EvalResult(checkpoint_on=micro_f1)
result.log('test_micro_f1', micro_f1, prog_bar=True, on_epoch=True)
result.log('test_macro_f1', macro_f1, prog_bar=True, on_epoch=True)
else:
acc = accuracy(preds, y)
result.log('test_acc', acc, prog_bar=True, on_epoch=True)
result.log('test_loss', loss, prog_bar=True, on_epoch=True)
return result
def validation_step(self, batch, batch_idx):
batch = from_batch_get_model_input(batch,
self.d_model,
use_pointer=self.use_pointer,
use_coverage=self.use_coverage)
batch[0] = patch_src(batch[0])
batch[6], batch[8] = patch_trg(batch[6])
inputs = {
'encoder_input': batch[0],
'encoder_mask': batch[1],
'encoder_with_oov': batch[2],
'oovs_zero': batch[3],
'context_vec': batch[4],
'coverage': batch[5],
'decoder_input': batch[6],
'decoder_mask': batch[7],
'decoder_target': batch[8],
'mode': 'train'
}
loss = self(**inputs)
#print(loss)
result = pl.EvalResult(checkpoint_on=loss)
result.log('val_loss', loss, prog_bar=True)
return result
def test_step(self, batch, batch_idx):
y_hat = self(batch['input_ids'])
loss = nn.CrossEntropyLoss()(y_hat, batch['label'].flatten())
acc = FM.accuracy(y_hat.detach().argmax(axis=1), batch['label'].flatten(), num_classes=2)
result = pl.EvalResult(checkpoint_on=loss)
result.log_dict({'val_acc': acc, 'val_loss': loss})
return result
def test_step(self, batch: TensorDict, _) -> pl.EvalResult:
loss, info = self.test_loss(batch)
info = self.stat_to_tensor_dict(info)
result = pl.EvalResult()
result.log("test/loss", loss)
result.log_dict({"test/" + k: v for k, v in info.items()})
return result
def validation_step(self, batch, batch_idx):
scans, true_masks = batch
predicted_masks = self(scans)
loss = self.criterion(predicted_masks, true_masks)
result = pl.EvalResult(checkpoint_on=loss)
result.log('val_loss', loss)
return result
def validation_step(self, batch, batch_idx):
images, target = batch
output = self(images)
loss = F.cross_entropy(output, target)
acc1, acc5 = self.__accuracy(output, target, topk=(1, 5))
result = pl.EvalResult(checkpoint_on=loss, early_stop_on=loss)
result.log('val_loss',
loss,
on_step=True,
on_epoch=True,
prog_bar=True,
logger=True,
sync_dist=True)
result.log('val_acc1',
acc1,
on_step=True,
on_epoch=True,
prog_bar=True,
logger=True,
sync_dist=True)
result.log('val_acc5',
acc5,
on_step=True,
on_epoch=True,
prog_bar=True,
logger=True,
sync_dist=True)
return result
def test_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = F.mse_loss(y_hat, y)
result = pl.EvalResult(checkpoint_on=loss)
result.log('test_loss', loss)
return result
def test_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = criterion(y_hat, y)
result = pl.EvalResult()
result.log('test_loss', loss)
return result
def validation_step(self, batch, batch_idx):
loss = self.shared_step(batch, batch_idx)
# checkpoint on the loss value
result = pl.EvalResult(checkpoint_on=loss)
# log the loss for each epoch only, you only want to validate for each epoch
result.log('avg_val_loss', loss, on_epoch=True, on_step=False)
return result
def validation_step(self, batch, batch_idx):
output, val_loss = self.shared_step(batch)
result = pl.EvalResult(checkpoint_on=val_loss)
result.log('val_loss', val_loss)
#Edge filter performance
preds = F.sigmoid(output) > 0.5 #Maybe send to CPU??
edge_positive = preds.sum().float()
if ('pid' in self.hparams["regime"]):
y_pid = batch.pid[batch.edge_index[0]] == batch.pid[
batch.edge_index[1]]
edge_true = y_pid.sum()
edge_true_positive = (y_pid & preds).sum().float()
else:
edge_true = batch.y.sum()
edge_true_positive = (batch.y.bool() & preds).sum().float()
result.log_dict({
'eff': torch.tensor(edge_true_positive / edge_true),
'pur': torch.tensor(edge_true_positive / edge_positive)
})
return result
def validation_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = F.l1_loss(y_hat, y)
result = pl.EvalResult(early_stop_on=loss, checkpoint_on=loss)
result.log("val_loss", loss, sync_dist=True)
return result
def validation_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = F.mse_loss(y_hat, y)
result = pl.EvalResult(checkpoint_on=loss)
result.log('val_loss', torch.sqrt(loss))
return result
def validation_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x).squeeze(1)
loss = F.binary_cross_entropy(y_hat, y.type(torch.cuda.FloatTensor))
result = pl.EvalResult(checkpoint_on=loss)
result.log('val_loss', loss)
return result
def test_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = F.cross_entropy(y_hat, y)
result = pl.EvalResult()
result.log('test_loss', loss)
return result
def validation_step(self, batch: TensorDict, _) -> pl.EvalResult:
loss, info = self.val_loss(batch)
info = self.stat_to_tensor_dict(info)
result = pl.EvalResult(early_stop_on=loss)
result.log("val/loss", loss)
result.log_dict({"val/" + k: v for k, v in info.items()})
return result
def validation_epoch_end(self, outputs):
# OPTIONAL
avg_loss = outputs.valid_batch_loss.mean()
result = pl.EvalResult(checkpoint_on=avg_loss)
result.log('valid_loss', avg_loss, on_epoch=True, prog_bar=True)
return result
def validation_step(self, batch, batch_idx):
u = self(batch)
Phi, = batch
loss = torch.nn.MSELoss()(u, Phi)
result = pl.EvalResult(checkpoint_on=loss)
result.log('val_loss', loss)
return result
def test_epoch_end(self, outputs):
# OPTIONAL
avg_loss = outputs.test_batch_loss.mean()
result = pl.EvalResult()
result.log('test_loss', avg_loss, on_epoch=True)
return result
def test_step(self, batch, batch_idx):
scans, true_masks = batch
predicted_masks = self(scans)
loss = self.criterion(predicted_masks, true_masks)
result = pl.EvalResult()
result.log('test_loss', loss)
return result
def test_epoch_end(self, test_outputs):
mrns = test_outputs.mrn
accounts = test_outputs.account
sent_orders = test_outputs.sent_order
sum_sents = test_outputs.sum_sent_toks
references = test_outputs.reference
output_df = {
'mrn': mrns,
'account': accounts,
'sent_orders': sent_orders,
'prediction': sum_sents,
'reference': references
}
output_df = pd.DataFrame(output_df)
exp_str = 'neusum_{}_{}'.format(self.args.experiment,
str(MAX_GEN_SUM_TOK_CT))
out_fn = os.path.join(out_dir, 'predictions',
'{}_validation.csv'.format(exp_str))
print(
'To evaluate, run: cd ../../evaluations && python rouge.py --experiment {}'
.format(exp_str))
output_df.to_csv(out_fn, index=False)
eval_result = pl.EvalResult()
eval_result.log('rank@1', self.avg(test_outputs.rel_r1), on_epoch=True)
eval_result.log('rank@2', self.avg(test_outputs.rel_r2), on_epoch=True)
eval_result.log('rank@3', self.avg(test_outputs.rel_r3), on_epoch=True)
eval_result.log('rank@4', self.avg(test_outputs.rel_r4), on_epoch=True)
eval_result.log('rank@5+',
self.avg(test_outputs.rel_r5plus),
on_epoch=True)
return eval_result
def test_step(self, batch, batch_idx):
query_tokens, batch_qids, batch_qrels = batch
queries = self.forward(query_tokens, token_type=1)
scores, indices = torch.einsum("ae, be->ba", self.index_tensor,
queries).topk(
self.hparams.test_rank_len,
sorted=True)
rankings = [row.tolist() for row in torch.take(self.pid_map, indices)]
if self.hparams.save_test_rankings:
with open(self.hparams.save_test_rankings, "a") as w:
for qid, ranking in zip(batch_qids, rankings):
for i, pid in enumerate(ranking):
w.write("{}\t{}\t{}\n".format(qid, pid, i + 1))
metrics = calc_ranking_metrics(rankings,
batch_qrels,
count_rankings=False)
metrics = {"test/" + k: v for k, v in metrics.items()}
result = pl.EvalResult()
result.log_dict(
metrics,
on_step=False,
on_epoch=True,
prog_bar=True,
)
return result
def validation_step(self, batch, batch_idx):
"""
Called every batch
Lightning calls this inside the validation loop with the data from the validation dataloader
passed in as `batch`.
"""
batch_imgs, heatmaps_gt, last_heatmap_preds_tch = batch
#batch_imgs, heatmaps_gt = batch
# get prediction
combined_heatmap_preds = self(batch_imgs)
# get prediction from teacher network
#combined_heatmap_preds_tch = self.net_tch(batch_imgs)
#last_heatmap_preds_tch = combined_heatmap_preds_tch[:, self.nstack_tch - 1]
gt_loss = self.calc_loss(combined_heatmap_preds, heatmaps_gt)
tch_loss = self.calc_loss(combined_heatmap_preds, last_heatmap_preds_tch)
# get total loss
val_loss = self.this_config['lambda'] * gt_loss + (1 - self.this_config['lambda']) * tch_loss
#val_result = pl.EvalResult(early_stop_on=val_loss, checkpoint_on=val_loss)
#val_result.log('val_loss', val_loss, on_step=True, on_epoch=True, prog_bar=True)
val_result = pl.EvalResult(early_stop_on=gt_loss, checkpoint_on=gt_loss)
val_result.log('val_loss', gt_loss, on_step=True, on_epoch=True, prog_bar=True)
return val_result
def test_step(self, batch, batch_idx):
var, x_mean = self.calculate_var(batch, batch_idx)
batch_size, n_class, length = var.size()
_, label = batch
targets = (torch.ones(
(batch_size, )).type_as(label) * label)[:,
None].expand(-1, length)
loss = torch.zeros((1, ))
result = pl.EvalResult(checkpoint_on=loss)
for threshold in [0.001, 0.002, 0.005, 0.01]:
confident_idx = torch.where((var < threshold).all(dim=1))
num_confident = len(confident_idx[0])
pred = x_mean.max(dim=1)[1]
tp_confident = (targets == pred)[confident_idx].sum()
acc = tp_confident / float(num_confident)
discarded = torch.Tensor([(batch_size * length) - num_confident])
result.log('val_loss', loss)
result.log(f'test_{threshold}_discarded_prct',
discarded / float(batch_size * length))
result.log(f'test_{threshold}_discarded', discarded)
result.log(f'test_{threshold}_acc', acc)
return result
def validation_step(self, batch, batch_idx):
mixtures, mixture_ids, window_offsets, input_conditions, target_names, targets = batch
estimated_targets = self.separate(
mixtures, input_conditions)[:, self.hop_length:-self.hop_length]
targets = targets[:, self.hop_length:-self.hop_length]
for mixture, mixture_idx, window_offset, input_condition, target_name, estimated_target \
in zip(mixtures, mixture_ids, window_offsets, input_conditions, target_names, estimated_targets):
self.valid_estimation_dict[target_name][mixture_idx.item(
)][window_offset.item()] = estimated_target.detach().cpu().numpy()
# SDR - like Loss
s_targets = (
(targets * estimated_targets).sum(axis=-2, keepdims=True) /
((targets**2).sum(axis=-2, keepdims=True) + 1e-8)) * targets
distortion = estimated_targets - s_targets
loss = (((s_targets**2).sum(-2) + 1e-8).log() -
((distortion**2).sum(-2) + 1e-8).log()).mean()
# large value of SDR means good performance, so that we take the negative of sdr for the validation loss
loss = -1 * loss
result = pl.EvalResult(checkpoint_on=loss, early_stop_on=loss)
result.log('loss/val_loss',
loss,
prog_bar=False,
logger=True,
on_step=False,
on_epoch=True,
reduce_fx=torch.mean)
return result
def validation_step(self, batch, batch_idx):
"""
The validation step, run per batch during training.
Parameters
----------
batch : torch.Tensor
The Data batch.
batch_idx : int
The batch id.
Returns
-------
Result
The output of forward pass.
Notes
-----
Supports Early Stopping on loss by default.
# TODO: make a validation dictionary / callback for more control.
"""
x, y = batch
y_hat = self.model(x)
loss = self.loss(y_hat, y)
metrics = self.metrics(y_hat, y)
result = pl.EvalResult(checkpoint_on=loss, early_stop_on=loss)
result.log_dict({'val_metric': metrics, 'val_loss': loss})
return result
def validation_step(self, batch, batch_idx):
x, y = batch
y_hat = self(x)
loss = F.cross_entropy(y_hat, y)
result = pl.EvalResult(checkpoint_on=loss)
result.log('val_loss', loss)
return result
def validation_epoch_end(self, validation_step_outputs):
gt_classes = validation_step_outputs.gt_classes
inout_mask = gt_classes < 2
gt_inout = gt_classes[inout_mask]
inout_prob = validation_step_outputs.inout_prob[inout_mask]
inout_acc = modules.accuracy(inout_prob, gt_inout, topk=(1, ))[0]
label_mask = gt_classes >= 2
label_prob = validation_step_outputs.label_prob[label_mask]
gt_label = gt_classes[label_mask] - 2
top1, top5 = modules.accuracy(label_prob, gt_label, (1, 5))
cate_topk = modules.accuracy_per_cate(label_prob, gt_label, (1, 5))
result = pl.EvalResult()
result.log_dict(
{
'top1': top1.item(),
'top5': top5.item(),
'inout_acc': inout_acc.item(),
},
prog_bar=True,
on_epoch=True)
print('cate_topk', cate_topk)
print('top1: ', top1.item())
print('top5: ', top5.item())
return result
def test_step(self, batch, batch_idx):
outputs = self(batch)
loss = outputs[0]
y_hat = outputs[1]
acc = FM.accuracy(y_hat.detach().argmax(axis=1), batch['labels'].flatten(), num_classes=2)
result = pl.EvalResult(checkpoint_on=loss)
result.log_dict({'val_acc': acc, 'val_loss': loss})
return result