def test_step(self, batch, batch_nb): sender_input, receiver_input, labels = batch test_result = self(sender_input, receiver_input, labels) result = pl.EvalResult() result.log('test_loss', test_result['log']['loss']) result.log('test_acc', test_result['log']['acc']) argmax_sample = test_result['log']['distr'].argmax(dim=-1) z_one_hot = \ torch.zeros( len(argmax_sample), test_result['log']['distr'].size(-1)).to(argmax_sample.device) z_one_hot.scatter_(1, argmax_sample.view(-1, 1), 1) z_one_hot = z_one_hot.view(len(argmax_sample), test_result['log']['distr'].size(-1)) if not hasattr(self, 'usage'): self.usage = z_one_hot.sum(dim=0).cpu().numpy() else: self.usage += z_one_hot.sum(dim=0).cpu().numpy() if 'support' in test_result['log'].keys(): result.log('test_support', test_result['log']['support'], prog_bar=True) return result
def validation_step(self, batch, batch_idx): x = self.get_input(batch, self.image_key) xrec, qloss = self(x) aeloss, log_dict_ae = self.loss(qloss, x, xrec, self.global_step, split="val") rec_loss = log_dict_ae["val/rec_loss"] output = pl.EvalResult(checkpoint_on=rec_loss) output.log("val/rec_loss", rec_loss, prog_bar=True, logger=True, on_step=True, on_epoch=True) output.log("val/aeloss", aeloss, prog_bar=True, logger=True, on_step=True, on_epoch=True) output.log_dict(log_dict_ae) return output
def test_step(self, batch, batch_idx): txt, segment, mask, img, y = batch if self.hparams.model == "bert": y_hat = self(txt, mask, segment) elif self.hparams.model == "mmbt": y_hat = self(txt, mask, segment, img) else: raise ValueError( f'Specified model ({hparams.model}) not implemented') loss = self.criterion(y_hat, y) if self.hparams.task_type == "multilabel": #preds = torch.sigmoid(y_hat).cpu().detach().numpy() > 0.5 preds = (torch.sigmoid(y_hat) > 0.5).float().detach() else: preds = torch.nn.functional.softmax( y_hat, dim=1).argmax(dim=1).cpu().detach().numpy() if self.hparams.task_type == "multilabel": macro_f1 = f1_score(preds, y, class_reduction='macro') micro_f1 = f1_score(preds, y, class_reduction='micro') result = pl.EvalResult(checkpoint_on=micro_f1) result.log('test_micro_f1', micro_f1, prog_bar=True, on_epoch=True) result.log('test_macro_f1', macro_f1, prog_bar=True, on_epoch=True) else: acc = accuracy(preds, y) result.log('test_acc', acc, prog_bar=True, on_epoch=True) result.log('test_loss', loss, prog_bar=True, on_epoch=True) return result
def validation_step(self, batch, batch_idx): batch = from_batch_get_model_input(batch, self.d_model, use_pointer=self.use_pointer, use_coverage=self.use_coverage) batch[0] = patch_src(batch[0]) batch[6], batch[8] = patch_trg(batch[6]) inputs = { 'encoder_input': batch[0], 'encoder_mask': batch[1], 'encoder_with_oov': batch[2], 'oovs_zero': batch[3], 'context_vec': batch[4], 'coverage': batch[5], 'decoder_input': batch[6], 'decoder_mask': batch[7], 'decoder_target': batch[8], 'mode': 'train' } loss = self(**inputs) #print(loss) result = pl.EvalResult(checkpoint_on=loss) result.log('val_loss', loss, prog_bar=True) return result
def test_step(self, batch, batch_idx): y_hat = self(batch['input_ids']) loss = nn.CrossEntropyLoss()(y_hat, batch['label'].flatten()) acc = FM.accuracy(y_hat.detach().argmax(axis=1), batch['label'].flatten(), num_classes=2) result = pl.EvalResult(checkpoint_on=loss) result.log_dict({'val_acc': acc, 'val_loss': loss}) return result
def test_step(self, batch: TensorDict, _) -> pl.EvalResult: loss, info = self.test_loss(batch) info = self.stat_to_tensor_dict(info) result = pl.EvalResult() result.log("test/loss", loss) result.log_dict({"test/" + k: v for k, v in info.items()}) return result
def validation_step(self, batch, batch_idx): scans, true_masks = batch predicted_masks = self(scans) loss = self.criterion(predicted_masks, true_masks) result = pl.EvalResult(checkpoint_on=loss) result.log('val_loss', loss) return result
def validation_step(self, batch, batch_idx): images, target = batch output = self(images) loss = F.cross_entropy(output, target) acc1, acc5 = self.__accuracy(output, target, topk=(1, 5)) result = pl.EvalResult(checkpoint_on=loss, early_stop_on=loss) result.log('val_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True, sync_dist=True) result.log('val_acc1', acc1, on_step=True, on_epoch=True, prog_bar=True, logger=True, sync_dist=True) result.log('val_acc5', acc5, on_step=True, on_epoch=True, prog_bar=True, logger=True, sync_dist=True) return result
def test_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = F.mse_loss(y_hat, y) result = pl.EvalResult(checkpoint_on=loss) result.log('test_loss', loss) return result
def test_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = criterion(y_hat, y) result = pl.EvalResult() result.log('test_loss', loss) return result
def validation_step(self, batch, batch_idx): loss = self.shared_step(batch, batch_idx) # checkpoint on the loss value result = pl.EvalResult(checkpoint_on=loss) # log the loss for each epoch only, you only want to validate for each epoch result.log('avg_val_loss', loss, on_epoch=True, on_step=False) return result
def validation_step(self, batch, batch_idx): output, val_loss = self.shared_step(batch) result = pl.EvalResult(checkpoint_on=val_loss) result.log('val_loss', val_loss) #Edge filter performance preds = F.sigmoid(output) > 0.5 #Maybe send to CPU?? edge_positive = preds.sum().float() if ('pid' in self.hparams["regime"]): y_pid = batch.pid[batch.edge_index[0]] == batch.pid[ batch.edge_index[1]] edge_true = y_pid.sum() edge_true_positive = (y_pid & preds).sum().float() else: edge_true = batch.y.sum() edge_true_positive = (batch.y.bool() & preds).sum().float() result.log_dict({ 'eff': torch.tensor(edge_true_positive / edge_true), 'pur': torch.tensor(edge_true_positive / edge_positive) }) return result
def validation_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = F.l1_loss(y_hat, y) result = pl.EvalResult(early_stop_on=loss, checkpoint_on=loss) result.log("val_loss", loss, sync_dist=True) return result
def validation_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = F.mse_loss(y_hat, y) result = pl.EvalResult(checkpoint_on=loss) result.log('val_loss', torch.sqrt(loss)) return result
def validation_step(self, batch, batch_idx): x, y = batch y_hat = self(x).squeeze(1) loss = F.binary_cross_entropy(y_hat, y.type(torch.cuda.FloatTensor)) result = pl.EvalResult(checkpoint_on=loss) result.log('val_loss', loss) return result
def test_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = F.cross_entropy(y_hat, y) result = pl.EvalResult() result.log('test_loss', loss) return result
def validation_step(self, batch: TensorDict, _) -> pl.EvalResult: loss, info = self.val_loss(batch) info = self.stat_to_tensor_dict(info) result = pl.EvalResult(early_stop_on=loss) result.log("val/loss", loss) result.log_dict({"val/" + k: v for k, v in info.items()}) return result
def validation_epoch_end(self, outputs): # OPTIONAL avg_loss = outputs.valid_batch_loss.mean() result = pl.EvalResult(checkpoint_on=avg_loss) result.log('valid_loss', avg_loss, on_epoch=True, prog_bar=True) return result
def validation_step(self, batch, batch_idx): u = self(batch) Phi, = batch loss = torch.nn.MSELoss()(u, Phi) result = pl.EvalResult(checkpoint_on=loss) result.log('val_loss', loss) return result
def test_epoch_end(self, outputs): # OPTIONAL avg_loss = outputs.test_batch_loss.mean() result = pl.EvalResult() result.log('test_loss', avg_loss, on_epoch=True) return result
def test_step(self, batch, batch_idx): scans, true_masks = batch predicted_masks = self(scans) loss = self.criterion(predicted_masks, true_masks) result = pl.EvalResult() result.log('test_loss', loss) return result
def test_epoch_end(self, test_outputs): mrns = test_outputs.mrn accounts = test_outputs.account sent_orders = test_outputs.sent_order sum_sents = test_outputs.sum_sent_toks references = test_outputs.reference output_df = { 'mrn': mrns, 'account': accounts, 'sent_orders': sent_orders, 'prediction': sum_sents, 'reference': references } output_df = pd.DataFrame(output_df) exp_str = 'neusum_{}_{}'.format(self.args.experiment, str(MAX_GEN_SUM_TOK_CT)) out_fn = os.path.join(out_dir, 'predictions', '{}_validation.csv'.format(exp_str)) print( 'To evaluate, run: cd ../../evaluations && python rouge.py --experiment {}' .format(exp_str)) output_df.to_csv(out_fn, index=False) eval_result = pl.EvalResult() eval_result.log('rank@1', self.avg(test_outputs.rel_r1), on_epoch=True) eval_result.log('rank@2', self.avg(test_outputs.rel_r2), on_epoch=True) eval_result.log('rank@3', self.avg(test_outputs.rel_r3), on_epoch=True) eval_result.log('rank@4', self.avg(test_outputs.rel_r4), on_epoch=True) eval_result.log('rank@5+', self.avg(test_outputs.rel_r5plus), on_epoch=True) return eval_result
def test_step(self, batch, batch_idx): query_tokens, batch_qids, batch_qrels = batch queries = self.forward(query_tokens, token_type=1) scores, indices = torch.einsum("ae, be->ba", self.index_tensor, queries).topk( self.hparams.test_rank_len, sorted=True) rankings = [row.tolist() for row in torch.take(self.pid_map, indices)] if self.hparams.save_test_rankings: with open(self.hparams.save_test_rankings, "a") as w: for qid, ranking in zip(batch_qids, rankings): for i, pid in enumerate(ranking): w.write("{}\t{}\t{}\n".format(qid, pid, i + 1)) metrics = calc_ranking_metrics(rankings, batch_qrels, count_rankings=False) metrics = {"test/" + k: v for k, v in metrics.items()} result = pl.EvalResult() result.log_dict( metrics, on_step=False, on_epoch=True, prog_bar=True, ) return result
def validation_step(self, batch, batch_idx): """ Called every batch Lightning calls this inside the validation loop with the data from the validation dataloader passed in as `batch`. """ batch_imgs, heatmaps_gt, last_heatmap_preds_tch = batch #batch_imgs, heatmaps_gt = batch # get prediction combined_heatmap_preds = self(batch_imgs) # get prediction from teacher network #combined_heatmap_preds_tch = self.net_tch(batch_imgs) #last_heatmap_preds_tch = combined_heatmap_preds_tch[:, self.nstack_tch - 1] gt_loss = self.calc_loss(combined_heatmap_preds, heatmaps_gt) tch_loss = self.calc_loss(combined_heatmap_preds, last_heatmap_preds_tch) # get total loss val_loss = self.this_config['lambda'] * gt_loss + (1 - self.this_config['lambda']) * tch_loss #val_result = pl.EvalResult(early_stop_on=val_loss, checkpoint_on=val_loss) #val_result.log('val_loss', val_loss, on_step=True, on_epoch=True, prog_bar=True) val_result = pl.EvalResult(early_stop_on=gt_loss, checkpoint_on=gt_loss) val_result.log('val_loss', gt_loss, on_step=True, on_epoch=True, prog_bar=True) return val_result
def test_step(self, batch, batch_idx): var, x_mean = self.calculate_var(batch, batch_idx) batch_size, n_class, length = var.size() _, label = batch targets = (torch.ones( (batch_size, )).type_as(label) * label)[:, None].expand(-1, length) loss = torch.zeros((1, )) result = pl.EvalResult(checkpoint_on=loss) for threshold in [0.001, 0.002, 0.005, 0.01]: confident_idx = torch.where((var < threshold).all(dim=1)) num_confident = len(confident_idx[0]) pred = x_mean.max(dim=1)[1] tp_confident = (targets == pred)[confident_idx].sum() acc = tp_confident / float(num_confident) discarded = torch.Tensor([(batch_size * length) - num_confident]) result.log('val_loss', loss) result.log(f'test_{threshold}_discarded_prct', discarded / float(batch_size * length)) result.log(f'test_{threshold}_discarded', discarded) result.log(f'test_{threshold}_acc', acc) return result
def validation_step(self, batch, batch_idx): mixtures, mixture_ids, window_offsets, input_conditions, target_names, targets = batch estimated_targets = self.separate( mixtures, input_conditions)[:, self.hop_length:-self.hop_length] targets = targets[:, self.hop_length:-self.hop_length] for mixture, mixture_idx, window_offset, input_condition, target_name, estimated_target \ in zip(mixtures, mixture_ids, window_offsets, input_conditions, target_names, estimated_targets): self.valid_estimation_dict[target_name][mixture_idx.item( )][window_offset.item()] = estimated_target.detach().cpu().numpy() # SDR - like Loss s_targets = ( (targets * estimated_targets).sum(axis=-2, keepdims=True) / ((targets**2).sum(axis=-2, keepdims=True) + 1e-8)) * targets distortion = estimated_targets - s_targets loss = (((s_targets**2).sum(-2) + 1e-8).log() - ((distortion**2).sum(-2) + 1e-8).log()).mean() # large value of SDR means good performance, so that we take the negative of sdr for the validation loss loss = -1 * loss result = pl.EvalResult(checkpoint_on=loss, early_stop_on=loss) result.log('loss/val_loss', loss, prog_bar=False, logger=True, on_step=False, on_epoch=True, reduce_fx=torch.mean) return result
def validation_step(self, batch, batch_idx): """ The validation step, run per batch during training. Parameters ---------- batch : torch.Tensor The Data batch. batch_idx : int The batch id. Returns ------- Result The output of forward pass. Notes ----- Supports Early Stopping on loss by default. # TODO: make a validation dictionary / callback for more control. """ x, y = batch y_hat = self.model(x) loss = self.loss(y_hat, y) metrics = self.metrics(y_hat, y) result = pl.EvalResult(checkpoint_on=loss, early_stop_on=loss) result.log_dict({'val_metric': metrics, 'val_loss': loss}) return result
def validation_step(self, batch, batch_idx): x, y = batch y_hat = self(x) loss = F.cross_entropy(y_hat, y) result = pl.EvalResult(checkpoint_on=loss) result.log('val_loss', loss) return result
def validation_epoch_end(self, validation_step_outputs): gt_classes = validation_step_outputs.gt_classes inout_mask = gt_classes < 2 gt_inout = gt_classes[inout_mask] inout_prob = validation_step_outputs.inout_prob[inout_mask] inout_acc = modules.accuracy(inout_prob, gt_inout, topk=(1, ))[0] label_mask = gt_classes >= 2 label_prob = validation_step_outputs.label_prob[label_mask] gt_label = gt_classes[label_mask] - 2 top1, top5 = modules.accuracy(label_prob, gt_label, (1, 5)) cate_topk = modules.accuracy_per_cate(label_prob, gt_label, (1, 5)) result = pl.EvalResult() result.log_dict( { 'top1': top1.item(), 'top5': top5.item(), 'inout_acc': inout_acc.item(), }, prog_bar=True, on_epoch=True) print('cate_topk', cate_topk) print('top1: ', top1.item()) print('top5: ', top5.item()) return result
def test_step(self, batch, batch_idx): outputs = self(batch) loss = outputs[0] y_hat = outputs[1] acc = FM.accuracy(y_hat.detach().argmax(axis=1), batch['labels'].flatten(), num_classes=2) result = pl.EvalResult(checkpoint_on=loss) result.log_dict({'val_acc': acc, 'val_loss': loss}) return result