def validation_step_result_only_epoch_metrics(self, batch, batch_idx): """ Only track epoch level metrics """ acc = self.step(batch, batch_idx) result = EvalResult(checkpoint_on=acc, early_stop_on=acc) # step only metrics result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc), prog_bar=False, logger=False) result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc), prog_bar=False, logger=True) result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc), prog_bar=True, logger=True) result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc), prog_bar=True, logger=False) self.validation_step_called = True return result
def test_step_result_obj(self, batch, batch_idx, *args, **kwargs): """ Default, baseline test_step :param batch: :return: """ x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_test = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) test_acc = torch.tensor(test_acc) test_acc = test_acc.type_as(x) result = EvalResult() # alternate possible outputs to test if batch_idx % 1 == 0: result.log_dict({'test_loss': loss_test, 'test_acc': test_acc}) return result if batch_idx % 2 == 0: return test_acc if batch_idx % 3 == 0: result.log_dict({'test_loss': loss_test, 'test_acc': test_acc}) result.test_dic = {'test_loss_a': loss_test} return result
def test_step(self, batch, batch_idx): input_ids, attention_mask, labels = batch outputs = self.forward(input_ids, attention_mask) loss = self.loss_function(outputs, labels) predictions = torch.where( outputs > 0.5, torch.ones(outputs.shape, device=self.device), torch.zeros(outputs.shape, device=self.device)) if int(self.hparams.amount_labels) == 1: batch_size = labels.shape[0] labels = labels[:, 0].view(-1) predictions = predictions[:, 0].view(-1) accuracy = Accuracy() acc = accuracy(predictions, labels) tp = float(torch.logical_and(predictions == 1, labels == 1).sum()) fp = float(torch.logical_and(predictions == 1, labels == 0).sum()) tn = float(torch.logical_and(predictions == 0, labels == 0).sum()) fn = float(torch.logical_and(predictions == 0, labels == 1).sum()) self.test_conf_matrix["tp"] += tp self.test_conf_matrix["fp"] += fp self.test_conf_matrix["tn"] += tn self.test_conf_matrix["fn"] += fn result = EvalResult() result.log("test_loss", loss) result.log("acc", acc) return result
def test_step(self, batch, batch_idx): logits = self(batch, model='best') labels = batch[3] loss = F.cross_entropy(logits, labels, reduction='mean') result = EvalResult() result.log('test_loss', loss, sync_dist=True) result.log_dict(self.calculate_metrics(logits, labels, prefix='test'), sync_dist=True) return result
def validation_step(self, batch, batch_idx): logits = self(batch) labels = batch[3] loss = F.cross_entropy(logits, labels, reduction='mean') result = EvalResult(early_stop_on=loss, checkpoint_on=loss) result.log('val_loss', loss, sync_dist=True) result.log_dict(self.calculate_metrics(logits, labels, prefix='val'), sync_dist=True) return result
def validation_step_result_callbacks(self, batch, batch_idx): acc = self.step(batch, batch_idx) self.assert_backward = False losses = [20, 19, 20, 21, 22, 23] idx = self.current_epoch loss = acc + losses[idx] result = EvalResult(early_stop_on=loss, checkpoint_on=loss) self.validation_step_called = True return result
def validation_step_for_epoch_end_result(self, batch, batch_idx): """ EvalResult flows to epoch end (without step_end) """ acc = self.step(batch, batch_idx) result = EvalResult(checkpoint_on=acc, early_stop_on=acc) # step only metrics result.log('val_step_metric', torch.tensor(batch_idx).type_as(acc), prog_bar=True, logger=True, on_epoch=True, on_step=False) result.log('batch_idx', torch.tensor(batch_idx).type_as(acc), prog_bar=True, logger=True, on_epoch=True, on_step=False) self.validation_step_called = True return result
def validation_step_result_obj(self, batch, batch_idx, *args, **kwargs): x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_val = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) val_acc = torch.tensor(val_acc).type_as(x) result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val) result.log_dict({ 'val_loss': loss_val, 'val_acc': val_acc, }) return result
def eval_step_full_loop_result_obj_dp(self, batch, batch_idx, optimizer_idx=None): """ Full loop flow train step (result obj + dp) """ x, y = batch x = x.view(x.size(0), -1) y_hat = self(x.to(self.device)) loss_val = y_hat.sum() result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val) eval_name = 'validation' if not self.trainer.testing else 'test' result.log(f'{eval_name}_step_metric', loss_val + 1, on_step=True) setattr(self, f'{eval_name}_step_called', True) return result
def test_step_result_preds(self, batch, batch_idx, optimizer_idx=None): x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) loss_test = self.loss(y, y_hat) # acc labels_hat = torch.argmax(y_hat, dim=1) test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) test_acc = torch.tensor(test_acc) test_acc = test_acc.type_as(x) # Do regular EvalResult Logging result = EvalResult(checkpoint_on=loss_test) result.log('test_loss', loss_test) result.log('test_acc', test_acc) batch_size = x.size(0) lst_of_str = [random.choice(['dog', 'cat']) for i in range(batch_size)] lst_of_int = [random.randint(500, 1000) for i in range(batch_size)] lst_of_lst = [[x] for x in lst_of_int] lst_of_dict = [{k: v} for k, v in zip(lst_of_str, lst_of_int)] # This is passed in from pytest via parameterization option = getattr(self, 'test_option', 0) prediction_file = getattr(self, 'prediction_file', 'predictions.pt') lazy_ids = torch.arange(batch_idx * self.batch_size, batch_idx * self.batch_size + x.size(0)) # Base if option == 0: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('preds', labels_hat, prediction_file) # Check mismatching tensor len elif option == 1: self.write_prediction('idxs', torch.cat((lazy_ids, lazy_ids)), prediction_file) self.write_prediction('preds', labels_hat, prediction_file) # write multi-dimension elif option == 2: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('preds', labels_hat, prediction_file) self.write_prediction('x', x, prediction_file) # write str list elif option == 3: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_str, prediction_file) # write int list elif option == 4: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_int, prediction_file) # write nested list elif option == 5: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_lst, prediction_file) # write dict list elif option == 6: self.write_prediction('idxs', lazy_ids, prediction_file) self.write_prediction('vals', lst_of_dict, prediction_file) elif option == 7: self.write_prediction_dict({'idxs': lazy_ids, 'preds': labels_hat}, prediction_file) return result