def validation_step_result_only_epoch_metrics(self, batch, batch_idx):
"""
Only track epoch level metrics
"""
acc = self.step(batch, batch_idx)
result = EvalResult(checkpoint_on=acc, early_stop_on=acc)
# step only metrics
result.log('no_val_no_pbar',
torch.tensor(11 + batch_idx).type_as(acc),
prog_bar=False,
logger=False)
result.log('val_step_log_acc',
torch.tensor(11 + batch_idx).type_as(acc),
prog_bar=False,
logger=True)
result.log('val_step_log_pbar_acc',
torch.tensor(12 + batch_idx).type_as(acc),
prog_bar=True,
logger=True)
result.log('val_step_pbar_acc',
torch.tensor(13 + batch_idx).type_as(acc),
prog_bar=True,
logger=False)
self.validation_step_called = True
return result
def test_step_result_obj(self, batch, batch_idx, *args, **kwargs):
"""
Default, baseline test_step
:param batch:
:return:
"""
x, y = batch
x = x.view(x.size(0), -1)
y_hat = self(x)
loss_test = self.loss(y, y_hat)
# acc
labels_hat = torch.argmax(y_hat, dim=1)
test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
test_acc = torch.tensor(test_acc)
test_acc = test_acc.type_as(x)
result = EvalResult()
# alternate possible outputs to test
if batch_idx % 1 == 0:
result.log_dict({'test_loss': loss_test, 'test_acc': test_acc})
return result
if batch_idx % 2 == 0:
return test_acc
if batch_idx % 3 == 0:
result.log_dict({'test_loss': loss_test, 'test_acc': test_acc})
result.test_dic = {'test_loss_a': loss_test}
return result
def validation_step_result_obj(self, batch, batch_idx, *args, **kwargs):
x, y = batch
x = x.view(x.size(0), -1)
y_hat = self(x)
loss_val = self.loss(y, y_hat)
# acc
labels_hat = torch.argmax(y_hat, dim=1)
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
val_acc = torch.tensor(val_acc).type_as(x)
result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val)
result.log_dict({
'val_loss': loss_val,
'val_acc': val_acc,
})
return result
def eval_step_full_loop_result_obj_dp(self,
batch,
batch_idx,
optimizer_idx=None):
"""
Full loop flow train step (result obj + dp)
"""
x, y = batch
x = x.view(x.size(0), -1)
y_hat = self(x.to(self.device))
loss_val = y_hat.sum()
result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val)
eval_name = 'validation' if not self.trainer.testing else 'test'
result.log(f'{eval_name}_step_metric', loss_val + 1, on_step=True)
setattr(self, f'{eval_name}_step_called', True)
return result
def validation_step_result_callbacks(self, batch, batch_idx):
acc = self.step(batch, batch_idx)
self.assert_backward = False
losses = [20, 19, 20, 21, 22, 23]
idx = self.current_epoch
loss = acc + losses[idx]
result = EvalResult(early_stop_on=loss, checkpoint_on=loss)
self.validation_step_called = True
return result
def test_step(self, batch, batch_idx):
input_ids, attention_mask, labels = batch
outputs = self.forward(input_ids, attention_mask)
loss = self.loss_function(outputs, labels)
predictions = torch.where(
outputs > 0.5, torch.ones(outputs.shape, device=self.device),
torch.zeros(outputs.shape, device=self.device))
if int(self.hparams.amount_labels) == 1:
batch_size = labels.shape[0]
labels = labels[:, 0].view(-1)
predictions = predictions[:, 0].view(-1)
accuracy = Accuracy()
acc = accuracy(predictions, labels)
tp = float(torch.logical_and(predictions == 1, labels == 1).sum())
fp = float(torch.logical_and(predictions == 1, labels == 0).sum())
tn = float(torch.logical_and(predictions == 0, labels == 0).sum())
fn = float(torch.logical_and(predictions == 0, labels == 1).sum())
self.test_conf_matrix["tp"] += tp
self.test_conf_matrix["fp"] += fp
self.test_conf_matrix["tn"] += tn
self.test_conf_matrix["fn"] += fn
result = EvalResult()
result.log("test_loss", loss)
result.log("acc", acc)
return result
def test_step(self, batch, batch_idx):
logits = self(batch, model='best')
labels = batch[3]
loss = F.cross_entropy(logits, labels, reduction='mean')
result = EvalResult()
result.log('test_loss', loss, sync_dist=True)
result.log_dict(self.calculate_metrics(logits, labels, prefix='test'),
sync_dist=True)
return result
def validation_step(self, batch, batch_idx):
logits = self(batch)
labels = batch[3]
loss = F.cross_entropy(logits, labels, reduction='mean')
result = EvalResult(early_stop_on=loss, checkpoint_on=loss)
result.log('val_loss', loss, sync_dist=True)
result.log_dict(self.calculate_metrics(logits, labels, prefix='val'),
sync_dist=True)
return result
def validation_step_for_epoch_end_result(self, batch, batch_idx):
"""
EvalResult flows to epoch end (without step_end)
"""
acc = self.step(batch, batch_idx)
result = EvalResult(checkpoint_on=acc, early_stop_on=acc)
# step only metrics
result.log('val_step_metric', torch.tensor(batch_idx).type_as(acc),
prog_bar=True, logger=True, on_epoch=True, on_step=False)
result.log('batch_idx', torch.tensor(batch_idx).type_as(acc),
prog_bar=True, logger=True, on_epoch=True, on_step=False)
self.validation_step_called = True
return result
def test_step_result_preds(self, batch, batch_idx, optimizer_idx=None):
x, y = batch
x = x.view(x.size(0), -1)
y_hat = self(x)
loss_test = self.loss(y, y_hat)
# acc
labels_hat = torch.argmax(y_hat, dim=1)
test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
test_acc = torch.tensor(test_acc)
test_acc = test_acc.type_as(x)
# Do regular EvalResult Logging
result = EvalResult(checkpoint_on=loss_test)
result.log('test_loss', loss_test)
result.log('test_acc', test_acc)
batch_size = x.size(0)
lst_of_str = [random.choice(['dog', 'cat']) for i in range(batch_size)]
lst_of_int = [random.randint(500, 1000) for i in range(batch_size)]
lst_of_lst = [[x] for x in lst_of_int]
lst_of_dict = [{k: v} for k, v in zip(lst_of_str, lst_of_int)]
# This is passed in from pytest via parameterization
option = getattr(self, 'test_option', 0)
prediction_file = getattr(self, 'prediction_file', 'predictions.pt')
lazy_ids = torch.arange(batch_idx * self.batch_size, batch_idx * self.batch_size + x.size(0))
# Base
if option == 0:
self.write_prediction('idxs', lazy_ids, prediction_file)
self.write_prediction('preds', labels_hat, prediction_file)
# Check mismatching tensor len
elif option == 1:
self.write_prediction('idxs', torch.cat((lazy_ids, lazy_ids)), prediction_file)
self.write_prediction('preds', labels_hat, prediction_file)
# write multi-dimension
elif option == 2:
self.write_prediction('idxs', lazy_ids, prediction_file)
self.write_prediction('preds', labels_hat, prediction_file)
self.write_prediction('x', x, prediction_file)
# write str list
elif option == 3:
self.write_prediction('idxs', lazy_ids, prediction_file)
self.write_prediction('vals', lst_of_str, prediction_file)
# write int list
elif option == 4:
self.write_prediction('idxs', lazy_ids, prediction_file)
self.write_prediction('vals', lst_of_int, prediction_file)
# write nested list
elif option == 5:
self.write_prediction('idxs', lazy_ids, prediction_file)
self.write_prediction('vals', lst_of_lst, prediction_file)
# write dict list
elif option == 6:
self.write_prediction('idxs', lazy_ids, prediction_file)
self.write_prediction('vals', lst_of_dict, prediction_file)
elif option == 7:
self.write_prediction_dict({'idxs': lazy_ids, 'preds': labels_hat}, prediction_file)
return result