def _compute_batch():
trainer = Trainer(fast_dev_run=True,
accelerator='horovod',
logger=False)
assert isinstance(trainer.accelerator, CPUAccelerator)
# TODO: test that we selected the correct training_type_plugin based on horovod flags
metric = Accuracy(compute_on_step=True,
dist_sync_on_step=True,
dist_sync_fn=trainer.training_type_plugin.all_gather,
threshold=threshold)
for i in range(hvd.rank(), num_batches, hvd.size()):
batch_result = metric(preds[i], target[i])
if hvd.rank() == 0:
dist_preds = torch.stack(
[preds[i + r] for r in range(hvd.size())])
dist_target = torch.stack(
[target[i + r] for r in range(hvd.size())])
sk_batch_result = sk_metric(dist_preds, dist_target)
assert np.allclose(batch_result.numpy(), sk_batch_result)
# check on all batches on all ranks
result = metric.compute()
assert isinstance(result, torch.Tensor)
total_preds = torch.stack([preds[i] for i in range(num_batches)])
total_target = torch.stack([target[i] for i in range(num_batches)])
sk_result = sk_metric(total_preds, total_target)
assert np.allclose(result.numpy(), sk_result)
def _compute_batch():
trainer = Trainer(
fast_dev_run=True,
distributed_backend='horovod',
)
accelerator_backend = trainer.accelerator_connector.select_accelerator(
)
assert isinstance(accelerator_backend, HorovodAccelerator)
metric = Accuracy(compute_on_step=True,
dist_sync_on_step=True,
dist_sync_fn=accelerator_backend.gather_all_tensors,
threshold=threshold)
for i in range(hvd.rank(), num_batches, hvd.size()):
batch_result = metric(preds[i], target[i])
if hvd.rank() == 0:
dist_preds = torch.stack(
[preds[i + r] for r in range(hvd.size())])
dist_target = torch.stack(
[target[i + r] for r in range(hvd.size())])
sk_batch_result = sk_metric(dist_preds, dist_target)
assert np.allclose(batch_result.numpy(), sk_batch_result)
# check on all batches on all ranks
result = metric.compute()
assert isinstance(result, torch.Tensor)
total_preds = torch.stack([preds[i] for i in range(num_batches)])
total_target = torch.stack([target[i] for i in range(num_batches)])
sk_result = sk_metric(total_preds, total_target)
assert np.allclose(result.numpy(), sk_result)
def __init__(self, n_classes):
super().__init__()
self.save_hyperparameters()
# gpu에서 오류남
# self.example_input_array = {"input_ids": torch.LongTensor(1, 100),
# "attention_mask": torch.LongTensor(1, 100)}
self.acc = Accuracy()
self.loss_fn = nn.CrossEntropyLoss()
self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)
self.drop = nn.Dropout(p=0.3)
self.out = nn.Linear(self.bert.config.hidden_size, n_classes)
def attack(model, dataset, attack_alg='fgsm', eps=0.1, device='cuda'):
accuracy = Accuracy().to(device)
model = model.to(device)
model.eval()
for x, y in dataset:
x, y = x.to(device), y.to(device)
# if model.binary:
# x[x >= 0.5] = 1
# x[x < 0.5] = 0
x_fgsm = fast_gradient_method(model,
x,
eps,
np.inf,
clip_min=0,
clip_max=1)
_, y_pred_fgsm = model(x_fgsm).max(1)
accuracy(y_pred_fgsm, y)
acc = accuracy.compute()
print(acc)
def test_accuracy_invalid_shape():
with pytest.raises(ValueError):
acc = Accuracy()
acc.update(preds=torch.rand(1), target=torch.rand(1, 2, 3))
