def test_wrong_params(average, mdmc_average, num_classes, inputs, ignore_index,
top_k, threshold):
preds, target = inputs.preds, inputs.target
with pytest.raises(ValueError):
acc = Accuracy(
average=average,
mdmc_average=mdmc_average,
num_classes=num_classes,
ignore_index=ignore_index,
threshold=threshold,
top_k=top_k,
)
acc(preds[0], target[0])
acc.compute()
with pytest.raises(ValueError):
accuracy(
preds[0],
target[0],
average=average,
mdmc_average=mdmc_average,
num_classes=num_classes,
ignore_index=ignore_index,
threshold=threshold,
top_k=top_k,
)
def test_topk_accuracy_wrong_input_types(preds, target):
topk = Accuracy(top_k=1)
with pytest.raises(ValueError):
topk(preds[0], target[0])
with pytest.raises(ValueError):
accuracy(preds[0], target[0], top_k=1)
def test_wrong_params(top_k, threshold):
preds, target = _input_mcls_prob.preds, _input_mcls_prob.target
with pytest.raises(ValueError):
acc = Accuracy(threshold=threshold, top_k=top_k)
acc(preds, target)
acc.compute()
with pytest.raises(ValueError):
accuracy(preds, target, threshold=threshold, top_k=top_k)
def main(args):
g, num_rels, num_classes, labels, train_idx, test_idx, target_idx = load_data(
args.dataset, get_norm=True)
num_nodes = g.num_nodes()
# Since the nodes are featureless, learn node embeddings from scratch
# This requires passing the node IDs to the model.
feats = th.arange(num_nodes)
model = RGCN(num_nodes,
args.n_hidden,
num_classes,
num_rels,
num_bases=args.n_bases)
if args.gpu >= 0 and th.cuda.is_available():
device = th.device(args.gpu)
else:
device = th.device('cpu')
feats = feats.to(device)
labels = labels.to(device)
model = model.to(device)
g = g.to(device)
optimizer = th.optim.Adam(model.parameters(),
lr=1e-2,
weight_decay=args.l2norm)
model.train()
for epoch in range(50):
logits = model(g, feats)
logits = logits[target_idx]
loss = F.cross_entropy(logits[train_idx], labels[train_idx])
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_acc = accuracy(logits[train_idx].argmax(dim=1),
labels[train_idx]).item()
print("Epoch {:05d} | Train Accuracy: {:.4f} | Train Loss: {:.4f}".
format(epoch, train_acc, loss.item()))
print()
model.eval()
with th.no_grad():
logits = model(g, feats)
logits = logits[target_idx]
test_acc = accuracy(logits[test_idx].argmax(dim=1),
labels[test_idx]).item()
print("Test Accuracy: {:.4f}".format(test_acc))
def test_step(self, batch, batch_idx):
inputs, targets = batch
outputs = self.forward(inputs)
test_accuracy = accuracy(outputs, targets)
loss = self.loss(outputs, targets)
self.log('test_accuracy', test_accuracy)
return {"test_loss": loss, "test_accuracy": test_accuracy}
def _loss(self, batch, batch_idx):
x, y = batch
body = self.body(x)
y_pred = self.policy_head(body)
loss = binary_cross_entropy_with_logits(body, y)
ba = accuracy((y_pred > 0.5).float(), y.bool())
return loss, ba
def layerwise_infer(device, graph, nid, model, batch_size):
model.eval()
with torch.no_grad():
pred = model.inference(graph, device, batch_size).to(device)
pred = pred[nid]
label = graph.ndata['label'][nid]
return MF.accuracy(pred, label)
def on_validation_batch_end(
self,
trainer: Trainer,
pl_module: LightningModule,
outputs: Sequence,
batch: Sequence,
batch_idx: int,
dataloader_idx: int,
) -> None:
x, y = self.to_device(batch, pl_module.device)
with torch.no_grad():
representations = self.get_representations(pl_module, x)
representations = representations.detach()
# forward pass
mlp_preds = pl_module.non_linear_evaluator(
representations) # type: ignore[operator]
mlp_loss = F.cross_entropy(mlp_preds, y)
# log metrics
val_acc = accuracy(mlp_preds, y)
pl_module.log('online_val_acc',
val_acc,
on_step=False,
on_epoch=True,
sync_dist=True)
pl_module.log('online_val_loss',
mlp_loss,
on_step=False,
on_epoch=True,
sync_dist=True)
def validation_step(self, batch: Tuple[Tensor, Tensor],
batch_idx: int) -> Dict[str, Tensor]:
x, y = batch
x = x.view(x.size(0), -1)
y_hat = self.linear(x)
acc = accuracy(F.softmax(y_hat, -1), y)
return {'val_loss': F.cross_entropy(y_hat, y), 'acc': acc}
def test_step(self, batch: Tuple[Tensor, Tensor],
batch_idx: int) -> Dict[str, Tensor]:
x, y = batch
x = x.view(x.size(0), -1)
y_hat = self.linear(x)
acc = accuracy(F.softmax(y_hat, -1), y)
return {"test_loss": F.cross_entropy(y_hat, y), "acc": acc}
def on_train_batch_end(
self,
trainer: Trainer,
pl_module: LightningModule,
outputs: Sequence,
batch: Sequence,
batch_idx: int,
dataloader_idx: int,
) -> None:
x, y = self.to_device(batch, pl_module.device)
with torch.no_grad():
representations = self.get_representations(pl_module, x)
representations = representations.detach()
# forward pass
mlp_logits = pl_module.non_linear_evaluator(representations) # type: ignore[operator]
mlp_loss = F.cross_entropy(mlp_logits, y)
# update finetune weights
mlp_loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
# log metrics
train_acc = accuracy(mlp_logits.softmax(-1), y)
pl_module.log('online_train_acc', train_acc, on_step=True, on_epoch=False)
pl_module.log('online_train_loss', mlp_loss, on_step=True, on_epoch=False)
def test_step(self, batch_data, batch_index):
x, y = batch_data
logits = self(x)
criterion = nn.CrossEntropyLoss()
probs = torch.softmax(logits, dim=1)
# validation metrics
acc = accuracy(torch.argmax(probs, dim=1), torch.argmax(y, dim=1))
loss = criterion(logits, torch.argmax(y, dim=1))
f_score = f1(torch.argmax(probs, dim=1),
torch.argmax(y, dim=1),
average='weighted',
num_classes=4)
self.log('test/f1', f_score, prog_bar=True)
self.log('test/loss', loss, prog_bar=True)
self.log('test/accuracy', acc, prog_bar=True)
predictions = torch.argmax(probs, dim=1)
targets = torch.argmax(y, dim=1)
return {
"test_loss": loss,
"test_accuracy": acc,
"f_score": f_score,
"predictions": predictions,
"targets": targets
}
def train(rank, world_size, graph, num_classes, split_idx):
torch.cuda.set_device(rank)
dist.init_process_group('nccl',
'tcp://127.0.0.1:12347',
world_size=world_size,
rank=rank)
model = SAGE(graph.ndata['feat'].shape[1], 256, num_classes).cuda()
model = nn.parallel.DistributedDataParallel(model,
device_ids=[rank],
output_device=rank)
opt = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=5e-4)
train_idx, valid_idx, test_idx = split_idx['train'], split_idx[
'valid'], split_idx['test']
if USE_WRAPPER:
import dglnew
graph = dglnew.graph.wrapper.DGLGraphStorage(graph)
sampler = dgl.dataloading.NeighborSampler([5, 5, 5],
output_device='cpu',
prefetch_node_feats=['feat'],
prefetch_labels=['label'])
dataloader = dgl.dataloading.NodeDataLoader(
graph,
train_idx,
sampler,
device='cuda',
batch_size=1000,
shuffle=True,
drop_last=False,
pin_memory=True,
num_workers=4,
persistent_workers=True,
use_ddp=True,
use_prefetch_thread=True) # TBD: could probably remove this argument
durations = []
for _ in range(10):
t0 = time.time()
for it, (input_nodes, output_nodes, blocks) in enumerate(dataloader):
x = blocks[0].srcdata['feat']
y = blocks[-1].dstdata['label'][:, 0]
y_hat = model(blocks, x)
loss = F.cross_entropy(y_hat, y)
opt.zero_grad()
loss.backward()
opt.step()
if it % 20 == 0:
acc = MF.accuracy(y_hat, y)
mem = torch.cuda.max_memory_allocated() / 1000000
print('Loss', loss.item(), 'Acc', acc.item(), 'GPU Mem', mem,
'MB')
tt = time.time()
if rank == 0:
print(tt - t0)
durations.append(tt - t0)
if rank == 0:
print(np.mean(durations[4:]), np.std(durations[4:]))
def update(
self,
predicted_y: Tensor,
true_y: Tensor,
task_labels: Union[float, Tensor],
) -> None:
"""
Update the running top-k accuracy given the true and predicted labels.
Parameter `task_labels` is used to decide how to update the inner
dictionary: if Float, only the dictionary value related to that task
is updated. If Tensor, all the dictionary elements belonging to the
task labels will be updated.
:param predicted_y: The model prediction. Both labels and logit vectors
are supported.
:param true_y: The ground truth. Both labels and one-hot vectors
are supported.
:param task_labels: the int task label associated to the current
experience or the task labels vector showing the task label
for each pattern.
:return: None.
"""
if len(true_y) != len(predicted_y):
raise ValueError(
"Size mismatch for true_y and predicted_y tensors")
if isinstance(task_labels, Tensor) and len(task_labels) != len(true_y):
raise ValueError(
"Size mismatch for true_y and task_labels tensors")
true_y = torch.as_tensor(true_y)
predicted_y = torch.as_tensor(predicted_y)
if isinstance(task_labels, int):
total_patterns = len(true_y)
self._topk_acc_dict[task_labels].update(
accuracy(predicted_y, true_y, top_k=self.top_k),
total_patterns)
elif isinstance(task_labels, Tensor):
for pred, true, t in zip(predicted_y, true_y, task_labels):
self._topk_acc_dict[t.item()].update(
accuracy(pred, true, top_k=self.top_k), 1)
else:
raise ValueError(f"Task label type: {type(task_labels)}, "
f"expected int/float or Tensor")
def test_step(self, batch, batch_idx):
out = self.forward(batch)
loss = self.criterion(out.float(), batch.label)
preds = torch.argmax(F.softmax(out, 1), 1)
self.test_correct += torch.sum(preds == batch.label).item()
self.log("test_loss", loss)
self.log("test_acc_step", accuracy(preds, batch.label), prog_bar=True)
return {'test_loss': loss}
def training_step(self, batch, batch_idx):
inputs, targets = batch
outputs = self(inputs)
train_accuracy = accuracy(outputs, targets)
loss = self.loss(outputs, targets)
self.log('train_accuracy', train_accuracy, prog_bar=True)
self.log('train_loss', loss)
return {"loss": loss, "train_accuracy": train_accuracy}
def validation_epoch_end(self, outputs):
val_loss = sum([x["val_loss"] for x in outputs])
pred = torch.cat([x["pred"] for x in outputs])
true = torch.cat([x["true"] for x in outputs])
f_score = metrics.f1(pred, true, num_classes=2)
accuracy = metrics.accuracy(pred, true)
out = {"val_loss": val_loss, "val_f_score": f_score, "val_accuracy": accuracy}
self.log_dict({"val_loss": val_loss, "val_f_score": f_score, "val_accuracy": accuracy})
return {**out, "log": out}
def _common_step(self, batch):
input_tensor, target = batch
output = self(input_tensor)
_, pred = torch.max(output, axis=1)
loss = F.cross_entropy(output, target)
acc = M.accuracy(pred, target)
return loss, acc
def score_epoch_end(self, results: list[dict[str, torch.Tensor]]):
"""
Score and evaluate the given dataset.
"""
items = {
"B-M": ("pm", "ym"),
"BIRAD": ("pb", "yb"),
"discrim": ("cy", "cy-GT"),
}
for res, caption in zip(results, self.score_caption):
res = deep_collate(res)
if (c := ('fi' in res and 'fi') or ('ft' in res and 'ft')):
self.logger.experiment.add_embedding(
res[c],
metadata=res["ym"].tolist(),
global_step=self.current_epoch,
tag=caption,
)
if "dice" in res:
self.log(f"segment/dice/{caption}",
res["dice"].mean(),
logger=True)
if 'confidence' in res:
self.log(f"segment/confidence/{caption}",
res["confidence"].mean(),
logger=True)
for k, (p, y) in items.items():
if p not in res or y not in res: continue
p, y = res[p], res[y]
if all(i is None for i in y): continue
if isinstance(y, list):
p = torch.stack(
[p[i] for i, v in enumerate(y) if v is not None])
y = torch.tensor(
[v for i, v in enumerate(y) if v is not None],
dtype=torch.long,
device=p.device)
assert len(p) == len(y)
err = 1 - accuracy(p, y)
self.log(f"err/{k}/{caption}", err, logger=True)
if p.dim() == 2 and p.size(1) <= 2:
p = p[:, -1]
if p.dim() == 1:
self.logger.experiment.add_pr_curve(
f"{k}/{caption}", y, p, self.current_epoch)
self.logger.experiment.add_histogram(
f"distribution/{k}/{caption}", p, self.current_epoch)
else:
self.logger.experiment.add_histogram(
f"distribution/{k}/{caption}", p, self.current_epoch)
def main(args):
g, num_rels, num_classes, labels, train_idx, test_idx, target_idx = load_data(
args.dataset, get_norm=True)
model = RGCN(g.num_nodes(),
args.n_hidden,
num_classes,
num_rels,
num_bases=args.n_bases)
if args.gpu >= 0 and th.cuda.is_available():
device = th.device(args.gpu)
else:
device = th.device('cpu')
labels = labels.to(device)
model = model.to(device)
g = g.int().to(device)
optimizer = th.optim.Adam(model.parameters(),
lr=1e-2,
weight_decay=args.wd)
model.train()
for epoch in range(100):
logits = model(g)
logits = logits[target_idx]
loss = F.cross_entropy(logits[train_idx], labels[train_idx])
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_acc = accuracy(logits[train_idx].argmax(dim=1),
labels[train_idx]).item()
print("Epoch {:05d} | Train Accuracy: {:.4f} | Train Loss: {:.4f}".
format(epoch, train_acc, loss.item()))
print()
model.eval()
with th.no_grad():
logits = model(g)
logits = logits[target_idx]
test_acc = accuracy(logits[test_idx].argmax(dim=1),
labels[test_idx]).item()
print("Test Accuracy: {:.4f}".format(test_acc))
def validation_step(self, batch, _batch_idx):
x, y, _idx = batch
logits = self(x)
loss = F.nll_loss(logits, y)
prediction = torch.argmax(logits, dim=1)
acc = accuracy(prediction, y)
self.log('val_loss', loss, prog_bar=True)
self.log('val_acc', acc, prog_bar=True)
return loss
def main(args):
g, num_rels, num_classes, labels, train_idx, test_idx, target_idx, inv_target = load_data(
args.dataset, inv_target=True)
if args.gpu >= 0 and th.cuda.is_available():
device = th.device(args.gpu)
else:
device = th.device('cpu')
train_loader, val_loader, test_loader = init_dataloaders(
args, g, train_idx, test_idx, target_idx, args.gpu)
model = RGCN(g.num_nodes(),
args.n_hidden,
num_classes,
num_rels,
num_bases=args.n_bases,
dropout=args.dropout,
self_loop=args.use_self_loop,
ns_mode=True)
labels = labels.to(device)
model = model.to(device)
optimizer = th.optim.Adam(model.parameters(),
lr=1e-2,
weight_decay=args.wd)
for epoch in range(args.n_epochs):
train_acc, loss = train(model, train_loader, inv_target, labels,
optimizer)
print(
"Epoch {:05d}/{:05d} | Train Accuracy: {:.4f} | Train Loss: {:.4f}"
.format(epoch, args.n_epochs, train_acc, loss))
val_logits, val_seeds = evaluate(model, val_loader, inv_target)
val_acc = accuracy(val_logits.argmax(dim=1),
labels[val_seeds].cpu()).item()
print("Validation Accuracy: {:.4f}".format(val_acc))
test_logits, test_seeds = evaluate(model, test_loader, inv_target)
test_acc = accuracy(test_logits.argmax(dim=1),
labels[test_seeds].cpu()).item()
print("Final Test Accuracy: {:.4f}".format(test_acc))
def evaluate(self, batch, stage=None):
outputs = self.forward(**batch)
logits = outputs.logits
loss = outputs.loss
preds = torch.argmax(logits, dim=-1)
acc = accuracy(preds, batch["labels"])
if stage:
self.log(f"{stage}_loss", loss, prog_bar=True)
self.log(f"{stage}_acc", acc, prog_bar=True)
def evaluate(model, graph, dataloader):
model.eval()
ys = []
y_hats = []
for it, (input_nodes, output_nodes, blocks) in enumerate(dataloader):
with torch.no_grad():
x = blocks[0].srcdata['feat']
ys.append(blocks[-1].dstdata['label'])
y_hats.append(model(blocks, x))
return MF.accuracy(torch.cat(y_hats), torch.cat(ys))
def main(args):
g, num_rels, num_classes, labels, train_idx, test_idx, target_idx, inv_target = load_data(
args.dataset, inv_target=True)
if args.gpu >= 0 and th.cuda.is_available():
device = th.device(args.gpu)
else:
device = th.device('cpu')
train_loader, val_loader, test_loader = init_dataloaders(
args, g, train_idx, test_idx, target_idx, args.gpu)
embed_layer, model = init_models(args, device, g.num_nodes(), num_classes,
num_rels)
labels = labels.to(device)
model = model.to(device)
emb_optimizer = th.optim.SparseAdam(embed_layer.parameters(),
lr=args.sparse_lr)
optimizer = th.optim.Adam(model.parameters(),
lr=1e-2,
weight_decay=args.l2norm)
for epoch in range(args.n_epochs):
train_acc, loss = train(model, embed_layer, train_loader, inv_target,
labels, emb_optimizer, optimizer)
print(
"Epoch {:05d}/{:05d} | Train Accuracy: {:.4f} | Train Loss: {:.4f}"
.format(epoch, args.n_epochs, train_acc, loss))
val_logits, val_seeds = evaluate(model, embed_layer, val_loader,
inv_target)
val_acc = accuracy(val_logits.argmax(dim=1),
labels[val_seeds].cpu()).item()
print("Validation Accuracy: {:.4f}".format(val_acc))
test_logits, test_seeds = evaluate(model, embed_layer, test_loader,
inv_target)
test_acc = accuracy(test_logits.argmax(dim=1),
labels[test_seeds].cpu()).item()
print("Final Test Accuracy: {:.4f}".format(test_acc))
def collect_eval(n_gpus, queue, labels):
eval_logits = []
eval_seeds = []
for _ in range(n_gpus):
eval_l, eval_s = queue.get()
eval_logits.append(eval_l)
eval_seeds.append(eval_s)
eval_logits = th.cat(eval_logits)
eval_seeds = th.cat(eval_seeds)
eval_acc = accuracy(eval_logits.argmax(dim=1), labels[eval_seeds].cpu()).item()
return eval_acc
def evaluate(self, batch, stage=None):
x, y = batch
x = x.permute(0, 2, 3, 1)
logits = self(x)
loss = F.cross_entropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = accuracy(preds, y)
if stage:
self.log(f'{stage}_loss', loss, prog_bar=True)
self.log(f'{stage}_acc', acc, prog_bar=True)
return {'loss': loss, 'acc': acc, 'preds': preds, 'y': y}
def _evaluate(self, batch, batch_idx, stage=None):
x, y = batch
out = self.forward(x)
logits = F.log_softmax(out, dim=-1)
loss = F.nll_loss(logits, y)
preds = torch.argmax(logits, dim=-1)
acc = accuracy(preds, y)
if stage:
self.log(f'{stage}_loss', loss, prog_bar=True)
self.log(f'{stage}_acc', acc, prog_bar=True)
return loss, acc
def test_ignore_index(preds, target, ignore_index, exp_result,
subset_accuracy):
ignoreindex = Accuracy(ignore_index=ignore_index,
subset_accuracy=subset_accuracy)
for batch in range(preds.shape[0]):
ignoreindex(preds[batch], target[batch])
assert ignoreindex.compute() == exp_result
assert accuracy(preds,
target,
ignore_index=ignore_index,
subset_accuracy=subset_accuracy) == exp_result
def test_step(self, test_batch, batch_idx):
"""
Performs test and computes the accuracy of the model
:param test_batch: Batch data
:param batch_idx: Batch indices
:return: output - Testing accuracy
"""
x, y = test_batch
output = self.forward(x)
_, y_hat = torch.max(output, dim=1)
test_acc = accuracy(y_hat.cpu(), y.cpu())
return {"test_acc": test_acc}
