def __init__(self, device, num_classes, topK=3):
if not pl:
return
self.device = device
self.topK = topK
# https://github.com/PyTorchLightning/metrics/blob/master/torchmetrics/classification/f_beta.py#L221
#mdmc_average = "samplewise"
mdmc_average = "global"
val_metrics = {
'hamming_dist':
HammingDistance() if HammingDistance is not None else None,
'iou':
IoU(num_classes=num_classes),
'auroc':
AUROC(num_classes=num_classes),
'f1':
F1(num_classes=num_classes,
multilabel=True,
mdmc_average=mdmc_average),
'avg_precision':
AveragePrecision(num_classes=num_classes),
#'acc': Accuracy(num_classes=num_classes, mdmc_average = mdmc_average)
}
for k in range(1, topK + 1):
val_metrics["top%d" % k] = Accuracy(top_k=k)
val_metrics["top%d_f1" % k] = F1(top_k=k)
self.val_metrics = torch.nn.ModuleDict(val_metrics).to(self.device)
self.class_names = list(range(num_classes))
self.label_binarizer = MultiLabelBinarizer(classes=self.class_names)
def test_v1_5_metric_auc_auroc():
AUC.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
AUC()
ROC.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
ROC()
AUROC.__init__._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
AUROC()
x = torch.tensor([0, 1, 2, 3])
y = torch.tensor([0, 1, 2, 2])
auc._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
assert auc(x, y) == torch.tensor(4.)
preds = torch.tensor([0, 1, 2, 3])
target = torch.tensor([0, 1, 1, 1])
roc._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
fpr, tpr, thrs = roc(preds, target, pos_label=1)
assert torch.equal(fpr, torch.tensor([0., 0., 0., 0., 1.]))
assert torch.allclose(tpr, torch.tensor([0.0000, 0.3333, 0.6667, 1.0000, 1.0000]), atol=1e-4)
assert torch.equal(thrs, torch.tensor([4, 3, 2, 1, 0]))
preds = torch.tensor([0.13, 0.26, 0.08, 0.19, 0.34])
target = torch.tensor([0, 0, 1, 1, 1])
auroc._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
assert auroc(preds, target) == torch.tensor(0.5)
def __init__(self, prefix, loss_type: str, threshold=0.5, top_k=[1, 5, 10], n_classes: int = None,
multilabel: bool = None, metrics=["precision", "recall", "top_k", "accuracy"]):
super().__init__()
self.loss_type = loss_type.upper()
self.threshold = threshold
self.n_classes = n_classes
self.multilabel = multilabel
self.top_ks = top_k
self.prefix = prefix
self.metrics = {}
for metric in metrics:
if "precision" == metric:
self.metrics[metric] = Precision(average=True, is_multilabel=multilabel)
elif "recall" == metric:
self.metrics[metric] = Recall(average=True, is_multilabel=multilabel)
elif "top_k" in metric:
if n_classes:
top_k = [k for k in top_k if k < n_classes]
if multilabel:
self.metrics[metric] = TopKMultilabelAccuracy(k_s=top_k)
else:
self.metrics[metric] = TopKCategoricalAccuracy(k=max(int(np.log(n_classes)), 1),
output_transform=None)
elif "macro_f1" in metric:
self.metrics[metric] = F1(num_classes=n_classes, average="macro", multilabel=multilabel)
elif "micro_f1" in metric:
self.metrics[metric] = F1(num_classes=n_classes, average="micro", multilabel=multilabel)
elif "mse" == metric:
self.metrics[metric] = MeanSquaredError()
elif "auroc" == metric:
self.metrics[metric] = AUROC(num_classes=n_classes)
elif "avg_precision" in metric:
self.metrics[metric] = AveragePrecision(num_classes=n_classes, )
elif "accuracy" in metric:
self.metrics[metric] = Accuracy(top_k=int(metric.split("@")[-1]) if "@" in metric else None)
elif "ogbn" in metric:
self.metrics[metric] = OGBNodeClfMetrics(NodeEvaluator(metric))
elif "ogbg" in metric:
self.metrics[metric] = OGBNodeClfMetrics(GraphEvaluator(metric))
elif "ogbl" in metric:
self.metrics[metric] = OGBLinkPredMetrics(LinkEvaluator(metric))
else:
print(f"WARNING: metric {metric} doesn't exist")
# Needed to add the PytorchGeometric methods as Modules, so they'll be on the correct CUDA device during training
if isinstance(self.metrics[metric], torchmetrics.metric.Metric):
setattr(self, metric, self.metrics[metric])
self.reset_metrics()
def __init__(self, args):
super(TransferLearning, self).__init__()
self.hparams = args
self.world_size = self.hparams.num_nodes * self.hparams.gpus
self.train_set_transform, self.val_set_transform = grab_transforms(
self.hparams)
#Grab the correct model - only want the embeddings from the final layer!
if self.hparams.saved_model_type == 'contrastive':
saved_model = NoisyCLIP.load_from_checkpoint(
self.hparams.checkpoint_path)
self.backbone = saved_model.noisy_visual_encoder
elif self.hparams.saved_model_type == 'baseline':
saved_model = Baseline.load_from_checkpoint(
self.hparams.checkpoint_path)
self.backbone = saved_model.encoder.feature_extractor
for param in self.backbone.parameters():
param.requires_grad = False
#Set up a classifier with the correct dimensions
self.output = nn.Linear(self.hparams.emb_dim, self.hparams.num_classes)
#Set up the criterion and stuff
#(3) Set up our criterion - here we use reduction as "sum" so that we are able to average over all validation sets
self.criterion = nn.CrossEntropyLoss(reduction="mean")
self.train_top_1 = Accuracy(top_k=1)
self.train_top_5 = Accuracy(top_k=5)
self.val_top_1 = Accuracy(top_k=1)
self.val_top_5 = Accuracy(top_k=5)
self.test_top_1 = Accuracy(top_k=1)
self.test_top_5 = Accuracy(top_k=5)
#class INFECTED has label 0
if self.hparams.dataset == 'COVID':
self.val_auc = AUROC(pos_label=0)
self.test_auc = AUROC(pos_label=0)
def __init__(self):
super().__init__()
self.metrics_singleclass = nn.ModuleDict({
'loss':
BCEWithLogitsLoss(),
'acc':
SoftMaxWrapper(Accuracy(), multiclass=False),
'auc':
AUROC(),
})
self.metrics_multiclass = nn.ModuleDict({
'loss':
CrossEntropyLoss(),
'acc':
SoftMaxWrapper(Accuracy(), multiclass=True),
})
def evaluate_metrics(self, batch_labels,
batch_predictions) -> Dict[str, torch.Tensor]:
if self.hparams.mode == 'classification':
# transformers convention is to output classification as two neurons.
# In order to convert this to a class label we take the argmax.
probs = nn.Softmax(dim=1)(batch_predictions)
preds = torch.argmax(probs, dim=1).squeeze()
probs_of_positive_class = probs[:, 1]
batch_labels = batch_labels.squeeze()
else:
preds = batch_predictions
if self.hparams.mode == 'classification':
metrics = {
'AUROC':
lambda: AUROC()(probs_of_positive_class, batch_labels),
'AveragePrecision':
lambda: AveragePrecision()
(probs_of_positive_class, batch_labels),
'Accuracy':
lambda: Accuracy()(preds, batch_labels),
}
else:
metrics = {
'MAE': lambda: MAE()(preds, batch_labels),
'RMSE': lambda: RMSE()(preds, batch_labels),
'MSE': lambda: MSE()(preds, batch_labels),
# sklearn metrics work the other way round metric_fn(y_true, y_pred)
'R2': lambda: r2_score(batch_labels.cpu(), preds.cpu()),
}
out = {}
for name, callable_metric in metrics.items():
try:
out[name] = callable_metric().item()
except Exception as e:
logger.info(f'unable to calculate {name} metric')
logger.info(e)
out[name] = np.nan
return out
class TransferLearning(LightningModule):
def __init__(self, args):
super(TransferLearning, self).__init__()
self.hparams = args
self.world_size = self.hparams.num_nodes * self.hparams.gpus
self.train_set_transform, self.val_set_transform = grab_transforms(
self.hparams)
#Grab the correct model - only want the embeddings from the final layer!
if self.hparams.saved_model_type == 'contrastive':
saved_model = NoisyCLIP.load_from_checkpoint(
self.hparams.checkpoint_path)
self.backbone = saved_model.noisy_visual_encoder
elif self.hparams.saved_model_type == 'baseline':
saved_model = Baseline.load_from_checkpoint(
self.hparams.checkpoint_path)
self.backbone = saved_model.encoder.feature_extractor
for param in self.backbone.parameters():
param.requires_grad = False
#Set up a classifier with the correct dimensions
self.output = nn.Linear(self.hparams.emb_dim, self.hparams.num_classes)
#Set up the criterion and stuff
#(3) Set up our criterion - here we use reduction as "sum" so that we are able to average over all validation sets
self.criterion = nn.CrossEntropyLoss(reduction="mean")
self.train_top_1 = Accuracy(top_k=1)
self.train_top_5 = Accuracy(top_k=5)
self.val_top_1 = Accuracy(top_k=1)
self.val_top_5 = Accuracy(top_k=5)
self.test_top_1 = Accuracy(top_k=1)
self.test_top_5 = Accuracy(top_k=5)
#class INFECTED has label 0
if self.hparams.dataset == 'COVID':
self.val_auc = AUROC(pos_label=0)
self.test_auc = AUROC(pos_label=0)
def forward(self, x):
#Grab the noisy image embeddings
self.backbone.eval()
with torch.no_grad():
if self.hparams.encoder == "clip":
noisy_embeddings = self.backbone(x.type(torch.float16)).float()
elif self.hparams.encoder == "resnet":
noisy_embeddings = self.backbone(x)
return self.output(noisy_embeddings.flatten(1))
def configure_optimizers(self):
if not hasattr(self.hparams, 'weight_decay'):
self.hparams.weight_decay = 0
opt = torch.optim.Adam(self.output.parameters(),
lr=self.hparams.lr,
weight_decay=self.hparams.weight_decay)
num_steps = self.hparams.max_epochs
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt,
T_max=num_steps)
return [opt], [scheduler]
def _grab_dataset(self, split):
"""
Given a split ("train" or "val" or "test") and a dataset, returns the proper dataset.
Dataset needed is defined in this object's hparams
Args:
split: the split to use in the dataset
Returns:
dataset: the desired dataset with the correct split
"""
if self.hparams.dataset == "CIFAR10":
if split == 'train':
train = True
transform = self.train_set_transform
else:
train = False
transform = self.val_set_transform
dataset = CIFAR10(root=self.hparams.dataset_dir,
train=train,
transform=transform,
download=True)
elif self.hparams.dataset == "CIFAR100":
if split == 'train':
train = True
transform = self.train_set_transform
else:
train = False
transform = self.val_set_transform
dataset = CIFAR100(root=self.hparams.dataset_dir,
train=train,
transform=transform,
download=True)
elif self.hparams.dataset == 'STL10':
if split == 'train':
stlsplit = 'train'
transform = self.train_set_transform
else:
stlsplit = 'test'
transform = self.val_set_transform
dataset = STL10(root=self.hparams.dataset_dir,
split=stlsplit,
transform=transform,
download=True)
elif self.hparams.dataset == 'COVID':
if split == 'train':
covidsplit = 'train'
transform = self.train_set_transform
else:
covidsplit = 'test'
transform = self.val_set_transform
dataset = torchvision.datasets.ImageFolder(
root=self.hparams.dataset_dir + covidsplit,
transform=transform)
elif self.hparams.dataset == 'ImageNet100B' or self.hparams.dataset == 'imagenet-100B':
if split == 'train':
transform = self.train_set_transform
else:
split = 'val'
transform = self.val_set_transform
dataset = ImageNet100(root=self.hparams.dataset_dir,
split=split,
transform=transform)
elif self.hparams.dataset == 'COVID':
if split == 'train':
covidsplit = 'train'
transform = self.train_set_transform
else:
covidsplit = 'test'
transform = self.val_set_transform
dataset = torchvision.datasets.ImageFolder(
root=self.hparams.dataset_dir + covidsplit,
transform=transform)
elif self.hparams.dataset == 'ImageNet100B' or self.hparams.dataset == 'imagenet-100B':
if split == 'train':
transform = self.train_set_transform
else:
split = 'val'
transform = self.val_set_transform
dataset = ImageNet100(root=self.hparams.dataset_dir,
split=split,
transform=transform)
return dataset
def train_dataloader(self):
train_dataset = self._grab_dataset(split='train')
N_train = len(train_dataset)
if self.hparams.use_subset:
train_dataset = few_shot_dataset(
train_dataset,
int(
np.ceil(N_train * self.hparams.subset_ratio /
self.hparams.num_classes)))
train_dataloader = DataLoader(train_dataset, batch_size=self.hparams.batch_size, num_workers=self.hparams.workers,\
pin_memory=True, shuffle=True)
return train_dataloader
def val_dataloader(self):
val_dataset = self._grab_dataset(split='val')
N_val = len(val_dataset)
#SET SHUFFLE TO TRUE SINCE AUROC FREAKS OUT IF IT GETS AN ALL-1 OR ALL-0 BATCH
val_dataloader = DataLoader(val_dataset, batch_size=self.hparams.batch_size, num_workers=self.hparams.workers,\
pin_memory=True, shuffle=True)
return val_dataloader
def test_dataloader(self):
test_dataset = self._grab_dataset(split='test')
N_test = len(test_dataset)
#SET SHUFFLE TO TRUE SINCE AUROC FREAKS OUT IF IT GETS AN ALL-1 OR ALL-0 BATCH
test_dataloader = DataLoader(test_dataset, batch_size=self.hparams.batch_size, num_workers=self.hparams.workers,\
pin_memory=True, shuffle=True)
return test_dataloader
def training_step(self, batch, batch_idx):
x, y = batch
if batch_idx == 0 and self.current_epoch == 0:
self.logger.experiment.add_image('Train_Sample', img_grid(x),
self.current_epoch)
logits = self.forward(x)
loss = self.criterion(logits, y)
self.log("train_loss", loss, prog_bar=False, on_step=True, \
on_epoch=True, logger=True, sync_dist=True, sync_dist_op='sum')
return loss
def validation_step(self, batch, batch_idx):
x, y = batch
if batch_idx == 0 and self.current_epoch == 0:
self.logger.experiment.add_image('Val_Sample', img_grid(x),
self.current_epoch)
logits = self.forward(x)
pred_probs = logits.softmax(dim=-1) #(N, num_classes)
if self.hparams.dataset == 'COVID':
positive_prob = pred_probs[:,
0].flatten() #class 0 is INFECTED label
true_labels = y.flatten()
self.val_auc.update(positive_prob, true_labels)
self.log("val_auc", self.val_auc, prog_bar=False, logger=False)
self.log("val_top_1",
self.val_top_1(pred_probs, y),
prog_bar=False,
logger=False)
if self.hparams.dataset != 'COVID':
self.log("val_top_5",
self.val_top_5(pred_probs, y),
prog_bar=False,
logger=False)
def validation_epoch_end(self, outputs):
self.log("val_top_1",
self.val_top_1.compute(),
prog_bar=True,
logger=True)
if self.hparams.dataset != 'COVID':
self.log("val_top_5",
self.val_top_5.compute(),
prog_bar=True,
logger=True)
if self.hparams.dataset == 'COVID':
self.log("val_auc",
self.val_auc.compute(),
prog_bar=True,
logger=True)
self.val_auc.reset()
self.val_top_1.reset()
self.val_top_5.reset()
def test_step(self, batch, batch_idx):
x, y = batch
logits = self.forward(x)
pred_probs = logits.softmax(dim=-1)
if self.hparams.dataset == 'COVID':
positive_prob = pred_probs[:,
0].flatten() #class 0 is INFECTED label
true_labels = y.flatten()
self.test_auc.update(positive_prob, true_labels)
self.log("test_auc", self.test_auc, prog_bar=False, logger=False)
self.log("test_top_1",
self.test_top_1(pred_probs, y),
prog_bar=False,
logger=False)
if self.hparams.dataset != 'COVID':
self.log("test_top_5",
self.test_top_5(pred_probs, y),
prog_bar=False,
logger=False)
def test_epoch_end(self, outputs):
self.log("test_top_1",
self.test_top_1.compute(),
prog_bar=True,
logger=True)
if self.hparams.dataset != 'COVID':
self.log("test_top_5",
self.test_top_5.compute(),
prog_bar=True,
logger=True)
if self.hparams.dataset == 'COVID':
self.log("test_auc",
self.test_auc.compute(),
prog_bar=True,
logger=True)
self.test_auc.reset()
self.test_top_1.reset()
self.test_top_5.reset()