def fewshot_classification(
feature_extractor,
feature_extractor_optimizer,
classifier,
classifier_optimizer,
images_train,
labels_train,
labels_train_1hot,
images_test,
labels_test,
is_train,
base_ids=None,
feature_name=None,
classification_coef=1.0,
):
"""Forward-backward propagation routine of the few-shot classification task.
Given as input a mini-batch of few-shot episodes, it applies the
forward and (optionally) backward propagation routines of the few-shot
classification task. Each episode consists of (1) num_train_examples number
of training examples for the novel classes of the few-shot episode, (2) the
labels of training examples of the novel classes, (3) num_test_examples
number of test examples of the few-shot episode (note that the test
examples can be from both base and novel classes), and (4) the labels of the
test examples. Each mini-batch consists of meta_batch_size number of
few-shot episodes. The code assumes that the few-shot classification model
is divided into a feature extractor network and a classification head
network.
Args:
feature_extractor: The feature extractor neural network.
feature_extractor_optimizer: The parameter optimizer of the feature
extractor. If None, then the feature extractor remains frozen during
training.
classifier: The classification head applied on the output of the feature
extractor.
classifier_optimizer: The parameter optimizer of the classification head.
images_train: A 5D tensor with shape
[meta_batch_size x num_train_examples x channels x height x width] that
represents a mini-batch of meta_batch_size number of few-shot episodes,
each with num_train_examples number of training examples.
labels_train: A 2D tensor with shape
[meta_batch_size x num_train_examples] that represents the discrete
labels of the training examples of each few-shot episode in the batch.
labels_train_1hot: A 3D tensor with shape
[meta_batch_size x num_train_examples x num_novel] that represents
the 1hot labels of the training examples of the novel classes of each
few-shot episode in the batch. num_novel is the number of novel classes
per few-shot episode.
images_test: A 5D tensor with shape
[meta_batch_size x num_test_examples x channels x height x width] that
represents a mini-batch of meta_batch_size number of few-shot episodes,
each with num_test_examples number of test examples.
labels_test: A 2D tensor with shape
[meta_batch_size x num_test_examples] that represents the discrete
labels of the test examples of each few-shot episode in the mini-batch.
is_train: Boolean value that indicates if this mini-batch will be
used for training or testing. If is_train is False, then the code does
not apply the backward propagation step and does not update the
parameter optimizers.
base_ids: A 2D tensor with shape [meta_batch_size x num_base], where
base_ids[m] are the indices of the base categories that are being used
in the m-th few-shot episode. num_base is the number of base classes per
few-shot episode.
feature_name: (optional) A string or list of strings with the name of
feature level(s) from which the feature extractor will extract features
for the classification task.
classification_coef: (optional) the loss weight of the few-shot
classification task.
Returns:
record: A dictionary of scalar values with the following items:
'loss': The cross entropy loss of this mini-batch.
'AccuracyNovel': The classification accuracy of the test examples among
only the novel classes.
'AccuracyBase': (optinional) The classification accuracy of the test
examples among only the base classes. Applicable, only if there are
test examples from base classes in the mini-batch.
'AccuracyBase': (optinional) The classification accuracy of the test
examples among both the base and novel classes. Applicable, only if
there are test examples from base classes in the mini-batch.
"""
assert images_train.dim() == 5
assert images_test.dim() == 5
assert images_train.size(0) == images_test.size(0)
assert images_train.size(2) == images_test.size(2)
assert images_train.size(3) == images_test.size(3)
assert images_train.size(4) == images_test.size(4)
assert labels_train.dim() == 2
assert labels_test.dim() == 2
assert labels_train.size(0) == labels_test.size(0)
assert labels_train.size(0) == images_train.size(0)
assert not (isinstance(feature_name,
(list, tuple)) and len(feature_name) > 1)
meta_batch_size = images_train.size(0)
if is_train: # zero the gradients
if feature_extractor_optimizer:
feature_extractor_optimizer.zero_grad()
classifier_optimizer.zero_grad()
record = {}
with torch.no_grad():
images_train = utils.convert_from_5d_to_4d(images_train)
images_test = utils.convert_from_5d_to_4d(images_test)
labels_test = labels_test.view(-1)
batch_size_train = images_train.size(0)
# batch_size_test = images_test.size(0)
images = torch.cat([images_train, images_test], dim=0)
train_feature_extractor = is_train and (feature_extractor_optimizer
is not None)
with torch.set_grad_enabled(train_feature_extractor):
# Extract features from the train and test images.
features = cls_utils.extract_features(feature_extractor,
images,
feature_name=feature_name)
if not train_feature_extractor:
# Make sure that no gradients are backproagated to the feature
# extractor when the feature extraction model is freezed.
features = features.detach()
with torch.set_grad_enabled(is_train):
features_train = features[:batch_size_train]
features_test = features[batch_size_train:]
features_train = utils.add_dimension(features_train, meta_batch_size)
features_test = utils.add_dimension(features_test, meta_batch_size)
# Apply the classification head of the few-shot classification model.
classification_scores, loss = few_shot_feature_classification(
classifier,
features_test,
features_train,
labels_train_1hot,
labels_test,
base_ids,
)
record["loss"] = loss.item()
loss_total = loss * classification_coef
# *******************************************************************
with torch.no_grad():
num_base = base_ids.size(1) if (base_ids is not None) else 0
record = compute_accuracy_metrics(classification_scores, labels_test,
num_base, record)
if is_train:
loss_total.backward()
if feature_extractor_optimizer:
feature_extractor_optimizer.step()
classifier_optimizer.step()
return record
def object_classification_with_cluster_selfsupervision(
feature_extractor,
feature_extractor_optimizer,
classifier,
classifier_optimizer,
classifier_clu,
classifier_clu_optimizer,
images,
labels,
is_train,
alpha=1.0,
base_ids=None,
feature_name=None,
images_unlabeled=None,
images_unlabeled_label=None,
):
"""Forward-backward propagation routine for the classification model with
the auxiliary rotation prediction task.
Given as input a mini-batch of images with their labels, it applies the
forward and (optionaly) backward propagation routines of a classification
model extended to perfrom the auxiliary self-supervised rotation prediction
task. The rotatation prediction task can optionally be applied to an extra
mini-batch of only unlabeled images. The code assumes that the model is
divided into a feature extractor, a classification head, and a rotation
prediction head.
Args:
feature_extractor: The feature extractor neural network.
feature_extractor_optimizer: The parameter optimizer of the feature
extractor.
classifier: The classification head applied on the output of the feature
extractor.
classifier_optimizer: The parameter optimizer of the classification head.
images: A 4D tensor of shape [batch_size x channels x height x width] with
the mini-batch images. It is assumed that this tensor is already on the
same device as the feature extractor and classification head networks.
labels: A 1D tensor with shape [batch_size] with the image labels. It is
assumed that this tensor is already on the same device as the feature
extractor and classification head networks.
is_train: Boolean value that indicates if this mini-batch of images will be
used for training or testing. If is_train is False, then the code does
not apply the backward propagation step and does not update the
parameter optimizers.
alpha: The weight coeficient of the rotation prediction loss.
base_ids: Optional argument used in case of episodic training of few-shot
classification models. In this case, it is assumed that the total input
batch_size consists of meta_batch_size training episodes, each with
(batch_size // meta_batch_size) inner batch size (i.e., it must hold
that batch_size % meta_batch_size == 0). In this context, base_ids is a
2D tensor with shape [meta_batch_size x num_base], where base_ids[m] are
the indices of the base categories that are being used in the m-th
training episode.
feature_name: (optional) A string or list of strings with the name of
feature level(s) from which the feature extractor will extract features
for the classification task.
images_unlabeled: (optional) A 4D tensor of shape
[batch_size2 x channels x height x width] with a mini-batch of unlabeled
images. Only the rotation prediction task will be applied on those
images.
Returns:
record: A dictionary of scalar values with the following items:
'loss_cls': The cross entropy loss of the classification task.
'loss_rot': The rotation prediction loss.
'loss_total': The total loss, i.e., loss_cls + alpha * loss_rot.
'Accuracy': The top-1 classification accuracy.
'AccuracyRot': The rotation prediction accuracy.
"""
if base_ids is not None:
assert base_ids.size(0) == 1
assert images.dim() == 4
assert labels.dim() == 1
assert images.size(0) == labels.size(0)
if is_train:
# Zero gradients.
feature_extractor_optimizer.zero_grad()
classifier_optimizer.zero_grad()
classifier_clu_optimizer.zero_grad()
batch_size_in = images.size(0)
batch_size_classification = labels.size(0)
record = {}
with torch.set_grad_enabled(is_train):
# Extract features from the images.
features = cls_utils.extract_features(feature_extractor,
images,
feature_name=feature_name)
# Perform the object classification task. From all the images only the
# top 'batch_size_classification' are used for this task.
features_cls = features[:batch_size_classification].contiguous()
scores_classification, loss_classsification = cls_utils.classification_task(
classifier, features_cls, labels, base_ids)
record["loss_cls"] = loss_classsification.item()
# Perform the self-supervised cluster prediction task.
features_unlabeled = cls_utils.extract_features(
feature_extractor, images_unlabeled, feature_name=feature_name)
scores_cluster, loss_cluster = cluster_task(classifier_clu,
features_unlabeled,
images_unlabeled_label)
record["loss_clu"] = loss_cluster.item()
# Compute total loss.
loss_total = loss_classsification + alpha * loss_cluster
record["loss_total"] = loss_total.item()
with torch.no_grad():
# Compute accuracies.
record["Accuracy"] = utils.top1accuracy(scores_classification, labels)
record["AccuracyRot"] = utils.top1accuracy(scores_cluster,
images_unlabeled_label)
if is_train:
# Backward loss and apply gradient steps.
loss_total.backward()
feature_extractor_optimizer.step()
classifier_optimizer.step()
classifier_clu_optimizer.step()
return record
def save_features(self,
dataloader,
filename,
feature_name=None,
global_pooling=True):
"""Saves features and labels for each image in the dataloader.
This routines uses the trained feature model (i.e.,
self.networks['feature_extractor']) in order to extract a feature for each
image in the dataloader. The extracted features along with the labels
of the images that they come from are saved in a h5py file.
Args:
dataloader: A dataloader that feeds images and labels.
filename: The file name where the features and the labels of each
images in the dataloader are saved.
feature_name:
"""
if isinstance(feature_name, (list, tuple)):
assert len(feature_name) == 1
feature_extractor = self.networks["feature_extractor"]
feature_extractor.eval()
self.dloader = dataloader
dataloader_iterator = dataloader.get_iterator()
self.logger.info(f"Destination filename for features: {filename}")
data_file = h5py.File(filename, "w")
max_count = len(dataloader_iterator) * dataloader_iterator.batch_size
all_labels = data_file.create_dataset("all_labels", (max_count, ),
dtype="i")
all_features = None
count = 0
for i, batch in enumerate(tqdm(dataloader_iterator)):
with torch.no_grad():
self.set_tensors(batch)
images = self.tensors["images"].detach()
labels = self.tensors["labels"].detach()
assert images.dim() == 4
assert labels.dim() == 1
features = cls_utils.extract_features(
feature_extractor, images, feature_name=feature_name)
if global_pooling and features.dim() == 4:
features = tools.global_pooling(features, pool_type="avg")
features = features.view(features.size(0), -1)
assert features.dim() == 2
if all_features is None:
self.logger.info("Image size: {}".format(images.size()))
self.logger.info("Feature size: {}".format(
features.size()))
self.logger.info(f"Max_count: {max_count}")
all_features = data_file.create_dataset(
"all_features", (max_count, features.size(1)),
dtype="f")
self.logger.info("Number of feature channels: {}".format(
features.size(1)))
all_features[count:(
count + features.size(0)), :] = features.cpu().numpy()
all_labels[count:(count +
features.size(0))] = labels.cpu().numpy()
count = count + features.size(0)
self.logger.info(f"Number of processed primages: {count}")
count_var = data_file.create_dataset("count", (1, ), dtype="i")
count_var[0] = count
data_file.close()
def fewshot_classification_with_rotation_selfsupervision(
feature_extractor,
feature_extractor_optimizer,
classifier,
classifier_optimizer,
classifier_rot,
classifier_rot_optimizer,
images_train,
labels_train,
labels_train_1hot,
images_test,
labels_test,
is_train,
alpha=1.0,
base_ids=None,
feature_name=None,
):
"""Forward-backward routine of a few-shot model with auxiliary rotation
prediction task.
Given as input a mini-batch of few-shot episodes, it applies the
forward and (optionally) backward propagation routines of the few-shot
classification task. Each episode consists of (1) num_train_examples number
of training examples for the novel classes of the few-shot episode, (2) the
labels of training examples of the novel classes, (3) num_test_examples
number of test examples of the few-shot episode (note that the test
examples can be from both base and novel classes), and (4) the labels of the
test examples. Each mini-batch consists of meta_batch_size number of
few-shot episodes. The code assumes that the few-shot classification model
is divided into a feature extractor network and a classification head
network. Also, the code applies the auxiliary self-supervised task of
predicting image rotations. The rotation prediction task is applied on both
the test and train examples of the few-shot episodes.
Args:
feature_extractor: The feature extractor neural network.
feature_extractor_optimizer: The parameter optimizer of the feature
extractor. If None, then the feature extractor remains frozen during
training.
classifier: The classification head applied on the output of the feature
extractor.
classifier_optimizer: The parameter optimizer of the classification head.
classifier_rot: The rotation prediction head applied on the output of the
feature extractor.
classifier_rot_optimizer: The parameter optimizer of the rotation prediction
head.
images_train: A 5D tensor with shape
[meta_batch_size x num_train_examples x channels x height x width] that
represents a mini-batch of meta_batch_size number of few-shot episodes,
each with num_train_examples number of training examples.
labels_train: A 2D tensor with shape
[meta_batch_size x num_train_examples] that represents the discrete
labels of the training examples of each few-shot episode in the batch.
labels_train_1hot: A 3D tensor with shape
[meta_batch_size x num_train_examples x num_novel] that represents
the 1hot labels of the training examples of the novel classes of each
few-shot episode in the batch. num_novel is the number of novel classes
per few-shot episode.
images_test: A 5D tensor with shape
[meta_batch_size x num_test_examples x channels x height x width] that
represents a mini-batch of meta_batch_size number of few-shot episodes,
each with num_test_examples number of test examples.
labels_test: A 2D tensor with shape
[meta_batch_size x num_test_examples] that represents the discrete
labels of the test examples of each few-shot episode in the mini-batch.
is_train: Boolean value that indicates if this mini-batch will be
used for training or testing. If is_train is False, then the code does
not apply the backward propagation step and does not update the
parameter optimizers.
base_ids: A 2D tensor with shape [meta_batch_size x num_base], where
base_ids[m] are the indices of the base categories that are being used
in the m-th few-shot episode. num_base is the number of base classes per
few-shot episode.
alpha: (optional) The loss weight of the rotation prediction task.
feature_name: (optional) A string or list of strings with the name of
feature level(s) from which the feature extractor will extract features
for the classification task.
Returns:
record: A dictionary of scalar values with the following items:
'loss_cls': The cross entropy loss of the few-shot classification task.
'loss_rot': The rotation prediction loss.
'loss_total': The total loss, i.e., loss_cls + alpha * loss_rot.
'AccuracyNovel': The classification accuracy of the test examples among
only the novel classes.
'AccuracyBase': (optinional) The classification accuracy of the test
examples among only the base classes. Applicable, only if there are
test examples from base classes in the mini-batch.
'AccuracyBase': (optinional) The classification accuracy of the test
examples among both the base and novel classes. Applicable, only if
there are test examples from base classes in the mini-batch.
'AccuracyRot': The accuracy of the rotation prediction task.
"""
pdb.set_trace()
assert images_train.dim() == 5
assert images_test.dim() == 5
assert images_train.size(0) == images_test.size(0)
assert images_train.size(2) == images_test.size(2)
assert images_train.size(3) == images_test.size(3)
assert images_train.size(4) == images_test.size(4)
assert labels_train.dim() == 2
assert labels_test.dim() == 2
assert labels_train.size(0) == labels_test.size(0)
assert labels_train.size(0) == images_train.size(0)
meta_batch_size = images_train.size(0)
num_train = images_train.size(1)
num_test = images_test.size(1)
if is_train: # zero the gradients
feature_extractor_optimizer.zero_grad()
classifier_optimizer.zero_grad()
classifier_rot_optimizer.zero_grad()
record = {}
with torch.no_grad():
images_train = utils.convert_from_5d_to_4d(images_train)
images_test = utils.convert_from_5d_to_4d(images_test)
labels_test = labels_test.view(-1)
images = torch.cat([images_train, images_test], dim=0)
batch_size_train = images_train.size(0)
batch_size_train_test = images.size(0)
assert batch_size_train == meta_batch_size * num_train
assert batch_size_train_test == meta_batch_size * (num_train +
num_test)
# Create the 4 rotated version of the images; this step increases
# the batch size by a multiple of 4.
images = create_4rotations_images(images)
labels_rotation = create_rotations_labels(batch_size_train_test,
images.device)
with torch.set_grad_enabled(is_train):
# Extract features from the train and test images.
features = cls_utils.extract_features(feature_extractor,
images,
feature_name=feature_name)
# Apply the few-shot classification head.
features_train = features[:batch_size_train]
features_test = features[batch_size_train:batch_size_train_test]
features_train = utils.add_dimension(features_train, meta_batch_size)
features_test = utils.add_dimension(features_test, meta_batch_size)
(
classification_scores,
loss_classsification,
) = fewshot_utils.few_shot_feature_classification(
classifier,
features_test,
features_train,
labels_train_1hot,
labels_test,
base_ids,
)
record["loss_cls"] = loss_classsification.item()
# Apply the rotation prediction head.
scores_rotation, loss_rotation = rotation_task(classifier_rot,
features,
labels_rotation)
record["loss_rot"] = loss_rotation.item()
# Compute total loss.
loss_total = loss_classsification + alpha * loss_rotation
record["loss_total"] = loss_total.item()
with torch.no_grad():
num_base = base_ids.size(1) if (base_ids is not None) else 0
record = fewshot_utils.compute_accuracy_metrics(
classification_scores, labels_test, num_base, record)
record["AccuracyRot"] = utils.top1accuracy(scores_rotation,
labels_rotation)
if is_train:
loss_total.backward()
feature_extractor_optimizer.step()
classifier_optimizer.step()
classifier_rot_optimizer.step()
return record
def object_classification_with_rot_loc_jig_clu_selfsupervision(
feature_extractor,
feature_extractor_optimizer,
classifier,
classifier_optimizer,
#att_classifier,
#att_classifier_optimizer,
#lamda,
rot_classifier,
rot_classifier_optimizer,
location_classifier,
location_classifier_optimizer,
jig_classifier,
jig_classifier_optimizer,
clu_classifier,
clu_classifier_optimizer,
patch_classifier,
patch_classifier_optimizer,
images,
labels,
patches,
labels_patches,
is_train,
rotation_loss_coef=1.0,
patch_location_loss_coef=1.0,
patch_classification_loss_coef=1.0,
cluster_loss_coef=1.0,
random_rotation=False,
rotation_invariant_classifier=False,
combine="average",
base_ids=None,
standardize_patches=True,
images_unlabeled=None,
images_unlabeled_label=None):
"""Forward-backward propagation routine for classification model extended
with the auxiliary self-supervised task of predicting the relative location
of patches."""
if base_ids is not None:
assert base_ids.size(0) == 1
assert images.dim() == 4
assert labels.dim() == 1
assert images.size(0) == labels.size(0)
#assert att_classifier != None
#assert att_classifier_optimizer != None
assert patches.dim() == 5 and patches.size(1) == 9
assert patches.size(0) == labels_patches.size(0)
patches = utils.convert_from_5d_to_4d(patches)
if standardize_patches:
patches = utils.standardize_image(patches)
if is_train:
# Zero gradients.
feature_extractor_optimizer.zero_grad()
classifier_optimizer.zero_grad()
#if att_classifier_optimizer != None:
# att_classifier_optimizer.zero_grad()
if rotation_loss_coef > 0.0:
rot_classifier_optimizer.zero_grad()
if patch_location_loss_coef > 0.0:
location_classifier_optimizer.zero_grad()
jig_classifier_optimizer.zero_grad()
if cluster_loss_coef > 0.0:
clu_classifier_optimizer.zero_grad()
if patch_classification_loss_coef > 0.0:
patch_classifier_optimizer.zero_grad()
with torch.no_grad():
images, labels, labels_rotation = preprocess_input_data(
images, labels, None, random_rotation,
rotation_invariant_classifier)
record = {}
with torch.set_grad_enabled(is_train):
# Extract features from the images.
features_images = feature_extractor(images)
# Extract features from the image patches.
features_patches = feature_extractor(patches)
#pdb.set_trace()
#pdb.set_trace()
# Perform object classification task.
scores_classification, loss_classsification = cls_utils.classification_task(
classifier, features_images, labels, base_ids)
record["loss_cls"] = loss_classsification.item()
loss_total = loss_classsification
#attention
#lamda_rot = lamda[0] / torch.sum(lamda)
#lamda_loc = lamda[1] / torch.sum(lamda)
#lamda_jig = lamda[2] / torch.sum(lamda)
#lamda_clu = lamda[3] / torch.sum(lamda)
#lamda_rot,lamda_loc, lamda_jig,lamda_clu = att_classifier(features_images).mean(0)
#record["lamda_rot"] = lamda_rot.item()
#record["lamda_loc"] = lamda_loc.item()
#record["lamda_jig"] = lamda_jig.item()
#record["lamda_clu"] = lamda_clu.item()
#print(lamda)
if rotation_loss_coef > 0.0:
scores_rotation, loss_rotation = rotation_task(
rot_classifier, features_images, labels_rotation)
record["loss_rot"] = loss_rotation.item()
loss_total = loss_total + loss_rotation
if patch_location_loss_coef > 0.0:
# patch location prediction.
scores_location, loss_location, labels_loc = patch_location_task(
location_classifier, features_patches)
record["loss_loc"] = loss_location.item()
loss_total = loss_total + loss_location
# patch puzzle prediction.
scores_puzzle, loss_puzzle, labels_puzzle = jigsaw_puzzle_task(
jig_classifier, features_patches)
record["loss_jig"] = loss_puzzle.item()
loss_total = loss_total + loss_puzzle
if cluster_loss_coef > 0.0:
#cluster prediction.
features_unlabeled = cls_utils.extract_features(
feature_extractor, images_unlabeled)
scores_cluster, loss_cluster = cluster_task(
clu_classifier, features_unlabeled, images_unlabeled_label)
record["loss_clu"] = loss_cluster.item()
loss_total = loss_total + loss_cluster
#pdb.set_trace()
# Perform the auxiliary task of classifying individual patches.
if patch_classification_loss_coef > 0.0:
scores_patch, loss_patch = patch_classification_task(
patch_classifier, features_patches, labels_patches, combine)
record["loss_patch_cls"] = loss_patch.item()
loss_total = loss_total + loss_patch * patch_classification_loss_coef
# Because the total loss consists of multiple individual losses
# (i.e., 3) scale it down by a factor of 0.5.
loss_total = loss_total * 0.5
with torch.no_grad():
# Compute accuracies.
record["Accuracy"] = utils.top1accuracy(scores_classification, labels)
if rotation_loss_coef > 0.0:
record["AccuracyRot"] = utils.top1accuracy(scores_rotation,
labels_rotation)
if patch_location_loss_coef > 0.0:
record["AccuracyLoc"] = utils.top1accuracy(scores_location,
labels_loc)
record["AccuracyJig"] = utils.top1accuracy(scores_puzzle,
labels_puzzle)
if cluster_loss_coef > 0.0:
record["AccuracyClu"] = utils.top1accuracy(scores_cluster,
images_unlabeled_label)
if patch_classification_loss_coef > 0.0:
record["AccuracyPatch"] = utils.top1accuracy(
scores_patch, labels_patches)
if is_train:
# Backward loss and apply gradient steps.
loss_total.backward()
feature_extractor_optimizer.step()
classifier_optimizer.step()
#att_classifier_optimizer.step()
if rotation_loss_coef > 0.0:
rot_classifier_optimizer.step()
if patch_location_loss_coef > 0.0:
location_classifier_optimizer.step()
jig_classifier_optimizer.step()
if cluster_loss_coef > 0.0:
clu_classifier_optimizer.step()
if patch_classification_loss_coef > 0.0:
patch_classifier_optimizer.step()
return record