def compute_accuracy_metrics(scores, labels, num_base, record={}, string_id=""): """Computes the classification accuracies of a mini-batch of episodes.""" assert isinstance(record, dict) if string_id != "": string_id = "_" + string_id if labels.dim() > 1: labels = labels.view(scores.size(0)) if num_base > 0: record["AccuracyBoth" + string_id] = utils.top1accuracy(scores, labels) base_indices = torch.nonzero(labels < num_base).view(-1) novel_indices = torch.nonzero(labels >= num_base).view(-1) if base_indices.dim() != 0 and base_indices.size(0) > 0: scores_base = scores[base_indices][:, :num_base] labels_base = labels[base_indices] record["AccuracyBase" + string_id] = utils.top1accuracy( scores_base, labels_base) scores_novel = scores[novel_indices, :][:, num_base:] labels_novel = labels[novel_indices] - num_base record["AccuracyNovel" + string_id] = utils.top1accuracy( scores_novel, labels_novel) else: record["AccuracyNovel" + string_id] = utils.top1accuracy( scores, labels) return record
def object_classification( feature_extractor, feature_extractor_optimizer, classifier, classifier_optimizer, images, labels, is_train, base_ids=None, feature_name=None, ): """Forward-backward propagation routine for the classification task. Given as input a mini-batch of images with their labels, it applies the forward and (optionaly) backward propagation routines of the classification task. The code assumes that the classification model is divided into a feature extractor network and a classification head network. Args: feature_extractor: The feature extractor neural network. feature_extractor_optimizer: The parameter optimizer of the feature extractor. If None, then the feature extractor remains frozen during training. classifier: The classification head applied on the output of the feature extractor. classifier_optimizer: The parameter optimizer of the classification head. images: A 4D tensor of shape [batch_size x channels x height x width] with the mini-batch images. It is assumed that this tensor is already on the same device as the feature extractor and classification head networks. labels: A 1D tensor with shape [batch_size] with the image labels. It is assumed that this tensor is already on the same device as the feature extractor and classification head networks. is_train: Boolean value that indicates if this mini-batch of images will be used for training or testing. If is_train is False, then the code does not apply the backward propagation step and does not update the parameter optimizers. base_ids: Optional argument used in case of episodic training of few-shot classification models. In this case, it is assumed that the total input batch_size consists of meta_batch_size training episodes, each with (batch_size // meta_batch_size) inner batch size (i.e., it must hold that batch_size % meta_batch_size == 0). In this context, base_ids is a 2D tensor with shape [meta_batch_size x num_base], where base_ids[m] are the indices of the base categories that are being used in the m-th training episode. feature_name: (optional) A string or list of strings with the name of feature level(s) from which the feature extractor will extract features for the classification task. Returns: record: A dictionary of scalar values with the following items: 'loss': The cross entropy loss of this mini-batch of images. 'AccuracyTop1': The top-1 classification accuracy. """ assert not (isinstance(feature_name, (list, tuple)) and len(feature_name) > 1) assert labels.dim() == 1 assert images.size(0) == labels.size(0) if is_train: # Zero gradients. if feature_extractor_optimizer: feature_extractor_optimizer.zero_grad() classifier_optimizer.zero_grad() record = {} train_feature_extractor = is_train and (feature_extractor_optimizer is not None) with torch.set_grad_enabled(train_feature_extractor): # Extract features from the images. features = extract_features(feature_extractor, images, feature_name=feature_name) if not train_feature_extractor: # Make sure that no gradients are backproagated to the feature # extractor when the feature extraction model is freezed. features = features.detach() with torch.set_grad_enabled(is_train): # Perform the object classification task. scores_classification, loss_classsification = classification_task( classifier, features, labels, base_ids) loss_total = loss_classsification record["loss"] = loss_total.item() with torch.no_grad(): # Compute accuracies. record["AccuracyTop1"] = utils.top1accuracy(scores_classification, labels) if is_train: # Backward loss and apply gradient steps. loss_total.backward() if feature_extractor_optimizer: feature_extractor_optimizer.step() classifier_optimizer.step() return record
def object_classification_with_cluster_selfsupervision( feature_extractor, feature_extractor_optimizer, classifier, classifier_optimizer, classifier_clu, classifier_clu_optimizer, images, labels, is_train, alpha=1.0, base_ids=None, feature_name=None, images_unlabeled=None, images_unlabeled_label=None, ): """Forward-backward propagation routine for the classification model with the auxiliary rotation prediction task. Given as input a mini-batch of images with their labels, it applies the forward and (optionaly) backward propagation routines of a classification model extended to perfrom the auxiliary self-supervised rotation prediction task. The rotatation prediction task can optionally be applied to an extra mini-batch of only unlabeled images. The code assumes that the model is divided into a feature extractor, a classification head, and a rotation prediction head. Args: feature_extractor: The feature extractor neural network. feature_extractor_optimizer: The parameter optimizer of the feature extractor. classifier: The classification head applied on the output of the feature extractor. classifier_optimizer: The parameter optimizer of the classification head. images: A 4D tensor of shape [batch_size x channels x height x width] with the mini-batch images. It is assumed that this tensor is already on the same device as the feature extractor and classification head networks. labels: A 1D tensor with shape [batch_size] with the image labels. It is assumed that this tensor is already on the same device as the feature extractor and classification head networks. is_train: Boolean value that indicates if this mini-batch of images will be used for training or testing. If is_train is False, then the code does not apply the backward propagation step and does not update the parameter optimizers. alpha: The weight coeficient of the rotation prediction loss. base_ids: Optional argument used in case of episodic training of few-shot classification models. In this case, it is assumed that the total input batch_size consists of meta_batch_size training episodes, each with (batch_size // meta_batch_size) inner batch size (i.e., it must hold that batch_size % meta_batch_size == 0). In this context, base_ids is a 2D tensor with shape [meta_batch_size x num_base], where base_ids[m] are the indices of the base categories that are being used in the m-th training episode. feature_name: (optional) A string or list of strings with the name of feature level(s) from which the feature extractor will extract features for the classification task. images_unlabeled: (optional) A 4D tensor of shape [batch_size2 x channels x height x width] with a mini-batch of unlabeled images. Only the rotation prediction task will be applied on those images. Returns: record: A dictionary of scalar values with the following items: 'loss_cls': The cross entropy loss of the classification task. 'loss_rot': The rotation prediction loss. 'loss_total': The total loss, i.e., loss_cls + alpha * loss_rot. 'Accuracy': The top-1 classification accuracy. 'AccuracyRot': The rotation prediction accuracy. """ if base_ids is not None: assert base_ids.size(0) == 1 assert images.dim() == 4 assert labels.dim() == 1 assert images.size(0) == labels.size(0) if is_train: # Zero gradients. feature_extractor_optimizer.zero_grad() classifier_optimizer.zero_grad() classifier_clu_optimizer.zero_grad() batch_size_in = images.size(0) batch_size_classification = labels.size(0) record = {} with torch.set_grad_enabled(is_train): # Extract features from the images. features = cls_utils.extract_features(feature_extractor, images, feature_name=feature_name) # Perform the object classification task. From all the images only the # top 'batch_size_classification' are used for this task. features_cls = features[:batch_size_classification].contiguous() scores_classification, loss_classsification = cls_utils.classification_task( classifier, features_cls, labels, base_ids) record["loss_cls"] = loss_classsification.item() # Perform the self-supervised cluster prediction task. features_unlabeled = cls_utils.extract_features( feature_extractor, images_unlabeled, feature_name=feature_name) scores_cluster, loss_cluster = cluster_task(classifier_clu, features_unlabeled, images_unlabeled_label) record["loss_clu"] = loss_cluster.item() # Compute total loss. loss_total = loss_classsification + alpha * loss_cluster record["loss_total"] = loss_total.item() with torch.no_grad(): # Compute accuracies. record["Accuracy"] = utils.top1accuracy(scores_classification, labels) record["AccuracyRot"] = utils.top1accuracy(scores_cluster, images_unlabeled_label) if is_train: # Backward loss and apply gradient steps. loss_total.backward() feature_extractor_optimizer.step() classifier_optimizer.step() classifier_clu_optimizer.step() return record
def fewshot_classification_with_rotation_selfsupervision( feature_extractor, feature_extractor_optimizer, classifier, classifier_optimizer, classifier_rot, classifier_rot_optimizer, images_train, labels_train, labels_train_1hot, images_test, labels_test, is_train, alpha=1.0, base_ids=None, feature_name=None, ): """Forward-backward routine of a few-shot model with auxiliary rotation prediction task. Given as input a mini-batch of few-shot episodes, it applies the forward and (optionally) backward propagation routines of the few-shot classification task. Each episode consists of (1) num_train_examples number of training examples for the novel classes of the few-shot episode, (2) the labels of training examples of the novel classes, (3) num_test_examples number of test examples of the few-shot episode (note that the test examples can be from both base and novel classes), and (4) the labels of the test examples. Each mini-batch consists of meta_batch_size number of few-shot episodes. The code assumes that the few-shot classification model is divided into a feature extractor network and a classification head network. Also, the code applies the auxiliary self-supervised task of predicting image rotations. The rotation prediction task is applied on both the test and train examples of the few-shot episodes. Args: feature_extractor: The feature extractor neural network. feature_extractor_optimizer: The parameter optimizer of the feature extractor. If None, then the feature extractor remains frozen during training. classifier: The classification head applied on the output of the feature extractor. classifier_optimizer: The parameter optimizer of the classification head. classifier_rot: The rotation prediction head applied on the output of the feature extractor. classifier_rot_optimizer: The parameter optimizer of the rotation prediction head. images_train: A 5D tensor with shape [meta_batch_size x num_train_examples x channels x height x width] that represents a mini-batch of meta_batch_size number of few-shot episodes, each with num_train_examples number of training examples. labels_train: A 2D tensor with shape [meta_batch_size x num_train_examples] that represents the discrete labels of the training examples of each few-shot episode in the batch. labels_train_1hot: A 3D tensor with shape [meta_batch_size x num_train_examples x num_novel] that represents the 1hot labels of the training examples of the novel classes of each few-shot episode in the batch. num_novel is the number of novel classes per few-shot episode. images_test: A 5D tensor with shape [meta_batch_size x num_test_examples x channels x height x width] that represents a mini-batch of meta_batch_size number of few-shot episodes, each with num_test_examples number of test examples. labels_test: A 2D tensor with shape [meta_batch_size x num_test_examples] that represents the discrete labels of the test examples of each few-shot episode in the mini-batch. is_train: Boolean value that indicates if this mini-batch will be used for training or testing. If is_train is False, then the code does not apply the backward propagation step and does not update the parameter optimizers. base_ids: A 2D tensor with shape [meta_batch_size x num_base], where base_ids[m] are the indices of the base categories that are being used in the m-th few-shot episode. num_base is the number of base classes per few-shot episode. alpha: (optional) The loss weight of the rotation prediction task. feature_name: (optional) A string or list of strings with the name of feature level(s) from which the feature extractor will extract features for the classification task. Returns: record: A dictionary of scalar values with the following items: 'loss_cls': The cross entropy loss of the few-shot classification task. 'loss_rot': The rotation prediction loss. 'loss_total': The total loss, i.e., loss_cls + alpha * loss_rot. 'AccuracyNovel': The classification accuracy of the test examples among only the novel classes. 'AccuracyBase': (optinional) The classification accuracy of the test examples among only the base classes. Applicable, only if there are test examples from base classes in the mini-batch. 'AccuracyBase': (optinional) The classification accuracy of the test examples among both the base and novel classes. Applicable, only if there are test examples from base classes in the mini-batch. 'AccuracyRot': The accuracy of the rotation prediction task. """ pdb.set_trace() assert images_train.dim() == 5 assert images_test.dim() == 5 assert images_train.size(0) == images_test.size(0) assert images_train.size(2) == images_test.size(2) assert images_train.size(3) == images_test.size(3) assert images_train.size(4) == images_test.size(4) assert labels_train.dim() == 2 assert labels_test.dim() == 2 assert labels_train.size(0) == labels_test.size(0) assert labels_train.size(0) == images_train.size(0) meta_batch_size = images_train.size(0) num_train = images_train.size(1) num_test = images_test.size(1) if is_train: # zero the gradients feature_extractor_optimizer.zero_grad() classifier_optimizer.zero_grad() classifier_rot_optimizer.zero_grad() record = {} with torch.no_grad(): images_train = utils.convert_from_5d_to_4d(images_train) images_test = utils.convert_from_5d_to_4d(images_test) labels_test = labels_test.view(-1) images = torch.cat([images_train, images_test], dim=0) batch_size_train = images_train.size(0) batch_size_train_test = images.size(0) assert batch_size_train == meta_batch_size * num_train assert batch_size_train_test == meta_batch_size * (num_train + num_test) # Create the 4 rotated version of the images; this step increases # the batch size by a multiple of 4. images = create_4rotations_images(images) labels_rotation = create_rotations_labels(batch_size_train_test, images.device) with torch.set_grad_enabled(is_train): # Extract features from the train and test images. features = cls_utils.extract_features(feature_extractor, images, feature_name=feature_name) # Apply the few-shot classification head. features_train = features[:batch_size_train] features_test = features[batch_size_train:batch_size_train_test] features_train = utils.add_dimension(features_train, meta_batch_size) features_test = utils.add_dimension(features_test, meta_batch_size) ( classification_scores, loss_classsification, ) = fewshot_utils.few_shot_feature_classification( classifier, features_test, features_train, labels_train_1hot, labels_test, base_ids, ) record["loss_cls"] = loss_classsification.item() # Apply the rotation prediction head. scores_rotation, loss_rotation = rotation_task(classifier_rot, features, labels_rotation) record["loss_rot"] = loss_rotation.item() # Compute total loss. loss_total = loss_classsification + alpha * loss_rotation record["loss_total"] = loss_total.item() with torch.no_grad(): num_base = base_ids.size(1) if (base_ids is not None) else 0 record = fewshot_utils.compute_accuracy_metrics( classification_scores, labels_test, num_base, record) record["AccuracyRot"] = utils.top1accuracy(scores_rotation, labels_rotation) if is_train: loss_total.backward() feature_extractor_optimizer.step() classifier_optimizer.step() classifier_rot_optimizer.step() return record
def fewshot_classification_with_jigsaw_puzzle__selfsupervision( feature_extractor, feature_extractor_optimizer, classifier, classifier_optimizer, location_classifier, location_classifier_optimizer, patch_classifier, patch_classifier_optimizer, images_train, patches_train, labels_train, labels_train_1hot, images_test, patches_test, labels_test, is_train, base_ids=None, patch_location_loss_coef=1.0, patch_classification_loss_coef=1.0, combine="average", standardize_patches=True, ): """Forward-backward propagation routine for few-shot model extended with the auxiliary self-supervised task of predicting the relative location of patches.""" pdb.set_trace() assert images_train.dim() == 5 assert images_test.dim() == 5 assert images_train.size(0) == images_test.size(0) assert images_train.size(2) == images_test.size(2) assert images_train.size(3) == images_test.size(3) assert images_train.size(4) == images_test.size(4) assert labels_train.dim() == 2 assert labels_test.dim() == 2 assert labels_train.size(0) == labels_test.size(0) assert labels_train.size(0) == images_train.size(0) assert patches_train.dim() == 6 assert patches_train.size(0) == images_train.size(0) assert patches_train.size(1) == images_train.size(1) assert patches_train.size(2) == 9 assert patches_test.dim() == 6 assert patches_test.size(0) == images_test.size(0) assert patches_test.size(1) == images_test.size(1) assert patches_test.size(2) == 9 meta_batch_size = images_train.size(0) num_train = images_train.size(1) num_test = images_test.size(1) if is_train: # Zero gradients. feature_extractor_optimizer.zero_grad() classifier_optimizer.zero_grad() if patch_location_loss_coef > 0.0: location_classifier_optimizer.zero_grad() if patch_classification_loss_coef > 0.0: patch_classifier_optimizer.zero_grad() record = {} with torch.no_grad(): images_train = utils.convert_from_5d_to_4d(images_train) images_test = utils.convert_from_5d_to_4d(images_test) labels_test = labels_test.view(-1) images = torch.cat([images_train, images_test], dim=0) batch_size_train = images_train.size(0) batch_size_train_test = images.size(0) assert batch_size_train == meta_batch_size * num_train assert batch_size_train_test == meta_batch_size * (num_train + num_test) patches_train = utils.convert_from_6d_to_4d(patches_train) patches_test = utils.convert_from_6d_to_4d(patches_test) if standardize_patches: patches_train = utils.standardize_image(patches_train) patches_test = utils.standardize_image(patches_test) patches = torch.cat([patches_train, patches_test], dim=0) assert patches_train.size(0) == batch_size_train * 9 assert patches.size(0) == batch_size_train_test * 9 with torch.set_grad_enabled(is_train): # Extract features from the images. features = feature_extractor(images) # Extract features from the image patches. features_patches = feature_extractor(patches) # Perform object classification task. features_train = features[:batch_size_train] features_test = features[batch_size_train:batch_size_train_test] features_train = utils.add_dimension(features_train, meta_batch_size) features_test = utils.add_dimension(features_test, meta_batch_size) ( classification_scores, loss_classsification, ) = fewshot_utils.few_shot_feature_classification( classifier, features_test, features_train, labels_train_1hot, labels_test, base_ids, ) record["loss_cls"] = loss_classsification.item() loss_total = loss_classsification # Perform the self-supervised task of relative patch locatioon # prediction. if patch_location_loss_coef > 0.0: scores_location, loss_location, labels_loc = patch_location_task( location_classifier, features_patches) record["loss_loc"] = loss_location.item() loss_total = loss_total + loss_location * patch_location_loss_coef # Perform the auxiliary task of classifying patches. if patch_classification_loss_coef > 0.0: pdb.set_trace() features_patches = add_patch_dimension(features_patches) pdb.set_trace() assert features_patches.size(0) == batch_size_train_test assert features_patches.size(1) == 9 features_patches = combine_multiple_patch_features( features_patches, combine) pdb.set_trace() features_patches_train = utils.add_dimension( features_patches[:batch_size_train], meta_batch_size) features_patches_test = utils.add_dimension( features_patches[batch_size_train:batch_size_train_test], meta_batch_size) scores_patch, loss_patch = fewshot_utils.few_shot_feature_classification( patch_classifier, features_patches_test, features_patches_train, labels_train_1hot, labels_test, base_ids, ) record["loss_patch_cls"] = loss_patch.item() loss_total = loss_total + loss_patch * patch_classification_loss_coef # Because the total loss consists of multiple individual losses # (i.e., 3) scale it down by a factor of 0.5. loss_total = loss_total * 0.5 with torch.no_grad(): num_base = base_ids.size(1) if (base_ids is not None) else 0 record = fewshot_utils.compute_accuracy_metrics( classification_scores, labels_test, num_base, record) if patch_location_loss_coef > 0.0: record["AccuracyLoc"] = utils.top1accuracy(scores_location, labels_loc) if patch_classification_loss_coef > 0.0: record = fewshot_utils.compute_accuracy_metrics(scores_patch, labels_test, num_base, record, string_id="Patch") if is_train: # Backward loss and apply gradient steps. loss_total.backward() feature_extractor_optimizer.step() classifier_optimizer.step() if patch_location_loss_coef > 0.0: location_classifier_optimizer.step() if patch_classification_loss_coef > 0.0: patch_classifier_optimizer.step() return record
def object_classification_with_rot_loc_jig_clu_selfsupervision( feature_extractor, feature_extractor_optimizer, classifier, classifier_optimizer, #att_classifier, #att_classifier_optimizer, #lamda, rot_classifier, rot_classifier_optimizer, location_classifier, location_classifier_optimizer, jig_classifier, jig_classifier_optimizer, clu_classifier, clu_classifier_optimizer, patch_classifier, patch_classifier_optimizer, images, labels, patches, labels_patches, is_train, rotation_loss_coef=1.0, patch_location_loss_coef=1.0, patch_classification_loss_coef=1.0, cluster_loss_coef=1.0, random_rotation=False, rotation_invariant_classifier=False, combine="average", base_ids=None, standardize_patches=True, images_unlabeled=None, images_unlabeled_label=None): """Forward-backward propagation routine for classification model extended with the auxiliary self-supervised task of predicting the relative location of patches.""" if base_ids is not None: assert base_ids.size(0) == 1 assert images.dim() == 4 assert labels.dim() == 1 assert images.size(0) == labels.size(0) #assert att_classifier != None #assert att_classifier_optimizer != None assert patches.dim() == 5 and patches.size(1) == 9 assert patches.size(0) == labels_patches.size(0) patches = utils.convert_from_5d_to_4d(patches) if standardize_patches: patches = utils.standardize_image(patches) if is_train: # Zero gradients. feature_extractor_optimizer.zero_grad() classifier_optimizer.zero_grad() #if att_classifier_optimizer != None: # att_classifier_optimizer.zero_grad() if rotation_loss_coef > 0.0: rot_classifier_optimizer.zero_grad() if patch_location_loss_coef > 0.0: location_classifier_optimizer.zero_grad() jig_classifier_optimizer.zero_grad() if cluster_loss_coef > 0.0: clu_classifier_optimizer.zero_grad() if patch_classification_loss_coef > 0.0: patch_classifier_optimizer.zero_grad() with torch.no_grad(): images, labels, labels_rotation = preprocess_input_data( images, labels, None, random_rotation, rotation_invariant_classifier) record = {} with torch.set_grad_enabled(is_train): # Extract features from the images. features_images = feature_extractor(images) # Extract features from the image patches. features_patches = feature_extractor(patches) #pdb.set_trace() #pdb.set_trace() # Perform object classification task. scores_classification, loss_classsification = cls_utils.classification_task( classifier, features_images, labels, base_ids) record["loss_cls"] = loss_classsification.item() loss_total = loss_classsification #attention #lamda_rot = lamda[0] / torch.sum(lamda) #lamda_loc = lamda[1] / torch.sum(lamda) #lamda_jig = lamda[2] / torch.sum(lamda) #lamda_clu = lamda[3] / torch.sum(lamda) #lamda_rot,lamda_loc, lamda_jig,lamda_clu = att_classifier(features_images).mean(0) #record["lamda_rot"] = lamda_rot.item() #record["lamda_loc"] = lamda_loc.item() #record["lamda_jig"] = lamda_jig.item() #record["lamda_clu"] = lamda_clu.item() #print(lamda) if rotation_loss_coef > 0.0: scores_rotation, loss_rotation = rotation_task( rot_classifier, features_images, labels_rotation) record["loss_rot"] = loss_rotation.item() loss_total = loss_total + loss_rotation if patch_location_loss_coef > 0.0: # patch location prediction. scores_location, loss_location, labels_loc = patch_location_task( location_classifier, features_patches) record["loss_loc"] = loss_location.item() loss_total = loss_total + loss_location # patch puzzle prediction. scores_puzzle, loss_puzzle, labels_puzzle = jigsaw_puzzle_task( jig_classifier, features_patches) record["loss_jig"] = loss_puzzle.item() loss_total = loss_total + loss_puzzle if cluster_loss_coef > 0.0: #cluster prediction. features_unlabeled = cls_utils.extract_features( feature_extractor, images_unlabeled) scores_cluster, loss_cluster = cluster_task( clu_classifier, features_unlabeled, images_unlabeled_label) record["loss_clu"] = loss_cluster.item() loss_total = loss_total + loss_cluster #pdb.set_trace() # Perform the auxiliary task of classifying individual patches. if patch_classification_loss_coef > 0.0: scores_patch, loss_patch = patch_classification_task( patch_classifier, features_patches, labels_patches, combine) record["loss_patch_cls"] = loss_patch.item() loss_total = loss_total + loss_patch * patch_classification_loss_coef # Because the total loss consists of multiple individual losses # (i.e., 3) scale it down by a factor of 0.5. loss_total = loss_total * 0.5 with torch.no_grad(): # Compute accuracies. record["Accuracy"] = utils.top1accuracy(scores_classification, labels) if rotation_loss_coef > 0.0: record["AccuracyRot"] = utils.top1accuracy(scores_rotation, labels_rotation) if patch_location_loss_coef > 0.0: record["AccuracyLoc"] = utils.top1accuracy(scores_location, labels_loc) record["AccuracyJig"] = utils.top1accuracy(scores_puzzle, labels_puzzle) if cluster_loss_coef > 0.0: record["AccuracyClu"] = utils.top1accuracy(scores_cluster, images_unlabeled_label) if patch_classification_loss_coef > 0.0: record["AccuracyPatch"] = utils.top1accuracy( scores_patch, labels_patches) if is_train: # Backward loss and apply gradient steps. loss_total.backward() feature_extractor_optimizer.step() classifier_optimizer.step() #att_classifier_optimizer.step() if rotation_loss_coef > 0.0: rot_classifier_optimizer.step() if patch_location_loss_coef > 0.0: location_classifier_optimizer.step() jig_classifier_optimizer.step() if cluster_loss_coef > 0.0: clu_classifier_optimizer.step() if patch_classification_loss_coef > 0.0: patch_classifier_optimizer.step() return record