def train_sample_places_low_shot( low_shot_trainer: SVMLowShotTrainer, k_values: List[int], sample_inds: List[int], sample_num: int, output_dir: str, layername: str, cfg: AttrDict, ): # setup the environment variables set_env_vars(local_rank=0, node_id=0, cfg=cfg) for low_shot_kvalue in k_values: checkpoint_dir = f"{output_dir}/sample{sample_num}_k{low_shot_kvalue}" train_data = merge_features(checkpoint_dir, "train", layername) train_features = train_data["features"] train_targets = train_data["targets"] checkpoint_dir = f"{output_dir}/sample{sample_inds[0]}_k{k_values[0]}" test_data = merge_features(checkpoint_dir, "test", layername) test_features = test_data["features"] test_targets = test_data["targets"] low_shot_trainer.train(train_features, train_targets, sample_num, low_shot_kvalue) low_shot_trainer.test(test_features, test_targets, sample_num, low_shot_kvalue)
def train_voc07_low_shot( k_values: List[int], sample_inds: List[int], output_dir: str, layername: str, cfg: AttrDict, ): dataset_name = cfg["SVM"]["low_shot"]["dataset_name"] low_shot_trainer = SVMLowShotTrainer(cfg["SVM"], layer=layername, output_dir=output_dir) train_data = merge_features(output_dir, "train", layername) train_features, train_targets = train_data["features"], train_data[ "targets"] test_data = merge_features(output_dir, "test", layername) test_features, test_targets = test_data["features"], test_data["targets"] # now we want to create the low-shot samples based on the kind of dataset. # We only create low-shot samples for training. We test on the full dataset. generate_low_shot_samples(dataset_name, train_targets, k_values, sample_inds, output_dir, layername) # Now, we train and test the low-shot SVM for every sample and k-value. for sample_num in sample_inds: for low_shot_kvalue in k_values: train_targets = load_file( f"{output_dir}/{layername}_sample{sample_num}_k{low_shot_kvalue}.npy" ) low_shot_trainer.train(train_features, train_targets, sample_num, low_shot_kvalue) low_shot_trainer.test(test_features, test_targets, sample_num, low_shot_kvalue) # now we aggregate the stats across all independent samples and for each # k-value and report mean/min/max/std stats results = low_shot_trainer.aggregate_stats(k_values, sample_inds) logging.info("All Done!") return results
def test_extract_cluster_assignment_ddp(self): with in_temporary_directory() as pretrain_dir: # Run a pre-training to have some weights to being with pretrain_config = self._create_pretraining_config() run_integration_test(pretrain_config) # Create a directory to contain the extracted features with in_temporary_directory() as extract_dir: # Run the extract engine in a separate directory to check that # it is correctly able to output the feature in a another dir with in_temporary_directory(): extract_config = self._create_extract_features_config( checkpoint_path=os.path.join(pretrain_dir, "checkpoint.torch")) extract_config.EXTRACT_FEATURES.OUTPUT_DIR = extract_dir run_integration_test(extract_config, engine_name="extract_features") # Check the content of the directory containing the extracted dirs folder_content = os.listdir(extract_dir) print(folder_content) for rank in [0, 1]: for chunk in range(5): for file in [ f"rank{rank}_chunk{chunk}_train_heads_features.npy", f"rank{rank}_chunk{chunk}_train_heads_inds.npy", f"rank{rank}_chunk{chunk}_train_heads_targets.npy", ]: self.assertIn(file, folder_content) # Verify that we can merge the features back (train split) train_feat = merge_features(extract_dir, "train", "heads") print(train_feat) self.assertEqual(train_feat["features"].shape, torch.Size([40, 128])) self.assertEqual(train_feat["targets"].shape, torch.Size([40, 1])) self.assertEqual(train_feat["inds"].shape, torch.Size([40])) # Verify that we can merge the features back (test split) test_feat = merge_features(extract_dir, "test", "heads") self.assertEqual(test_feat["features"].shape, torch.Size([20, 128])) self.assertEqual(test_feat["targets"].shape, torch.Size([20, 1])) self.assertEqual(test_feat["inds"].shape, torch.Size([20]))
def train_svm(cfg: AttrDict, output_dir: str, layername: str): # setup the environment variables set_env_vars(local_rank=0, node_id=0, cfg=cfg) # train the svm logging.info(f"Training SVM for layer: {layername}") trainer = SVMTrainer(cfg["SVM"], layer=layername, output_dir=output_dir) train_data = merge_features(output_dir, "train", layername) train_features, train_targets = train_data["features"], train_data["targets"] trainer.train(train_features, train_targets) # test the svm test_data = merge_features(output_dir, "test", layername) test_features, test_targets = test_data["features"], test_data["targets"] trainer.test(test_features, test_targets) logging.info("All Done!")
def get_data_features_and_images(cfg: AttrDict): output_dir = get_checkpoint_folder(cfg) split = cfg.RANKING.FEATURES.DATA_PARTITION logging.info("Merging features...") # merge the features across all nodes/gpus into one feature_data = merge_features(output_dir, split.lower(), cfg.RANKING.FEATURES.LAYER_NAME) logging.info("Getting the image paths...") # get the list of image Ids dataset = build_dataset(cfg=cfg, split=split) feature_image_paths = dataset.get_image_paths() # due to multi-modality, we get image_paths as a nested list, one for each # dataset. Check it's a list and extract images. assert type(feature_image_paths) == list, "Image paths must be a list" assert len(feature_image_paths) == 1, "Multi-modality not supported yet!" return feature_data, feature_image_paths[0]
def nearest_neighbor_test(cfg: AttrDict, layer_name: str = "heads"): temperature = cfg.NEAREST_NEIGHBOR.SIGMA num_neighbors = cfg.NEAREST_NEIGHBOR.TOPK output_dir = get_checkpoint_folder(cfg) logging.info(f"Testing with sigma: {temperature}, topk neighbors: {num_neighbors}") ############################################################################ # Step 1: get train and test features train_out = merge_features(output_dir, "train", layer_name, cfg) train_features, train_labels = train_out["features"], train_out["targets"] # put train features and labels on gpu and transpose train features train_features = torch.from_numpy(train_features).float().cuda().t() train_labels = torch.LongTensor(train_labels).cuda() num_classes = train_labels.max() + 1 if cfg.NEAREST_NEIGHBOR.L2_NORM_FEATS: train_features = nn.functional.normalize(train_features, dim=0, p=2) test_out = merge_features(output_dir, "test", layer_name, cfg) test_features, test_labels = test_out["features"], test_out["targets"] ########################################################################### # Step 2: calculate the nearest neighbor and the metrics top1, top5, total = 0.0, 0.0, 0 num_test_images, num_chunks = test_labels.shape[0], 100 imgs_per_chunk = num_test_images // num_chunks with torch.no_grad(): retrieval_one_hot = torch.zeros(num_neighbors, num_classes).cuda() for idx in range(0, num_test_images, imgs_per_chunk): # get the features for test images and normalize the features if needed features = test_features[ idx : min((idx + imgs_per_chunk), num_test_images), : ] targets = test_labels[idx : min((idx + imgs_per_chunk), num_test_images), :] batch_size = targets.shape[0] features = torch.from_numpy(features).float().cuda() targets = torch.LongTensor(targets).cuda() if cfg.NEAREST_NEIGHBOR.L2_NORM_FEATS: features = nn.functional.normalize(features, dim=1, p=2) # calculate the dot product and compute top-k neighbors similarity = torch.mm(features, train_features) distances, indices = similarity.topk( num_neighbors, largest=True, sorted=True ) candidates = train_labels.view(1, -1).expand(batch_size, -1) retrieved_neighbors = torch.gather(candidates, 1, indices) retrieval_one_hot.resize_(batch_size * num_neighbors, num_classes).zero_() retrieval_one_hot.scatter_(1, retrieved_neighbors.view(-1, 1), 1) distances_transform = distances.clone().div_(temperature).exp_() probs = torch.sum( torch.mul( retrieval_one_hot.view(batch_size, -1, num_classes), distances_transform.view(batch_size, -1, 1), ), 1, ) _, predictions = probs.sort(1, True) # find the predictions that match the target correct = predictions.eq(targets.data.view(-1, 1)) top1 = top1 + correct.narrow(1, 0, 1).sum().item() top5 = top5 + correct.narrow(1, 0, 5).sum().item() total += targets.size(0) top1 = top1 * 100.0 / total top5 = top5 * 100.0 / total logging.info(f"Total images: {total}, Top1: {top1}, Top5: {top5}") return top1, top5
def test_extract_cluster_assignment_ddp(self): with in_temporary_directory() as pretrain_dir: # Run a pre-training to have some weights to being with pretrain_config = self._create_pretraining_config() run_integration_test(pretrain_config) # Create a directory to contain the extracted features with in_temporary_directory() as extract_dir: # Run the extract engine in a separate directory to check that # it is correctly able to output the feature in a another dir with in_temporary_directory(): extract_config = self._create_extract_features_config_head( checkpoint_path=os.path.join(pretrain_dir, "checkpoint.torch") ) extract_config.EXTRACT_FEATURES.OUTPUT_DIR = extract_dir run_integration_test(extract_config, engine_name="extract_features") # Check the content of the directory containing the extracted dirs folder_content = os.listdir(extract_dir) print(folder_content) for rank in [0, 1]: for chunk in range(5): for file in [ f"rank{rank}_chunk{chunk}_train_heads_features.npy", f"rank{rank}_chunk{chunk}_train_heads_inds.npy", f"rank{rank}_chunk{chunk}_train_heads_targets.npy", ]: self.assertIn(file, folder_content) # Verify that we can merge the features back (train split) train_feat = merge_features(extract_dir, "train", "heads") print(train_feat) self.assertEqual(train_feat["features"].shape, torch.Size([40, 128])) self.assertEqual(train_feat["targets"].shape, torch.Size([40, 1])) self.assertEqual(train_feat["inds"].shape, torch.Size([40])) # Verify that we can merge the features back (test split) test_feat = merge_features(extract_dir, "test", "heads") self.assertEqual(test_feat["features"].shape, torch.Size([20, 128])) self.assertEqual(test_feat["targets"].shape, torch.Size([20, 1])) self.assertEqual(test_feat["inds"].shape, torch.Size([20])) # Run the extract engine this time for the features of the trunk with in_temporary_directory(): extract_config = self._create_extract_features_config_trunk( checkpoint_path=os.path.join(pretrain_dir, "checkpoint.torch") ) extract_config.EXTRACT_FEATURES.OUTPUT_DIR = extract_dir run_integration_test(extract_config, engine_name="extract_features") # Verify that we can merge the features back without flattening them train_feat = merge_features(extract_dir, "train", "res5") self.assertEqual( train_feat["features"].shape, torch.Size([40, 2048, 2, 2]) ) self.assertEqual(train_feat["targets"].shape, torch.Size([40, 1])) self.assertEqual(train_feat["inds"].shape, torch.Size([40])) # Verify that we can merge the features back without flattening them (second approach) train_feat = ExtractedFeaturesLoader.load_features( extract_dir, "train", "res5" ) self.assertEqual( train_feat["features"].shape, torch.Size([40, 2048, 2, 2]) ) # Verify that we can merge the features back but flattened train_feat = ExtractedFeaturesLoader.load_features( extract_dir, "train", "res5", flatten_features=True ) self.assertEqual( train_feat["features"].shape, torch.Size([40, 2048 * 2 * 2]) ) self.assertEqual(train_feat["targets"].shape, torch.Size([40, 1])) self.assertEqual(train_feat["inds"].shape, torch.Size([40])) # Verify that we can sample the features (unflattened) train_feat = ExtractedFeaturesLoader.sample_features( input_dir=extract_dir, split="train", layer="res5", num_samples=10, seed=0, ) self.assertEqual( train_feat["features"].shape, torch.Size([10, 2048, 2, 2]) ) self.assertEqual(train_feat["targets"].shape, torch.Size([10, 1])) self.assertEqual(train_feat["inds"].shape, torch.Size([10])) # Verify that we can sample the features (flattened) train_feat = ExtractedFeaturesLoader.sample_features( input_dir=extract_dir, split="train", layer="res5", num_samples=10, seed=0, flatten_features=True, ) self.assertEqual( train_feat["features"].shape, torch.Size([10, 2048 * 2 * 2]) ) self.assertEqual(train_feat["targets"].shape, torch.Size([10, 1])) self.assertEqual(train_feat["inds"].shape, torch.Size([10]))