def cleanup_folders(output_path: str):
"""
Cleanup the zipped folders, as the data now exists in the VISSL compatible format.
"""
for file_to_delete in ["dtd", "dtd-r1.0.1.tar.gz"]:
file_to_delete = os.path.join(output_path, file_to_delete)
cleanup_dir(file_to_delete, recursive=True)
def cleanup_unused_files(output_path: str):
"""
Cleanup the unused folders, as the data now exists in the VISSL compatible format.
"""
for file_to_delete in ["food-101", "food-101.tar.gz"]:
file_to_delete = os.path.join(output_path, file_to_delete)
cleanup_dir(file_to_delete)
def cleanup_unused_files(output_path: str):
"""
Cleanup the unused folders, as the data now exists in the VISSL compatible format.
"""
for file_to_delete in ["annotations", "images"]:
file_to_delete = os.path.join(output_path, file_to_delete)
cleanup_dir(file_to_delete)
def cleanup_unused_files(output_path: str):
"""
Cleanup the unused folders, as the data now exists in the VISSL compatible format.
"""
for file_to_delete in ["ImageNet-Sketch.zip", "images"]:
file_to_delete = os.path.join(output_path, file_to_delete)
cleanup_dir(file_to_delete, recursive=True)
def cleanup_unused_files(output_path: str):
"""
Cleanup the unused folders, as the data now exists in the VISSL compatible format.
"""
for file_to_delete in [
"101_ObjectCategories", "101_ObjectCategories.tar.gz"
]:
file_to_delete = os.path.join(output_path, file_to_delete)
cleanup_dir(file_to_delete, recursive=True)
def cleanup_unused_files(output_path: str):
"""
Cleanup the unused folders, as the data now exists in the VISSL compatible format.
"""
for file_to_delete in [
"cars_train",
"cars_test",
"car_devkit.tgz",
"cars_test.tgz",
"cars_test_annos_withlabels.mat",
"cars_train.tgz",
]:
file_to_delete = os.path.join(output_path, file_to_delete)
cleanup_dir(file_to_delete)
def cleanup_unused_files(output_path: str):
"""
Cleanup the unused folders, as the data now exists in the VISSL compatible format.
"""
file_to_delete = os.path.join(output_path, "oxbuild_images.tgz")
cleanup_dir(file_to_delete)
def instance_retrieval_test(args, cfg):
# We require 1-gpu for feature extraction. Hence check CUDA is available.
assert torch.cuda.is_available(), "CUDA not available, Exit!"
train_dataset_name = cfg.IMG_RETRIEVAL.TRAIN_DATASET_NAME
eval_dataset_name = cfg.IMG_RETRIEVAL.EVAL_DATASET_NAME
spatial_levels = cfg.IMG_RETRIEVAL.SPATIAL_LEVELS
resize_img = cfg.IMG_RETRIEVAL.RESIZE_IMG
eval_binary_path = cfg.IMG_RETRIEVAL.EVAL_BINARY_PATH
root_dataset_path = cfg.IMG_RETRIEVAL.DATASET_PATH
temp_dir = f"{cfg.IMG_RETRIEVAL.TEMP_DIR}/{str(uuid.uuid4())}"
logging.info(f"Temp directory: {temp_dir}")
############################################################################
# Step 1: Prepare the train/eval datasets, create model and load weights
# We only create the train dataset if we need PCA/whitening otherwise
# train_dataset is None
train_dataset = get_train_dataset(cfg, root_dataset_path,
train_dataset_name, eval_binary_path)
# create the eval dataset. INSTRE data evaluation requires whitening.
eval_dataset = get_eval_dataset(cfg, root_dataset_path, eval_dataset_name,
eval_binary_path)
# Setup the data transforms (basic) that we apply on the train/eval dataset.
transforms = get_transforms(cfg, eval_dataset_name)
# Create the image helper
image_helper = InstanceRetrievalImageLoader(S=resize_img,
transforms=transforms)
# Build the model on gpu and set in the eval mode
model = build_retrieval_model(cfg)
model = copy_model_to_gpu(model)
logging.info("Freezing the model.....")
model.eval()
model.freeze_head_and_trunk()
############################################################################
# Step 2: Extract the features for the train dataset, calculate PCA or
# whitening and save
if cfg.IMG_RETRIEVAL.SHOULD_TRAIN_PCA_OR_WHITENING:
logging.info("Extracting training features...")
# the features are already processed based on type: rmac | GeM | l2 norm
train_features = get_train_features(
cfg,
temp_dir,
train_dataset_name,
resize_img,
spatial_levels,
image_helper,
train_dataset,
model,
)
########################################################################
# Train PCA on the train features
pca_out_fname = f"{temp_dir}/{train_dataset_name}_S{resize_img}_PCA.pickle"
if PathManager.exists(pca_out_fname):
logging.info("Loading PCA...")
pca = load_pca(pca_out_fname)
else:
logging.info("Training and saving PCA...")
pca = train_and_save_pca(train_features, cfg.IMG_RETRIEVAL.N_PCA,
pca_out_fname)
else:
pca = None
############################################################################
# Step 4: Extract db_features and q_features for the eval dataset
logging.info("Extracting Queries features...")
features_queries = get_queries_features(
cfg,
temp_dir,
eval_dataset_name,
resize_img,
spatial_levels,
image_helper,
eval_dataset,
model,
pca,
)
logging.info("Extracting Dataset features...")
features_dataset = get_dataset_features(
cfg,
temp_dir,
eval_dataset_name,
resize_img,
spatial_levels,
image_helper,
eval_dataset,
model,
pca,
)
############################################################################
# Step 5: Compute similarity and score
logging.info("Calculating similarity and score...")
sim = features_queries.dot(features_dataset.T)
logging.info(f"Similarity tensor: {sim.shape}")
eval_dataset.score(sim, temp_dir)
############################################################################
# Step 6: cleanup the temp directory
logging.info(f"Cleaning up temp directory: {temp_dir}")
cleanup_dir(temp_dir)
logging.info("All done!!")
def _cleanup_local_dir(cfg: AttrDict):
available_splits = _get_available_splits(cfg)
for split in available_splits:
if cfg.DATA[split].COPY_TO_LOCAL_DISK:
dest_dir = cfg.DATA[split]["COPY_DESTINATION_DIR"]
cleanup_dir(dest_dir)
