def cluster_features_and_label(args: Namespace, cfg: AttrDict):
# faiss is an optional dependency for VISSL.
assert is_faiss_available(), (
"Please install faiss using conda install faiss-gpu -c pytorch "
"if using conda or pip install faiss-gpu"
)
import faiss
cluster_backend = cfg.CLUSTERFIT.CLUSTER_BACKEND
num_clusters = cfg.CLUSTERFIT.NUM_CLUSTERS
data_split = cfg.CLUSTERFIT.FEATURES.DATA_PARTITION
data_name = cfg.CLUSTERFIT.FEATURES.DATASET_NAME
n_iter = cfg.CLUSTERFIT.N_ITER
output_dir = get_checkpoint_folder(cfg)
########### Step 1: Extract the features on full dataset ###################
feature_data, image_paths = get_data_features_and_images(cfg)
########### Step 2: Get the data information ###################
features = feature_data["features"]
# features are of shape num_samples x feature_dim
assert features.ndim == 2, f"Features incorrect shape: {features.shape}"
assert features.dtype == np.float32, "Features are not float32 type"
logging.info(f"Clustering Features: {features.shape}")
########### Step 3: L2 normalize features ###################
# TODO: we could support PCA here if needed in future.
logging.info("L2 normalizing the features now...")
feat_norm = np.linalg.norm(features, axis=1) + 1e-5
features = features / feat_norm[:, np.newaxis]
########### Step 4: Cluster the features ###################
logging.info("Clustering the features now...")
assert cluster_backend == "faiss", "Only faiss clustering is supported currently"
kmeans = faiss.Kmeans(features.shape[1], num_clusters, niter=n_iter, verbose=True)
kmeans.train(features)
centroids = kmeans.centroids
########### Step 5: Get the cluster assignment for the features ############
logging.info("Getting cluster label assignment now...")
distances, hard_cluster_labels = kmeans.index.search(features, 1)
#### Step 6: Save clustering data and hard cluster labels for the images ###
data_split = data_split.lower()
clustering_output_dict = {
"hard_labels": hard_cluster_labels,
"centroids": centroids,
"distances": distances,
}
cluster_output_filepath = (
f"{output_dir}/{data_name}_{data_split}_N{num_clusters}_{cluster_backend}.pkl"
)
hard_labels_output_filepath = (
f"{output_dir}/"
f"{data_name}_{data_split}_N{num_clusters}_{cluster_backend}_lbls.npy"
)
out_hard_labels = np.array(hard_cluster_labels.tolist(), dtype=np.int64).reshape(-1)
save_file(clustering_output_dict, cluster_output_filepath)
save_file(out_hard_labels, hard_labels_output_filepath)
logging.info("All Done!")
def generate_output_json_data(
model_name, predictions_file, pred_confidence_scores_file, pred_img_indices_file
):
my_model_in22k_subset_dollar_street_full_finetuned = {
"model_name": model_name,
"image_paths": "https://dl.fbaipublicfiles.com/vissl/fairness/dollarstreet_in22k_cls_overlapped_images.npy",
"targets": "https://dl.fbaipublicfiles.com/vissl/fairness//subset_dollarstreet_in22k_cls_overlapped_labels.npy",
"metadata": "https://dl.fbaipublicfiles.com/vissl/fairness/metadata_full_dollar_street.json",
"id_to_label_map": "https://dl.fbaipublicfiles.com/vissl/fairness/in22k_cls_idx_to_dollar_street_labels_map.json",
"predictions": predictions_file,
"pred_img_indices": pred_img_indices_file,
"pred_confidence_scores": pred_confidence_scores_file,
}
(
my_model_output_attributes_acc_map,
my_model_output_metadata_map,
) = generate_dollar_street_analysis(
my_model_in22k_subset_dollar_street_full_finetuned,
topk=PRED_TOPK,
confidence_threshold=PRED_CONFIDENCE_THRESHOLD,
)
print(list(my_model_output_metadata_map.values())[:10])
output_dir = "/tmp/dollar_street_models"
output_file = f"{output_dir}/my_model_output_metadata_map.json"
save_file(my_model_output_metadata_map, output_file)
test_data_save = load_file(output_file)
print(len(test_data_save))
return output_dir
def _save_extracted_label_predictions(
all_image_paths: List[str],
predictions: Dict[str, Dict[int, Any]],
confidence_scores: Dict[str, Dict[str, Any]],
targets: Dict[str, Dict[int, Any]],
dist_rank: int,
split: str,
output_folder: str,
):
output = {}
for layer_name in predictions.keys():
predictions[layer_name] = dict(sorted(predictions[layer_name].items()))
targets[layer_name] = dict(sorted(targets[layer_name].items()))
confidence_scores[layer_name] = dict(
sorted(confidence_scores[layer_name].items())
)
preds = np.array(torch.stack(list(predictions[layer_name].values())))
scores = np.array(torch.stack(list(confidence_scores[layer_name].values())))
indices = np.array(list(predictions[layer_name].keys()))
image_paths = np.array([all_image_paths[i] for i in indices])
N = preds.shape[0]
output[layer_name] = {
"predictions": preds.reshape(N, -1),
"confidence_scores": scores.reshape(N, -1),
"targets": np.array(list(targets[layer_name].values())),
"inds": indices,
"image_paths": image_paths,
}
split = split.lower()
for layer_name, layer_prediction in output.items():
out_pred_file = (
f"{output_folder}/rank{dist_rank}_{split}_{layer_name}_predictions.npy"
)
out_scores_file = (
f"{output_folder}/rank{dist_rank}_{split}_{layer_name}_conf_scores.npy"
)
out_target_file = (
f"{output_folder}/rank{dist_rank}_{split}_{layer_name}_targets.npy"
)
out_inds_file = (
f"{output_folder}/rank{dist_rank}_{split}_{layer_name}_inds.npy"
)
out_images_file = (
f"{output_folder}/rank{dist_rank}_{split}_{layer_name}_images.npy"
)
logging.info(
f"For {layer_name}, "
f"saving predictions: {layer_prediction['predictions'].shape}, "
f"saving scores: {layer_prediction['confidence_scores'].shape}, "
f"targets: {layer_prediction['targets'].shape}, "
f"inds: {layer_prediction['inds'].shape}, "
f"images: {layer_prediction['image_paths'].shape}"
)
save_file(layer_prediction["predictions"], out_pred_file)
save_file(layer_prediction["confidence_scores"], out_scores_file)
save_file(layer_prediction["targets"], out_target_file)
save_file(layer_prediction["inds"], out_inds_file)
save_file(layer_prediction["image_paths"], out_images_file)
def process_train_image(i, out_dir):
if i % LOG_FREQUENCY == 0:
logging.info(f"Train Image: {i}"),
fname_out = f"{out_dir}/{i}.npy"
if PathManager.exists(fname_out):
feat = load_file(fname_out)
train_features.append(feat)
else:
fname_in = train_dataset.get_filename(i)
if is_revisited_dataset(train_dataset_name):
img = image_helper.load_and_prepare_revisited_image(fname_in)
elif is_whiten_dataset(train_dataset_name):
img = image_helper.load_and_prepare_whitening_image(fname_in)
else:
img = image_helper.load_and_prepare_image(fname_in, roi=None)
v = torch.autograd.Variable(img.unsqueeze(0))
vc = v.cuda()
# the model output is a list always.
activation_map = model(vc)[0].cpu()
# once we have the features,
# we can perform: rmac | gem pooling | l2 norm
if cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "rmac":
descriptors = get_rmac_descriptors(activation_map,
spatial_levels)
else:
descriptors = activation_map
save_file(descriptors.data.numpy(), fname_out)
train_features.append(descriptors.data.numpy())
def process_eval_image(
cfg,
fname_in,
roi,
fname_out,
spatial_levels,
image_helper,
model,
pca,
eval_dataset_name,
):
if is_revisited_dataset(eval_dataset_name):
img = image_helper.load_and_prepare_revisited_image(fname_in, roi=roi)
elif is_instre_dataset(eval_dataset_name):
img = image_helper.load_and_prepare_instre_image(fname_in)
else:
img = image_helper.load_and_prepare_image(fname_in, roi=roi)
v = torch.autograd.Variable(img.unsqueeze(0))
vc = v.cuda()
# the model output is a list always.
activation_map = model(vc)[0].cpu()
# process the features: rmac | l2 norm
if cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "rmac":
descriptors = get_rmac_descriptors(activation_map,
spatial_levels,
pca=pca)
elif cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "l2_norm":
# we simply L2 normalize the features otherwise
descriptors = F.normalize(activation_map, p=2, dim=0)
else:
descriptors = activation_map
save_file(descriptors.data.numpy(), fname_out)
return descriptors.data.numpy()
def get_coco_imgs_labels_info(split, data_source_dir, args):
# pycocotools is an optional dependency for VISSL
from pycocotools.coco import COCO
json_file = f"{data_source_dir}/annotations/instances_{split}2014.json"
assert PathManager.exists(
json_file), "Annotations file does not exist. Abort"
json_data = json.load(PathManager.open(json_file, "r"))
image_index = [x["id"] for x in json_data["images"]]
coco = COCO(json_file)
num_cats = len(json_data["categories"])
logging.info("partition: {} num_cats: {} num_images: {}".format(
split, num_cats, len(image_index)))
cat_ids = [x["id"] for x in json_data["categories"]]
coco_to_me = {val: ind for ind, val in enumerate(cat_ids)}
cat_names = [str(x["name"]) for x in json_data["categories"]]
cat_name_to_id, cat_id_to_name = {}, {}
for ind, name in enumerate(cat_names):
cat_name_to_id[name] = ind
cat_id_to_name[ind] = name
class_ids = cat_id_to_name.keys()
assert len(list(class_ids)) == num_cats
assert min(class_ids) == 0
assert max(class_ids) == len(class_ids) - 1
assert len(set(class_ids)) == len(class_ids)
# label_matrix = np.zeros((len(image_index), len(cat_names)), dtype=np.float32)
# area_matrix = np.zeros((len(image_index), len(cat_names)), dtype=np.float32)
img_labels_map = {}
num_classes = len(cat_names)
for _, im_id in enumerate(image_index):
ann_ids = coco.getAnnIds(imgIds=im_id)
entry = coco.imgs[im_id]
img_name = entry["file_name"]
objs = coco.loadAnns(ann_ids)
valid_objs = get_valid_objs(entry, objs)
if img_name not in img_labels_map:
img_labels_map[img_name] = np.zeros(num_classes, dtype=np.int32)
for _, obj in enumerate(valid_objs):
cocoCatId = obj["category_id"]
myId = coco_to_me[cocoCatId]
img_labels_map[img_name][myId] = 1.0
# label = 1 (present), 0 (not present)
img_paths, img_labels = [], []
train_imgs_path = f"{data_source_dir}/train2014"
val_imgs_path = f"{data_source_dir}/val2014"
prefix = train_imgs_path if split == "train" else val_imgs_path
for item in sorted(img_labels_map.keys()):
img_paths.append(f"{prefix}/{item}")
img_labels.append(img_labels_map[item])
# save to the datasets folder and return the path
output_dir = get_output_dir()
img_info_out_path = f"{output_dir}/{split}_images.npy"
label_info_out_path = f"{output_dir}/{split}_labels.npy"
save_file(np.array(img_paths), img_info_out_path)
save_file(np.array(img_labels), label_info_out_path)
return [img_info_out_path, label_info_out_path]
def train_cls(self, features, targets, cls_num):
"""
Train SVM on the input features and targets for a given class.
The SVMs are trained for all costs values for the given class. We
also save the cross-validation AP at each cost value for the given
class.
"""
logging.info(f"Training cls: {cls_num}")
for cost_idx in range(len(self.costs_list)):
cost = self.costs_list[cost_idx]
out_file, ap_out_file = self._get_svm_model_filename(cls_num, cost)
if (g_pathmgr.exists(out_file) and g_pathmgr.exists(ap_out_file)
and not self.config.force_retrain):
logging.info(f"SVM model exists: {out_file}")
logging.info(f"AP file exists: {ap_out_file}")
continue
logging.info(
f"Training model with the cost: {cost} cls: {cls_num}")
clf = LinearSVC(
C=cost,
class_weight={
1: 2,
-1: 1
},
intercept_scaling=1.0,
verbose=1,
penalty=self.config["penalty"],
loss=self.config["loss"],
tol=0.0001,
dual=self.config["dual"],
max_iter=self.config["max_iter"],
)
cls_labels = targets[:, cls_num].astype(dtype=np.int32, copy=True)
# meaning of labels in VOC/COCO original loaded target files:
# label 0 = not present, set it to -1 as svm train target
# label 1 = present. Make the svm train target labels as -1, 1.
cls_labels[np.where(cls_labels == 0)] = -1
num_positives = len(np.where(cls_labels == 1)[0])
num_negatives = len(cls_labels) - num_positives
logging.info(
f"cls: {cls_num} has +ve: {num_positives} -ve: {num_negatives} "
f"ratio: {float(num_positives) / num_negatives} "
f"features: {features.shape} cls_labels: {cls_labels.shape}")
ap_scores = cross_val_score(
clf,
features,
cls_labels,
cv=self.config["cross_val_folds"],
scoring="average_precision",
)
self.train_ap_matrix[cls_num][cost_idx] = ap_scores.mean()
clf.fit(features, cls_labels)
logging.info(f"cls: {cls_num} cost: {cost} AP: {ap_scores} "
f"mean:{ap_scores.mean()}")
logging.info(f"Saving cls cost AP to: {ap_out_file}")
save_file(np.array([ap_scores.mean()]), ap_out_file)
logging.info(f"Saving SVM model to: {out_file}")
with g_pathmgr.open(out_file, "wb") as fwrite:
pickle.dump(clf, fwrite)
def generate_places_low_shot_samples(targets, k_values, sample_inds,
output_path, images_data_file):
logging.info("Generating low-shot samples for places data...")
k_values = [int(val) for val in k_values]
logging.info(f"Loading images data file: {images_data_file}")
images = load_file(images_data_file)
# get the maximum and minumum number of positives per class
num_pos = find_num_positives(targets)
logging.info(f"min #num_pos: {min(num_pos)}, max #num_pos: {max(num_pos)}")
# start sampling now. the way sampling works is:
# for each independent sample, and a given k value,
# we create an output targets vector of shape same as the input targets.
# We initialize this matrix with -1 (ignore values). We sample k positive
# for each given class and set the value in the matrix to the class number.
# Thus the resulting matrix has (k * num_classes) samples sampled and
# remaining are ignored.
for idx in sample_inds:
for k in k_values:
if k > min(num_pos):
logging.info(f"Skip k: {k} min #pos: {min(num_pos)}")
continue
logging.info(f"Sampling: {idx} time for k-value: {k}")
out_lbls = np.ones(targets.shape, dtype=np.int32) * -1
out_imgs, out_lbls = sample_places_data(images, targets, k)
out_img_file = f"{output_path}/train_images_sample{idx}_k{k}.npy"
out_lbls_file = f"{output_path}/train_labels_sample{idx}_k{k}.npy"
logging.info(f"Saving imgs file: {out_img_file} {len(out_imgs)}")
logging.info(f"Saving lbls file: {out_lbls_file} {len(out_lbls)}")
save_file(out_lbls, out_lbls_file)
save_file(out_imgs, out_img_file)
logging.info("Done!!")
def _print_and_save_meters(self, task, train_phase_idx):
"""
Executed only on master gpu at the end of each epoch. Computes the
meters and logs the metrics to the json file and to logger streams
(stdout, file).
"""
phase_type = "train" if task.train else "test"
rank, _ = get_machine_local_and_dist_rank()
checkpoint_folder = task.checkpoint_folder
save_metrics = {}
save_metrics["iteration"] = task.iteration
save_metrics["phase_idx"] = task.phase_idx
save_metrics["train_phase_idx"] = train_phase_idx
for meter in task.meters:
if len(task.meters) > 0 and (
(task.train and task.config["METERS"]["enable_training_meter"])
or (not task.train)):
meter_value = meter.value
metric_key = f"{phase_type}_{meter.name}"
if metric_key not in task.metrics:
task.metrics[metric_key] = []
task.metrics[metric_key].append(meter_value)
save_metrics[metric_key] = meter_value
logging.info(
f"Rank: {rank}, name: {metric_key}, value: {meter_value}")
meter_file = f"{checkpoint_folder}/metrics.json"
save_file(save_metrics, meter_file, append_to_json=True)
def train_and_save_pca(features, n_pca, pca_out_fname):
pca = PCA(n_pca)
pca.fit(features)
logging.info(f"Saving PCA features to: {pca_out_fname}")
save_file(pca, pca_out_fname)
logging.info(f"Saved PCA features to: {pca_out_fname}")
return pca
def process_eval_image(
cfg,
fname_in,
roi,
fname_out,
spatial_levels,
image_helper,
model,
pca,
eval_dataset_name,
verbose=False,
):
if is_revisited_dataset(eval_dataset_name):
img = image_helper.load_and_prepare_revisited_image(fname_in, roi=roi)
elif is_instre_dataset(eval_dataset_name):
img = image_helper.load_and_prepare_instre_image(fname_in)
else:
img = image_helper.load_and_prepare_image(fname_in, roi=roi)
v = torch.autograd.Variable(img.unsqueeze(0))
vc = v.cuda()
# the model output is a list always.
activation_map = model(vc)[0].cpu()
if verbose:
print(f"Eval image raw activation map shape: { activation_map.shape }")
# process the features: rmac | l2 norm
if cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "rmac":
descriptors = get_rmac_descriptors(
activation_map,
spatial_levels,
pca=pca,
normalize=cfg.IMG_RETRIEVAL.NORMALIZE_FEATURES,
)
elif cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "gem":
descriptors = gem(
activation_map,
p=cfg.IMG_RETRIEVAL.GEM_POOL_POWER,
add_bias=True,
)
else:
descriptors = activation_map
# Optionally l2 normalize the features.
if (cfg.IMG_RETRIEVAL.NORMALIZE_FEATURES
and cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE != "rmac"):
# RMAC performs normalization within the algorithm, hence we skip it here.
descriptors = l2n(descriptors, dim=1)
# Optionally apply pca.
if pca and cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE != "rmac":
# RMAC performs pca within the algorithm, hence we skip it here.
descriptors = pca.apply(descriptors)
if fname_out:
save_file(descriptors.data.numpy(), fname_out, verbose=False)
return descriptors.data.numpy()
def process_train_image(i, out_dir, verbose=False):
if i % LOG_FREQUENCY == 0:
logging.info(f"Train Image: {i}"),
fname_out = None
if out_dir:
fname_out = f"{out_dir}/{i}.npy"
if fname_out and PathManager.exists(fname_out):
feat = load_file(fname_out)
train_features.append(feat)
else:
fname_in = train_dataset.get_filename(i)
if is_revisited_dataset(train_dataset_name):
img = image_helper.load_and_prepare_revisited_image(fname_in,
roi=None)
elif is_whiten_dataset(train_dataset_name):
img = image_helper.load_and_prepare_whitening_image(fname_in)
else:
img = image_helper.load_and_prepare_image(fname_in, roi=None)
v = torch.autograd.Variable(img.unsqueeze(0))
vc = v.cuda()
# the model output is a list always.
activation_map = model(vc)[0].cpu()
if verbose:
print(
f"Train Image raw activation map shape: { activation_map.shape }"
)
# once we have the features,
# we can perform: rmac | gem pooling | l2 norm
if cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "rmac":
descriptors = get_rmac_descriptors(
activation_map,
spatial_levels,
normalize=cfg.IMG_RETRIEVAL.NORMALIZE_FEATURES,
)
elif cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE == "gem":
descriptors = gem(
activation_map,
p=cfg.IMG_RETRIEVAL.GEM_POOL_POWER,
add_bias=True,
)
else:
descriptors = activation_map
# Optionally l2 normalize the features.
if (cfg.IMG_RETRIEVAL.NORMALIZE_FEATURES
and cfg.IMG_RETRIEVAL.FEATS_PROCESSING_TYPE != "rmac"):
# RMAC performs normalization within the algorithm, hence we skip it here.
descriptors = l2n(descriptors, dim=1)
if fname_out:
save_file(descriptors.data.numpy(), fname_out, verbose=False)
train_features.append(descriptors.data.numpy())
def _save_label_cls_idx_map(self, cls_idx_map: Dict[str, int], split: str):
local_rank, dist_rank = get_machine_local_and_dist_rank()
if dist_rank == 0:
checkpoint_folder = get_checkpoint_folder(self.cfg)
class_idx_file_path = (
f"{checkpoint_folder}/{split.lower()}_label_to_index_map.json")
if not g_pathmgr.exists(class_idx_file_path):
save_file(cls_idx_map,
class_idx_file_path,
append_to_json=False)
def gem_pool_and_save_features(features, p, add_bias, gem_out_fname):
if PathManager.exists(gem_out_fname):
logging.info("Loading train GeM features...")
features = load_file(gem_out_fname)
else:
logging.info(f"GeM pooling features: {features.shape}")
features = l2n(gem(features, p=p, add_bias=True))
save_file(features, gem_out_fname)
logging.info(f"Saved GeM features to: {gem_out_fname}")
return features
def train_and_save_pca(features, n_pca, pca_out_fname):
logging.info(
f"Fitting PCA with { n_pca } dimensions to features of shape: {features.shape}"
)
pca = PCA(n_pca)
pca.fit(features)
if pca_out_fname:
logging.info(f"Saving PCA features to: {pca_out_fname}")
save_file(pca, pca_out_fname, verbose=False)
logging.info(f"Saved PCA features to: {pca_out_fname}")
return pca
def save_img_labels_filelist(img_paths, img_labels, out_image_filepath,
out_label_filepath):
# Remove the split .npy filelist if they exist and resave them.
if g_pathmgr.exists(out_image_filepath):
g_pathmgr.rm(out_image_filepath)
save_file(img_paths, out_image_filepath)
print(f"Saved: {out_image_filepath}")
if g_pathmgr.exists(out_label_filepath):
g_pathmgr.rm(out_label_filepath)
save_file(img_labels, out_label_filepath)
print(f"Saved: {out_label_filepath}")
print("Saved!!")
def main():
parser = argparse.ArgumentParser(
description="Create the iNaturalist2018 data information file."
)
parser.add_argument(
"-i",
"--input_dir_path",
type=str,
help="Path to the parent directory of the iNaturalist2018 data set",
)
parser.add_argument(
"-o",
"--output_dir_path",
type=str,
help="Folder where the classification dataset will be written",
)
parser.add_argument(
"-d",
"--download",
action="store_const",
const=True,
default=False,
help="To download the original dataset and decompress it in the input folder",
)
args = parser.parse_args()
# Make sure that the input and output directories exist.
assert PathManager.exists(
args.input_dir_path
), "Data input directory not found! Please create the directory"
assert PathManager.exists(
args.output_dir_path
), "Data output directory not found! Please create the directory"
# Download dataset to input path
if args.download:
download_dataset(args.input_dir_path)
# Process training and validation datasets into numpy arrays
logger.info("========Preparing train data files========")
train_images, train_labels = get_images_labels_info(
"/train2018.json", args.input_dir_path
)
logger.info("========Preparing val data files========")
val_images, val_labels = get_images_labels_info(
"/val2018.json", args.input_dir_path
)
# Save as numpy files to output path
logger.info("=================Saving train data files=======================")
train_label_file_name = f"{ args.output_dir_path }/train_labels.npy"
train_image_file_name = f"{ args.output_dir_path }/train_images.npy"
save_file(train_images, train_image_file_name)
save_file(train_labels, train_label_file_name)
logger.info("=================Saving val data files=======================")
val_label_file_name = f"{ args.output_dir_path }/val_labels.npy"
val_image_file_name = f"{ args.output_dir_path }/val_images.npy"
save_file(val_images, val_image_file_name)
save_file(val_labels, val_label_file_name)
def test(self, features, targets, sample_num, low_shot_kvalue):
"""
Test the SVM for the input test features and targets for the given:
low-shot k-value, sample number
We compute the meanAP across all classes for a given cost value.
We get the output matrix of shape (1, #costs) for the given sample_num and
k-value and save the matrix. We use this information to aggregate
later.
"""
logging.info("Testing SVM")
# normalize the features: N x 9216 (example shape)
if self.normalize:
# normalize the features: N x 9216 (example shape)
features = self._normalize_features(features)
sample_ap_matrix = np.zeros((1, len(self.costs_list)))
suffix = f"sample{sample_num}_k{low_shot_kvalue}"
for cost_idx in range(len(self.costs_list)):
cost = self.costs_list[cost_idx]
local_cost_ap = np.zeros((self.num_classes, 1))
for cls_num in self.cls_list:
logging.info(
f"Test sample/k_value/cost/cls: "
f"{sample_num}/{low_shot_kvalue}/{cost}/{cls_num}"
)
model_file = self._get_svm_low_shot_model_filename(
cls_num, cost, suffix
)
model = load_file(model_file)
prediction = model.decision_function(features)
eval_preds, eval_cls_labels = self._get_cls_feats_labels(
cls_num, prediction, targets
)
P, R, score, ap = get_precision_recall(eval_cls_labels, eval_preds)
local_cost_ap[cls_num][0] = ap
mean_cost_ap = np.mean(local_cost_ap, axis=0)
sample_ap_matrix[0][cost_idx] = mean_cost_ap
out_k_sample_file = (
f"{self.output_dir}/test_ap_sample{sample_num}_k{low_shot_kvalue}.npy"
)
save_data = sample_ap_matrix.reshape((1, -1))
save_file(save_data, out_k_sample_file)
logging.info(
f"Saved sample test k_idx AP: {out_k_sample_file} {save_data.shape}"
)
def generate_voc07_low_shot_samples(targets, k_values, sample_inds,
output_path, layername):
k_values = [int(val) for val in k_values]
# the way sample works is: for each independent sample, and a given k value
# we create a matrix of the same shape as given targets file. We initialize
# this matrix with -1 (ignore label). We then sample k positive and
# (num_classes-1) * k negatives.
num_classes = targets.shape[1] # N x 20 shape
for idx in sample_inds:
for k in k_values:
logging.info(f"Sampling: {idx} time for k-value: {k}")
output = np.ones(targets.shape, dtype=np.int32) * -1
output = sample_symbol(targets, output, 1, k)
output = sample_symbol(targets, output, 0, (num_classes - 1) * k)
output_file = f"{output_path}/{layername}_sample{idx}_k{k}.npy"
logging.info(f"Saving file: {output_file}")
save_file(output, output_file)
logging.info("Done!!")
def create_iwilds_cam_disk_filelist(input_path: str, output_path: str):
meta_data_path = os.path.join(input_path, "metadata.csv")
image_folder = os.path.join(input_path, "train")
meta_data = pd.read_csv(meta_data_path)
splits = sorted(set(meta_data["split"]))
with tqdm(total=len(meta_data)) as pbar:
for split in splits:
image_paths, image_labels = [], []
split_meta_data = meta_data[meta_data["split"] == split]
split_meta_data = split_meta_data[["filename", "y"]]
for _, (file_name, y) in split_meta_data.iterrows():
image_paths.append(os.path.join(image_folder, file_name))
image_labels.append(y)
pbar.update(1)
image_paths_file = os.path.join(output_path, f"{split}_images.npy")
image_labels_file = os.path.join(output_path,
f"{split}_labels.npy")
save_file(np.array(image_paths), image_paths_file)
save_file(np.array(image_labels), image_labels_file)
def get_voc_images_labels_info(split, data_source_dir):
assert PathManager.exists(data_source_dir), "Data source NOT found. Abort"
data_files = get_data_files(split, data_source_dir)
# we will construct a map for image name to the vector of -1, 0, 1
# we sort the data_files which gives sorted class names as well
img_labels_map = {}
for cls_num, data_path in enumerate(sorted(data_files)):
# for this class, we have images and each image will have label
# 1, -1, 0 -> present, not present, ignore respectively as in VOC data.
with PathManager.open(data_path, "r") as fopen:
for line in fopen:
try:
img_name, orig_label = line.strip().split()
if img_name not in img_labels_map:
img_labels_map[img_name] = -(
np.ones(len(data_files), dtype=np.int32)
)
orig_label = int(orig_label)
# in VOC data, -1 (not present), set it to 0 as train target
if orig_label == -1:
orig_label = 0
# in VOC data, 0 (ignore), set it to -1 as train target
elif orig_label == 0:
orig_label = -1
img_labels_map[img_name][cls_num] = orig_label
except Exception:
logging.info(
"Error processing: {} data_path: {}".format(line, data_path)
)
img_paths, img_labels = [], []
for item in sorted(img_labels_map.keys()):
img_paths.append(f"{data_source_dir}/JPEGImages/{item}.jpg")
img_labels.append(img_labels_map[item])
# save to the datasets folder and return the path
output_dir = get_output_dir()
img_info_out_path = f"{output_dir}/{split}_images.npy"
label_info_out_path = f"{output_dir}/{split}_labels.npy"
save_file(np.array(img_paths), img_info_out_path)
save_file(np.array(img_labels), label_info_out_path)
return [img_info_out_path, label_info_out_path]
def test(self, features, targets):
"""
Test the trained SVM models on the test features and targets values.
We use the cost per class that gives the maximum cross validation AP on
the training and load the correspond trained SVM model for the cost value
and the class.
Log the test ap to stdout and also save the AP in a file.
"""
logging.info("Testing SVM")
# normalize the features: N x 9216 (example shape)
if self.normalize:
# normalize the features: N x 9216 (example shape)
features = self._normalize_features(features)
num_classes = targets.shape[1]
logging.info("Num test classes: {}".format(num_classes))
# get the chosen cost that maximizes the cross-validation AP per class
costs_list = self.get_best_cost_value()
ap_matrix = np.zeros((num_classes, 1))
for cls_num in range(num_classes):
cost = costs_list[cls_num]
logging.info(f"Testing model for cls: {cls_num} cost: {cost}")
model_file, _ = self._get_svm_model_filename(cls_num, cost)
model = load_file(model_file)
prediction = model.decision_function(features)
cls_labels = targets[:, cls_num]
# meaning of labels in VOC/COCO original loaded target files:
# label 0 = not present, set it to -1 as svm train target
# label 1 = present. Make the svm train target labels as -1, 1.
evaluate_data_inds = targets[:, cls_num] != -1
eval_preds = prediction[evaluate_data_inds]
eval_cls_labels = cls_labels[evaluate_data_inds]
eval_cls_labels[np.where(eval_cls_labels == 0)] = -1
P, R, score, ap = get_precision_recall(eval_cls_labels, eval_preds)
ap_matrix[cls_num][0] = ap
logging.info(f"Mean test AP: {np.mean(ap_matrix, axis=0)}")
test_ap_filepath = f"{self.output_dir}/test_ap.npy"
save_file(np.array(ap_matrix), test_ap_filepath)
logging.info(f"saved test AP to file: {test_ap_filepath}")
def _save_extracted_features(
features,
targets,
dist_rank: int,
chunk_index: int,
split: str,
output_folder: str,
):
output = {}
for layer_name in features.keys():
indices = sorted(features[layer_name].keys())
if len(indices) > 0:
output[layer_name] = {
"inds": np.array(indices),
"features": np.array([features[layer_name][i] for i in indices]),
"targets": np.array([targets[layer_name][i] for i in indices]),
}
for layer_name, layer_features in output.items():
out_feat_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_{layer_name}_features.npy",
)
out_target_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_{layer_name}_targets.npy",
)
out_inds_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_{layer_name}_inds.npy",
)
save_file(layer_features["features"], out_feat_file)
save_file(layer_features["targets"], out_target_file)
save_file(layer_features["inds"], out_inds_file)
def prepare_data(self, split: str, layer: str, num_shards: int,
feat_shape: Tuple[int, int]):
batch_size, feat_size = feat_shape
total = batch_size * num_shards
# Generate a dataset
indices = np.arange(0, total)
features = np.random.random(size=(total, feat_size))
targets = np.random.randint(low=0, high=10, size=(total, 1))
# Randomly shuffle it
permutation = np.random.permutation(total)
permuted_features = features[permutation]
permuted_targets = targets[permutation]
permuted_indices = indices[permutation]
# And save each part in shards
for i in range(num_shards):
shard_features = permuted_features[i * batch_size:(i + 1) *
batch_size]
shard_targets = permuted_targets[i * batch_size:(i + 1) *
batch_size]
shard_indices = permuted_indices[i * batch_size:(i + 1) *
batch_size]
save_file(shard_features, f"chunk{i}_{split}_{layer}_features.npy")
save_file(shard_targets, f"chunk{i}_{split}_{layer}_targets.npy")
save_file(shard_indices, f"chunk{i}_{split}_{layer}_inds.npy")
# Return the data used to generate the files
return indices, features, targets
def _save_extracted_prototypes(
soft_assignments: Dict[int, np.ndarray],
out_indices: List[int],
dist_rank: int,
chunk_index: int,
split: str,
output_folder: str,
):
out_indices = np.array(out_indices)
out_protos = np.concatenate([soft_assignments[i] for i in out_indices], axis=0)
out_proto_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_heads_protos.npy",
)
out_inds_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_heads_inds.npy",
)
logging.info(
f"Saving features: {out_protos.shape}, " f"inds: {out_indices.shape}"
)
save_file(out_protos, out_proto_file)
save_file(out_indices, out_inds_file)
def get_best_cost_value(self):
"""
During the SVM training, we write the cross vaildation
AP value for training at each class and cost value
combination. We load the AP values and for each
class, determine the cost value that gives the maximum
AP. We return the chosen cost values for each class as a
numpy matrix.
"""
crossval_ap_file = f"{self.output_dir}/crossval_ap.npy"
chosen_cost_file = f"{self.output_dir}/chosen_cost.npy"
if PathManager.exists(crossval_ap_file) and PathManager.exists(
chosen_cost_file):
self.chosen_cost = load_file(chosen_cost_file)
self.train_ap_matrix = load_file(crossval_ap_file)
return self.chosen_cost
if self.train_ap_matrix is None:
num_classes = len(self.cls_list)
self.train_ap_matrix = np.zeros(
(num_classes, len(self.costs_list)))
for cls_num in range(num_classes):
for cost_idx in range(len(self.costs_list)):
cost = self.costs_list[cost_idx]
_, ap_out_file = self._get_svm_model_filename(
cls_num, cost)
self.train_ap_matrix[cls_num][cost_idx] = float(
load_file(ap_out_file)[0])
argmax_cls = np.argmax(self.train_ap_matrix, axis=1)
chosen_cost = [self.costs_list[idx] for idx in argmax_cls]
logging.info(f"chosen_cost: {chosen_cost}")
save_file(np.array(self.train_ap_matrix), crossval_ap_file)
save_file(np.array(chosen_cost), chosen_cost_file)
logging.info(f"saved crossval_ap AP to file: {crossval_ap_file}")
logging.info(f"saved chosen costs to file: {chosen_cost_file}")
self.chosen_cost = chosen_cost
return np.array(chosen_cost)
def _save_extracted_features(
self,
features,
targets,
dist_rank: int,
chunk_index: int,
split: str,
output_folder: str,
):
output = {}
for layer_name in features.keys():
indices = sorted(features[layer_name].keys())
if len(indices) > 0:
feats = [features[layer_name][i] for i in indices]
if self._is_list_of_tensors_same_shape(features[layer_name]):
feats = np.array(feats)
else:
# If each tensor is not the same shape (e.g. images are of variable size)
# we need to create a np.array(dtype=object).
feats = np.array(
[features[layer_name][i].tolist() for i in indices])
output[layer_name] = {
"inds": np.array(indices),
"features": feats,
"targets":
np.array([targets[layer_name][i] for i in indices]),
}
for layer_name, layer_features in output.items():
out_feat_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_{layer_name}_features.npy",
)
out_target_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_{layer_name}_targets.npy",
)
out_inds_file = os.path.join(
output_folder,
f"rank{dist_rank}_chunk{chunk_index}_{split.lower()}_{layer_name}_inds.npy",
)
logging.info(
f"Saving features: {layer_features['features'].shape}, "
f"targets: {layer_features['targets'].shape}, "
f"inds: {layer_features['inds'].shape}")
save_file(layer_features["features"], out_feat_file)
save_file(layer_features["targets"], out_target_file)
save_file(layer_features["inds"], out_inds_file)
def _save_knn_results(
output_dir: str,
layer_name: str,
output_inds: List[int],
output_predicted_label: List[np.ndarray],
output_targets: List[np.ndarray],
):
output_targets = np.vstack(output_targets)
output_predicted_label = np.vstack(output_predicted_label)
logging.info(
f"Saving targets: {output_targets.shape}, "
f"output predictions: {output_predicted_label.shape}, "
f"output indices: {len(output_inds)}"
)
save_file(
output_predicted_label, f"{output_dir}/kNN_{layer_name}_output_predictions.npy"
)
save_file(output_targets, f"{output_dir}/kNN_{layer_name}_output_targets.npy")
save_file(output_inds, f"{output_dir}/kNN_{layer_name}_output_image_indices.npy")
def create_split(input_path: str, output_path: str, split: str, num_workers: int):
"""
Create one split of the disk_folder format and the associated disk_filelist files
"""
image_paths = []
image_labels = []
error_paths = []
# Create the disk_folder format
dataset = KineticsMiddleFrameDataset(data_path=input_path, split=split)
loader = DataLoader(
dataset, num_workers=num_workers, batch_size=1, collate_fn=lambda x: x[0]
)
for mid_frame, image_name, label, video_path in tqdm(loader, total=len(dataset)):
if mid_frame is not None:
label = clean_label(label)
label_folder = os.path.join(output_path, f"{split}_images", label)
os.makedirs(label_folder, exist_ok=True)
image_path = os.path.join(label_folder, image_name)
with open(image_path, "w") as image_file:
mid_frame.save(image_file)
image_paths.append(image_path)
image_labels.append(label)
else:
error_paths.append(video_path)
# Save the disk_filelist format
save_file(
np.array(image_paths), filename=os.path.join(output_path, f"{split}_images.npy")
)
save_file(
np.array(image_labels),
filename=os.path.join(output_path, f"{split}_labels.npy"),
)
if len(error_paths):
print(f"Number of errors in '{split}' split: {len(error_paths)}")
error_paths_file = os.path.join(output_path, f"{split}_errors.npy")
print(f"Errors are saved in: {error_paths_file}")
save_file(error_paths, filename=error_paths_file)
def save_attrdict_to_disk(cfg: AttrDict):
from vissl.utils.checkpoint import get_checkpoint_folder
yaml_output_file = f"{get_checkpoint_folder(cfg)}/train_config.yaml"
save_file(cfg.to_dict(), yaml_output_file)
