def _get_raw2label(self):
'''
ScanNet label classes in DC.type2class (18 classes):
0 'cabinet',
1 'bed',
2 'chair',
3 'sofa',
4 'table',
5 'door',
6 'window',
7 'bookshelf',
8 'picture',
9 'counter',
10 'desk',
11 'curtain',
12 'refrigerator',
13 'shower curtain',
14 'toilet',
15 'sink',
16 'bathtub',
17 'others'
'''
### Mapping NYU40 labeled classes to ScanNet labeled classes.
DC = ScannetDatasetConfig()
scannet_labels = DC.type2class.keys()
scannet2label = {label: i for i, label in enumerate(scannet_labels)}
raw2label = {}
for line in self.classes_map:
label_classes_set = set(scannet_labels)
ele = line.split('\t')
raw_name = ele[1]
nyu40_name = ele[7]
if nyu40_name not in label_classes_set:
raw2label[raw_name] = scannet2label['others']
else:
raw2label[raw_name] = scannet2label[nyu40_name]
return raw2label
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import os
import sys
sys.path.append(os.path.join(os.getcwd(), "lib")) # HACK add the lib folder
import lib.pointnet2.pointnet2_utils
from data.scannet.model_util_scannet import ScannetDatasetConfig
from lib.pointnet2.pointnet2_modules import PointnetSAModuleVotes
from utils.box_util import get_3d_box_batch
# constants
DC = ScannetDatasetConfig()
class ProposalModule(nn.Module):
def __init__(self, num_class, num_heading_bin, num_size_cluster, mean_size_arr, num_proposal, sampling, seed_feat_dim=256):
super().__init__()
self.num_class = num_class
self.num_heading_bin = num_heading_bin
self.num_size_cluster = num_size_cluster
self.mean_size_arr = mean_size_arr
self.num_proposal = num_proposal
self.sampling = sampling
self.seed_feat_dim = seed_feat_dim
# Vote clustering
self.vote_aggregation = PointnetSAModuleVotes(
def predict(args):
print("predict bounding boxes...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "test", DC)
# model
model = get_model(args, DC)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
} if not args.no_nms else None
# predict
print("predicting...")
pred_bboxes = []
for data_dict in tqdm(dataloader):
for key in data_dict:
data_dict[key] = data_dict[key].cuda()
# feed
data_dict = model(data_dict)
_, data_dict = get_loss(
data_dict=data_dict,
config=DC,
detection=False,
reference=True
)
objectness_preds_batch = torch.argmax(data_dict['objectness_scores'], 2).long()
if POST_DICT:
_ = parse_predictions(data_dict, POST_DICT)
nms_masks = torch.LongTensor(data_dict['pred_mask']).cuda()
# construct valid mask
pred_masks = (nms_masks * objectness_preds_batch == 1).float()
else:
# construct valid mask
pred_masks = (objectness_preds_batch == 1).float()
pred_ref = torch.argmax(data_dict['cluster_ref'] * pred_masks, 1) # (B,)
pred_center = data_dict['center'] # (B,K,3)
pred_heading_class = torch.argmax(data_dict['heading_scores'], -1) # B,num_proposal
pred_heading_residual = torch.gather(data_dict['heading_residuals'], 2, pred_heading_class.unsqueeze(-1)) # B,num_proposal,1
pred_heading_class = pred_heading_class # B,num_proposal
pred_heading_residual = pred_heading_residual.squeeze(2) # B,num_proposal
pred_size_class = torch.argmax(data_dict['size_scores'], -1) # B,num_proposal
pred_size_residual = torch.gather(data_dict['size_residuals'], 2, pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(1,1,1,3)) # B,num_proposal,1,3
pred_size_class = pred_size_class
pred_size_residual = pred_size_residual.squeeze(2) # B,num_proposal,3
for i in range(pred_ref.shape[0]):
# compute the iou
pred_ref_idx = pred_ref[i]
pred_obb = DC.param2obb(
pred_center[i, pred_ref_idx, 0:3].detach().cpu().numpy(),
pred_heading_class[i, pred_ref_idx].detach().cpu().numpy(),
pred_heading_residual[i, pred_ref_idx].detach().cpu().numpy(),
pred_size_class[i, pred_ref_idx].detach().cpu().numpy(),
pred_size_residual[i, pred_ref_idx].detach().cpu().numpy()
)
pred_bbox = get_3d_box(pred_obb[3:6], pred_obb[6], pred_obb[0:3])
# construct the multiple mask
multiple = data_dict["unique_multiple"][i].item()
# construct the others mask
others = 1 if data_dict["object_cat"][i] == 17 else 0
# store data
scanrefer_idx = data_dict["scan_idx"][i].item()
pred_data = {
"scene_id": scanrefer[scanrefer_idx]["scene_id"],
"object_id": scanrefer[scanrefer_idx]["object_id"],
"ann_id": scanrefer[scanrefer_idx]["ann_id"],
"bbox": pred_bbox.tolist(),
"unique_multiple": multiple,
"others": others
}
pred_bboxes.append(pred_data)
# dump
print("dumping...")
pred_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "pred.json")
with open(pred_path, "w") as f:
json.dump(pred_bboxes, f, indent=4)
print("done!")
def eval_ref(args):
print("evaluate...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer()
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val")
# model
model = get_model(args)
# random seeds
seeds = [args.manual_seed]
# evaluate
print("evaluating...")
score_path = os.path.join(args.use_checkpoint, "scores.p")
pred_path = os.path.join(args.use_checkpoint, "predictions.p")
if not os.path.exists(score_path):
ref_acc_all = []
ious_all = []
masks_all = []
others_all = []
lang_acc_all = []
for seed in seeds:
# reproducibility
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
print("generating the scores for seed {}...".format(seed))
ref_acc = []
ious = []
masks = []
others = []
lang_acc = []
predictions = {}
for data in tqdm(dataloader):
for key in data:
if key in ['lang_feat', 'lang_len', 'object_cat', 'lidar', 'point_min', 'point_max', 'mlm_label',
'ref_center_label', 'ref_size_residual_label']:
data[key] = data[key].cuda()
# feed
data = model(data)
data = get_loss(data, DC)
data = get_eval(
data_dict=data,
config=DC,
)
ref_acc += data["ref_acc"]
ious += data["ref_iou"]
masks += data["ref_multiple_mask"]
others += data["ref_others_mask"]
lang_acc.append(data["lang_acc"].item())
# store predictions
ids = data["scan_idx"].detach().cpu().numpy()
for i in range(ids.shape[0]):
idx = ids[i]
scene_id = scanrefer[idx]["scene_id"]
object_id = scanrefer[idx]["object_id"]
ann_id = scanrefer[idx]["ann_id"]
if scene_id not in predictions:
predictions[scene_id] = {}
if object_id not in predictions[scene_id]:
predictions[scene_id][object_id] = {}
if ann_id not in predictions[scene_id][object_id]:
predictions[scene_id][object_id][ann_id] = {}
predictions[scene_id][object_id][ann_id][""] = data["pred_bboxes"][i]
predictions[scene_id][object_id][ann_id]["gt_bpred_bboxbox"] = data["gt_bboxes"][i]
predictions[scene_id][object_id][ann_id]["iou"] = data["ref_iou"][i]
# save the last predictions
with open(pred_path, "wb") as f:
pickle.dump(predictions, f)
# save to global
ref_acc_all.append(ref_acc)
ious_all.append(ious)
masks_all.append(masks)
others_all.append(others)
lang_acc_all.append(lang_acc)
# convert to numpy array
ref_acc = np.array(ref_acc_all)
ious = np.array(ious_all)
masks = np.array(masks_all)
others = np.array(others_all)
lang_acc = np.array(lang_acc_all)
# save the global scores
with open(score_path, "wb") as f:
scores = {
"ref_acc": ref_acc_all,
"ious": ious_all,
"masks": masks_all,
"others": others_all,
"lang_acc": lang_acc_all
}
pickle.dump(scores, f)
else:
print("loading the scores...")
with open(score_path, "rb") as f:
scores = pickle.load(f)
# unpack
ref_acc = np.array(scores["ref_acc"])
ious = np.array(scores["ious"])
masks = np.array(scores["masks"])
others = np.array(scores["others"])
lang_acc = np.array(scores["lang_acc"])
multiple_dict = {
"unique": 0,
"multiple": 1
}
others_dict = {
"not_in_others": 0,
"in_others": 1
}
# evaluation stats
stats = {k: np.sum(masks[0] == v) for k, v in multiple_dict.items()}
stats["overall"] = masks[0].shape[0]
stats = {}
for k, v in multiple_dict.items():
stats[k] = {}
for k_o, v_o in others_dict.items():
stats[k][k_o] = np.sum(np.logical_and(masks[0] == v, others[0] == v_o))
stats[k]["overall"] = np.sum(masks[0] == v)
stats["overall"] = {}
for k_o, v_o in others_dict.items():
stats["overall"][k_o] = np.sum(others[0] == v_o)
stats["overall"]["overall"] = masks[0].shape[0]
# aggregate scores
scores = {}
for k, v in multiple_dict.items():
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(
ref_acc[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])]) \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(
np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]),
ious[i] >= 0.25)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k],
others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(
np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.5)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k],
others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
if k not in scores:
scores[k] = {k_o: {} for k_o in others_dict.keys()}
scores[k][k_o]["ref_acc"] = np.mean(ref_accs)
scores[k][k_o]["[email protected]"] = np.mean(acc_025ious)
scores[k][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i][masks[i] == multiple_dict[k]]) if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.25)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.5)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
scores[k]["overall"] = {}
scores[k]["overall"]["ref_acc"] = np.mean(ref_accs)
scores[k]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores[k]["overall"]["[email protected]"] = np.mean(acc_05ious)
scores["overall"] = {}
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i][others[i] == others_dict[k_o]]) if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.25)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.5)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"][k_o] = {}
scores["overall"][k_o]["ref_acc"] = np.mean(ref_accs)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_025ious)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i])
running_acc_025iou = ious[i][ious[i] >= 0.25].shape[0] / ious[i].shape[0]
running_acc_05iou = ious[i][ious[i] >= 0.5].shape[0] / ious[i].shape[0]
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"]["overall"] = {}
scores["overall"]["overall"]["ref_acc"] = np.mean(ref_accs)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_05ious)
# report
print("\nstats:")
for k_s in stats.keys():
for k_o in stats[k_s].keys():
print("{} | {}: {}".format(k_s, k_o, stats[k_s][k_o]))
for k_s in scores.keys():
print("\n{}:".format(k_s))
for k_m in scores[k_s].keys():
for metric in scores[k_s][k_m].keys():
print("{} | {} | {}: {:.4f}".format(k_s, k_m, metric, scores[k_s][k_m][metric]))
print("\nlanguage classification accuracy: {:.4f}".format(np.mean(lang_acc)))
def eval_detection(args):
print("evaluate detection...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
# get eval data
scanrefer_eval, eval_scene_list = get_eval_data(args)
# get dataloader
dataset, dataloader = get_dataloader(args, scanrefer_eval, eval_scene_list,
DC)
# model
print("initializing...")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
root = CONF.PATH.PRETRAINED if args.eval_pretrained else CONF.PATH.OUTPUT
model = get_model(args, dataset, device, root)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
}
AP_IOU_THRESHOLDS = [0.25, 0.5]
AP_CALCULATOR_LIST = [
APCalculator(iou_thresh, DC.class2type)
for iou_thresh in AP_IOU_THRESHOLDS
]
sem_acc = []
for data in tqdm(dataloader):
for key in data:
data[key] = data[key].cuda()
# feed
with torch.no_grad():
data = model(data, use_tf=False, is_eval=True)
data = get_scene_cap_loss(data,
device,
DC,
weights=dataset.weights,
detection=True,
caption=False)
batch_pred_map_cls = parse_predictions(data, POST_DICT)
batch_gt_map_cls = parse_groundtruths(data, POST_DICT)
for ap_calculator in AP_CALCULATOR_LIST:
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
# aggregate object detection results and report
for i, ap_calculator in enumerate(AP_CALCULATOR_LIST):
print()
print("-" * 10, "iou_thresh: %f" % (AP_IOU_THRESHOLDS[i]), "-" * 10)
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
print("eval %s: %f" % (key, metrics_dict[key]))
def eval_det(args):
print("evaluate detection...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC)
# model
model = get_model(args, DC)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
}
AP_IOU_THRESHOLDS = [0.25, 0.5]
AP_CALCULATOR_LIST = [
APCalculator(iou_thresh, DC.class2type)
for iou_thresh in AP_IOU_THRESHOLDS
]
sem_acc = []
for data in tqdm(dataloader):
for key in data:
data[key] = data[key].cuda()
# feed
with torch.no_grad():
data = model(data)
_, data = get_loss(data_dict=data,
config=DC,
detection=True,
reference=False)
data = get_eval(data_dict=data,
config=DC,
reference=False,
post_processing=POST_DICT)
sem_acc.append(data["sem_acc"].item())
batch_pred_map_cls = parse_predictions(data, POST_DICT)
batch_gt_map_cls = parse_groundtruths(data, POST_DICT)
for ap_calculator in AP_CALCULATOR_LIST:
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
# aggregate object detection results and report
print("\nobject detection sem_acc: {}".format(np.mean(sem_acc)))
for i, ap_calculator in enumerate(AP_CALCULATOR_LIST):
print()
print("-" * 10, "iou_thresh: %f" % (AP_IOU_THRESHOLDS[i]), "-" * 10)
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
print("eval %s: %f" % (key, metrics_dict[key]))
def eval_ref(args):
print("evaluate localization...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC)
# model
model = get_model(args, DC)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
} if not args.no_nms else None
# random seeds
seeds = [args.seed] + [2 * i for i in range(args.repeat - 1)]
# evaluate
print("evaluating...")
score_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "scores.p")
pred_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "predictions.p")
gen_flag = (
not os.path.exists(score_path)) or args.force or args.repeat > 1
if gen_flag:
ref_acc_all = []
ious_all = []
masks_all = []
others_all = []
lang_acc_all = []
for seed in seeds:
# reproducibility
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
print("generating the scores for seed {}...".format(seed))
ref_acc = []
ious = []
masks = []
others = []
lang_acc = []
predictions = {}
for data in tqdm(dataloader):
print(data.keys())
print(data['lang_feat'].shape)
assert (1 == 2)
for key in data:
data[key] = data[key].cuda()
# feed
data = model(data)
_, data = get_loss(data_dict=data,
config=DC,
detection=True,
reference=True,
use_lang_classifier=not args.no_lang_cls)
data = get_eval(data_dict=data,
config=DC,
reference=True,
use_lang_classifier=not args.no_lang_cls,
use_oracle=args.use_oracle,
use_cat_rand=args.use_cat_rand,
use_best=args.use_best,
post_processing=POST_DICT)
ref_acc += data["ref_acc"]
ious += data["ref_iou"]
masks += data["ref_multiple_mask"]
others += data["ref_others_mask"]
lang_acc.append(data["lang_acc"].item())
# store predictions
ids = data["scan_idx"].detach().cpu().numpy()
for i in range(ids.shape[0]):
idx = ids[i]
scene_id = scanrefer[idx]["scene_id"]
object_id = scanrefer[idx]["object_id"]
ann_id = scanrefer[idx]["ann_id"]
if scene_id not in predictions:
predictions[scene_id] = {}
if object_id not in predictions[scene_id]:
predictions[scene_id][object_id] = {}
if ann_id not in predictions[scene_id][object_id]:
predictions[scene_id][object_id][ann_id] = {}
predictions[scene_id][object_id][ann_id][
"pred_bbox"] = data["pred_bboxes"][i]
predictions[scene_id][object_id][ann_id]["gt_bbox"] = data[
"gt_bboxes"][i]
predictions[scene_id][object_id][ann_id]["iou"] = data[
"ref_iou"][i]
# save the last predictions
with open(pred_path, "wb") as f:
pickle.dump(predictions, f)
# save to global
ref_acc_all.append(ref_acc)
ious_all.append(ious)
masks_all.append(masks)
others_all.append(others)
lang_acc_all.append(lang_acc)
# convert to numpy array
ref_acc = np.array(ref_acc_all)
ious = np.array(ious_all)
masks = np.array(masks_all)
others = np.array(others_all)
lang_acc = np.array(lang_acc_all)
# save the global scores
with open(score_path, "wb") as f:
scores = {
"ref_acc": ref_acc_all,
"ious": ious_all,
"masks": masks_all,
"others": others_all,
"lang_acc": lang_acc_all
}
pickle.dump(scores, f)
else:
print("loading the scores...")
with open(score_path, "rb") as f:
scores = pickle.load(f)
# unpack
ref_acc = np.array(scores["ref_acc"])
ious = np.array(scores["ious"])
masks = np.array(scores["masks"])
others = np.array(scores["others"])
lang_acc = np.array(scores["lang_acc"])
multiple_dict = {"unique": 0, "multiple": 1}
others_dict = {"not_in_others": 0, "in_others": 1}
# evaluation stats
stats = {k: np.sum(masks[0] == v) for k, v in multiple_dict.items()}
stats["overall"] = masks[0].shape[0]
stats = {}
for k, v in multiple_dict.items():
stats[k] = {}
for k_o, v_o in others_dict.items():
stats[k][k_o] = np.sum(
np.logical_and(masks[0] == v, others[0] == v_o))
stats[k]["overall"] = np.sum(masks[0] == v)
stats["overall"] = {}
for k_o, v_o in others_dict.items():
stats["overall"][k_o] = np.sum(others[0] == v_o)
stats["overall"]["overall"] = masks[0].shape[0]
# aggregate scores
scores = {}
for k, v in multiple_dict.items():
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])]) \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.25)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.5)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
if k not in scores:
scores[k] = {k_o: {} for k_o in others_dict.keys()}
scores[k][k_o]["ref_acc"] = np.mean(ref_accs)
scores[k][k_o]["[email protected]"] = np.mean(acc_025ious)
scores[k][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(
ref_acc[i][masks[i] == multiple_dict[k]]) if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.25)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(masks[i] == multiple_dict[k]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.5)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(masks[i] == multiple_dict[k]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
scores[k]["overall"] = {}
scores[k]["overall"]["ref_acc"] = np.mean(ref_accs)
scores[k]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores[k]["overall"]["[email protected]"] = np.mean(acc_05ious)
scores["overall"] = {}
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(
ref_acc[i][others[i] == others_dict[k_o]]) if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.25)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(others[i] == others_dict[k_o]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.5)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(others[i] == others_dict[k_o]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"][k_o] = {}
scores["overall"][k_o]["ref_acc"] = np.mean(ref_accs)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_025ious)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i])
running_acc_025iou = ious[i][
ious[i] >= 0.25].shape[0] / ious[i].shape[0]
running_acc_05iou = ious[i][ious[i] >= 0.5].shape[0] / ious[i].shape[0]
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"]["overall"] = {}
scores["overall"]["overall"]["ref_acc"] = np.mean(ref_accs)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_05ious)
# report
print("\nstats:")
for k_s in stats.keys():
for k_o in stats[k_s].keys():
print("{} | {}: {}".format(k_s, k_o, stats[k_s][k_o]))
for k_s in scores.keys():
print("\n{}:".format(k_s))
for k_m in scores[k_s].keys():
for metric in scores[k_s][k_m].keys():
print("{} | {} | {}: {}".format(k_s, k_m, metric,
scores[k_s][k_m][metric]))
print("\nlanguage classification accuracy: {}".format(np.mean(lang_acc)))
