def _compute_loss(self, data_dict):
data_dict = get_loss(
data_dict=data_dict,
config=self.config,
)
# dump
self._running_log["ref_loss"] = data_dict["ref_loss"]
self._running_log["lang_loss"] = data_dict["lang_loss"]
self._running_log["seg_loss"] = data_dict["seg_loss"]
self._running_log["loss"] = data_dict["loss"]
def _compute_loss(self, data_dict):
_, data_dict = get_loss(data_dict, self.config, True,
self.use_lang_classifier, self.use_max_iou)
# dump
self._running_log["ref_loss"] = data_dict["ref_loss"]
self._running_log["lang_loss"] = data_dict["lang_loss"]
self._running_log["objectness_loss"] = data_dict["objectness_loss"]
self._running_log["vote_loss"] = data_dict["vote_loss"]
self._running_log["box_loss"] = data_dict["box_loss"]
self._running_log["loss"] = data_dict["loss"]
def _compute_loss(self, data_dict):
_, data_dict = get_loss(data_dict=data_dict,
config=self.config,
detection=self.detection,
reference=self.reference,
use_lang_classifier=self.use_lang_classifier)
# dump
self._running_log["ref_loss"] = data_dict["ref_loss"]
self._running_log["lang_loss"] = data_dict["lang_loss"]
self._running_log["objectness_loss"] = data_dict["objectness_loss"]
self._running_log["vote_loss"] = data_dict["vote_loss"]
self._running_log["box_loss"] = data_dict["box_loss"]
self._running_log["loss"] = data_dict["loss"]
def visualize(args):
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC, False)
# model
model = get_model(args)
# config
POST_DICT = {
'remove_empty_box': True,
'use_3d_nms': True,
'nms_iou': 0.25,
'use_old_type_nms': False,
'cls_nms': True,
'per_class_proposal': True,
'conf_thresh': 0.05,
'dataset_config': DC
} if not args.no_nms else None
# evaluate
print("visualizing...")
for data in tqdm(dataloader):
for key in data:
data[key] = data[key].cuda()
# feed
data = model(data)
# _, data = get_loss(data, DC, True, True, POST_DICT)
_, data = get_loss(
data_dict=data,
config=DC,
detection=True,
reference=True
)
data = get_eval(
data_dict=data,
config=DC,
reference=True,
post_processing=POST_DICT
)
# visualize
dump_results(args, scanrefer, data, DC)
print("done!")
def _compute_loss(self, data_dict):
_, data_dict = get_loss(data_dict=data_dict,
config=self.config,
detection=self.detection,
reference=self.reference,
pg=self.pg,
use_lang_classifier=self.use_lang_classifier,
loss_weights=self.loss_weights)
# dump
self._running_log["ref_loss"] = data_dict["ref_loss"]
self._running_log["lang_loss"] = data_dict["lang_loss"]
self._running_log["pg_loss"] = data_dict["pg_loss"]
# PointGroup:
# no objectness loss, vote loss and box loss
#self._running_log["objectness_loss"] = data_dict["objectness_loss"]
#self._running_log["vote_loss"] = data_dict["vote_loss"]
#self._running_log["box_loss"] = data_dict["box_loss"]
self._running_log["loss"] = data_dict["loss"]
def predict(args):
print("predict bounding boxes...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "test", DC)
# model
model = get_model(args, DC)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
} if not args.no_nms else None
# predict
print("predicting...")
pred_bboxes = []
for data_dict in tqdm(dataloader):
for key in data_dict:
data_dict[key] = data_dict[key].cuda()
# feed
data_dict = model(data_dict)
_, data_dict = get_loss(
data_dict=data_dict,
config=DC,
detection=False,
reference=True
)
objectness_preds_batch = torch.argmax(data_dict['objectness_scores'], 2).long()
if POST_DICT:
_ = parse_predictions(data_dict, POST_DICT)
nms_masks = torch.LongTensor(data_dict['pred_mask']).cuda()
# construct valid mask
pred_masks = (nms_masks * objectness_preds_batch == 1).float()
else:
# construct valid mask
pred_masks = (objectness_preds_batch == 1).float()
pred_ref = torch.argmax(data_dict['cluster_ref'] * pred_masks, 1) # (B,)
pred_center = data_dict['center'] # (B,K,3)
pred_heading_class = torch.argmax(data_dict['heading_scores'], -1) # B,num_proposal
pred_heading_residual = torch.gather(data_dict['heading_residuals'], 2, pred_heading_class.unsqueeze(-1)) # B,num_proposal,1
pred_heading_class = pred_heading_class # B,num_proposal
pred_heading_residual = pred_heading_residual.squeeze(2) # B,num_proposal
pred_size_class = torch.argmax(data_dict['size_scores'], -1) # B,num_proposal
pred_size_residual = torch.gather(data_dict['size_residuals'], 2, pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(1,1,1,3)) # B,num_proposal,1,3
pred_size_class = pred_size_class
pred_size_residual = pred_size_residual.squeeze(2) # B,num_proposal,3
for i in range(pred_ref.shape[0]):
# compute the iou
pred_ref_idx = pred_ref[i]
pred_obb = DC.param2obb(
pred_center[i, pred_ref_idx, 0:3].detach().cpu().numpy(),
pred_heading_class[i, pred_ref_idx].detach().cpu().numpy(),
pred_heading_residual[i, pred_ref_idx].detach().cpu().numpy(),
pred_size_class[i, pred_ref_idx].detach().cpu().numpy(),
pred_size_residual[i, pred_ref_idx].detach().cpu().numpy()
)
pred_bbox = get_3d_box(pred_obb[3:6], pred_obb[6], pred_obb[0:3])
# construct the multiple mask
multiple = data_dict["unique_multiple"][i].item()
# construct the others mask
others = 1 if data_dict["object_cat"][i] == 17 else 0
# store data
scanrefer_idx = data_dict["scan_idx"][i].item()
pred_data = {
"scene_id": scanrefer[scanrefer_idx]["scene_id"],
"object_id": scanrefer[scanrefer_idx]["object_id"],
"ann_id": scanrefer[scanrefer_idx]["ann_id"],
"bbox": pred_bbox.tolist(),
"unique_multiple": multiple,
"others": others
}
pred_bboxes.append(pred_data)
# dump
print("dumping...")
pred_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "pred.json")
with open(pred_path, "w") as f:
json.dump(pred_bboxes, f, indent=4)
print("done!")
def eval_ref(args):
print("evaluate...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer()
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val")
# model
model = get_model(args)
# random seeds
seeds = [args.manual_seed]
# evaluate
print("evaluating...")
score_path = os.path.join(args.use_checkpoint, "scores.p")
pred_path = os.path.join(args.use_checkpoint, "predictions.p")
if not os.path.exists(score_path):
ref_acc_all = []
ious_all = []
masks_all = []
others_all = []
lang_acc_all = []
for seed in seeds:
# reproducibility
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
print("generating the scores for seed {}...".format(seed))
ref_acc = []
ious = []
masks = []
others = []
lang_acc = []
predictions = {}
for data in tqdm(dataloader):
for key in data:
if key in ['lang_feat', 'lang_len', 'object_cat', 'lidar', 'point_min', 'point_max', 'mlm_label',
'ref_center_label', 'ref_size_residual_label']:
data[key] = data[key].cuda()
# feed
data = model(data)
data = get_loss(data, DC)
data = get_eval(
data_dict=data,
config=DC,
)
ref_acc += data["ref_acc"]
ious += data["ref_iou"]
masks += data["ref_multiple_mask"]
others += data["ref_others_mask"]
lang_acc.append(data["lang_acc"].item())
# store predictions
ids = data["scan_idx"].detach().cpu().numpy()
for i in range(ids.shape[0]):
idx = ids[i]
scene_id = scanrefer[idx]["scene_id"]
object_id = scanrefer[idx]["object_id"]
ann_id = scanrefer[idx]["ann_id"]
if scene_id not in predictions:
predictions[scene_id] = {}
if object_id not in predictions[scene_id]:
predictions[scene_id][object_id] = {}
if ann_id not in predictions[scene_id][object_id]:
predictions[scene_id][object_id][ann_id] = {}
predictions[scene_id][object_id][ann_id][""] = data["pred_bboxes"][i]
predictions[scene_id][object_id][ann_id]["gt_bpred_bboxbox"] = data["gt_bboxes"][i]
predictions[scene_id][object_id][ann_id]["iou"] = data["ref_iou"][i]
# save the last predictions
with open(pred_path, "wb") as f:
pickle.dump(predictions, f)
# save to global
ref_acc_all.append(ref_acc)
ious_all.append(ious)
masks_all.append(masks)
others_all.append(others)
lang_acc_all.append(lang_acc)
# convert to numpy array
ref_acc = np.array(ref_acc_all)
ious = np.array(ious_all)
masks = np.array(masks_all)
others = np.array(others_all)
lang_acc = np.array(lang_acc_all)
# save the global scores
with open(score_path, "wb") as f:
scores = {
"ref_acc": ref_acc_all,
"ious": ious_all,
"masks": masks_all,
"others": others_all,
"lang_acc": lang_acc_all
}
pickle.dump(scores, f)
else:
print("loading the scores...")
with open(score_path, "rb") as f:
scores = pickle.load(f)
# unpack
ref_acc = np.array(scores["ref_acc"])
ious = np.array(scores["ious"])
masks = np.array(scores["masks"])
others = np.array(scores["others"])
lang_acc = np.array(scores["lang_acc"])
multiple_dict = {
"unique": 0,
"multiple": 1
}
others_dict = {
"not_in_others": 0,
"in_others": 1
}
# evaluation stats
stats = {k: np.sum(masks[0] == v) for k, v in multiple_dict.items()}
stats["overall"] = masks[0].shape[0]
stats = {}
for k, v in multiple_dict.items():
stats[k] = {}
for k_o, v_o in others_dict.items():
stats[k][k_o] = np.sum(np.logical_and(masks[0] == v, others[0] == v_o))
stats[k]["overall"] = np.sum(masks[0] == v)
stats["overall"] = {}
for k_o, v_o in others_dict.items():
stats["overall"][k_o] = np.sum(others[0] == v_o)
stats["overall"]["overall"] = masks[0].shape[0]
# aggregate scores
scores = {}
for k, v in multiple_dict.items():
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(
ref_acc[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])]) \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(
np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]),
ious[i] >= 0.25)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k],
others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(
np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.5)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k],
others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
if k not in scores:
scores[k] = {k_o: {} for k_o in others_dict.keys()}
scores[k][k_o]["ref_acc"] = np.mean(ref_accs)
scores[k][k_o]["[email protected]"] = np.mean(acc_025ious)
scores[k][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i][masks[i] == multiple_dict[k]]) if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.25)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.5)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
scores[k]["overall"] = {}
scores[k]["overall"]["ref_acc"] = np.mean(ref_accs)
scores[k]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores[k]["overall"]["[email protected]"] = np.mean(acc_05ious)
scores["overall"] = {}
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i][others[i] == others_dict[k_o]]) if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.25)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.5)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"][k_o] = {}
scores["overall"][k_o]["ref_acc"] = np.mean(ref_accs)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_025ious)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i])
running_acc_025iou = ious[i][ious[i] >= 0.25].shape[0] / ious[i].shape[0]
running_acc_05iou = ious[i][ious[i] >= 0.5].shape[0] / ious[i].shape[0]
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"]["overall"] = {}
scores["overall"]["overall"]["ref_acc"] = np.mean(ref_accs)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_05ious)
# report
print("\nstats:")
for k_s in stats.keys():
for k_o in stats[k_s].keys():
print("{} | {}: {}".format(k_s, k_o, stats[k_s][k_o]))
for k_s in scores.keys():
print("\n{}:".format(k_s))
for k_m in scores[k_s].keys():
for metric in scores[k_s][k_m].keys():
print("{} | {} | {}: {:.4f}".format(k_s, k_m, metric, scores[k_s][k_m][metric]))
print("\nlanguage classification accuracy: {:.4f}".format(np.mean(lang_acc)))
def eval_det(args):
print("evaluate detection...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC)
# model
model = get_model(args, DC)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
}
AP_IOU_THRESHOLDS = [0.25, 0.5]
AP_CALCULATOR_LIST = [
APCalculator(iou_thresh, DC.class2type)
for iou_thresh in AP_IOU_THRESHOLDS
]
sem_acc = []
for data in tqdm(dataloader):
for key in data:
data[key] = data[key].cuda()
# feed
with torch.no_grad():
data = model(data)
_, data = get_loss(data_dict=data,
config=DC,
detection=True,
reference=False)
data = get_eval(data_dict=data,
config=DC,
reference=False,
post_processing=POST_DICT)
sem_acc.append(data["sem_acc"].item())
batch_pred_map_cls = parse_predictions(data, POST_DICT)
batch_gt_map_cls = parse_groundtruths(data, POST_DICT)
for ap_calculator in AP_CALCULATOR_LIST:
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
# aggregate object detection results and report
print("\nobject detection sem_acc: {}".format(np.mean(sem_acc)))
for i, ap_calculator in enumerate(AP_CALCULATOR_LIST):
print()
print("-" * 10, "iou_thresh: %f" % (AP_IOU_THRESHOLDS[i]), "-" * 10)
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
print("eval %s: %f" % (key, metrics_dict[key]))
def eval_ref(args):
print("evaluate localization...")
# constant
DC = ScannetDatasetConfig()
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC)
# model
model = get_model(args, DC)
# config
POST_DICT = {
"remove_empty_box": True,
"use_3d_nms": True,
"nms_iou": 0.25,
"use_old_type_nms": False,
"cls_nms": True,
"per_class_proposal": True,
"conf_thresh": 0.05,
"dataset_config": DC
} if not args.no_nms else None
# random seeds
seeds = [args.seed] + [2 * i for i in range(args.repeat - 1)]
# evaluate
print("evaluating...")
score_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "scores.p")
pred_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "predictions.p")
gen_flag = (
not os.path.exists(score_path)) or args.force or args.repeat > 1
if gen_flag:
ref_acc_all = []
ious_all = []
masks_all = []
others_all = []
lang_acc_all = []
for seed in seeds:
# reproducibility
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
print("generating the scores for seed {}...".format(seed))
ref_acc = []
ious = []
masks = []
others = []
lang_acc = []
predictions = {}
for data in tqdm(dataloader):
print(data.keys())
print(data['lang_feat'].shape)
assert (1 == 2)
for key in data:
data[key] = data[key].cuda()
# feed
data = model(data)
_, data = get_loss(data_dict=data,
config=DC,
detection=True,
reference=True,
use_lang_classifier=not args.no_lang_cls)
data = get_eval(data_dict=data,
config=DC,
reference=True,
use_lang_classifier=not args.no_lang_cls,
use_oracle=args.use_oracle,
use_cat_rand=args.use_cat_rand,
use_best=args.use_best,
post_processing=POST_DICT)
ref_acc += data["ref_acc"]
ious += data["ref_iou"]
masks += data["ref_multiple_mask"]
others += data["ref_others_mask"]
lang_acc.append(data["lang_acc"].item())
# store predictions
ids = data["scan_idx"].detach().cpu().numpy()
for i in range(ids.shape[0]):
idx = ids[i]
scene_id = scanrefer[idx]["scene_id"]
object_id = scanrefer[idx]["object_id"]
ann_id = scanrefer[idx]["ann_id"]
if scene_id not in predictions:
predictions[scene_id] = {}
if object_id not in predictions[scene_id]:
predictions[scene_id][object_id] = {}
if ann_id not in predictions[scene_id][object_id]:
predictions[scene_id][object_id][ann_id] = {}
predictions[scene_id][object_id][ann_id][
"pred_bbox"] = data["pred_bboxes"][i]
predictions[scene_id][object_id][ann_id]["gt_bbox"] = data[
"gt_bboxes"][i]
predictions[scene_id][object_id][ann_id]["iou"] = data[
"ref_iou"][i]
# save the last predictions
with open(pred_path, "wb") as f:
pickle.dump(predictions, f)
# save to global
ref_acc_all.append(ref_acc)
ious_all.append(ious)
masks_all.append(masks)
others_all.append(others)
lang_acc_all.append(lang_acc)
# convert to numpy array
ref_acc = np.array(ref_acc_all)
ious = np.array(ious_all)
masks = np.array(masks_all)
others = np.array(others_all)
lang_acc = np.array(lang_acc_all)
# save the global scores
with open(score_path, "wb") as f:
scores = {
"ref_acc": ref_acc_all,
"ious": ious_all,
"masks": masks_all,
"others": others_all,
"lang_acc": lang_acc_all
}
pickle.dump(scores, f)
else:
print("loading the scores...")
with open(score_path, "rb") as f:
scores = pickle.load(f)
# unpack
ref_acc = np.array(scores["ref_acc"])
ious = np.array(scores["ious"])
masks = np.array(scores["masks"])
others = np.array(scores["others"])
lang_acc = np.array(scores["lang_acc"])
multiple_dict = {"unique": 0, "multiple": 1}
others_dict = {"not_in_others": 0, "in_others": 1}
# evaluation stats
stats = {k: np.sum(masks[0] == v) for k, v in multiple_dict.items()}
stats["overall"] = masks[0].shape[0]
stats = {}
for k, v in multiple_dict.items():
stats[k] = {}
for k_o, v_o in others_dict.items():
stats[k][k_o] = np.sum(
np.logical_and(masks[0] == v, others[0] == v_o))
stats[k]["overall"] = np.sum(masks[0] == v)
stats["overall"] = {}
for k_o, v_o in others_dict.items():
stats["overall"][k_o] = np.sum(others[0] == v_o)
stats["overall"]["overall"] = masks[0].shape[0]
# aggregate scores
scores = {}
for k, v in multiple_dict.items():
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])]) \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.25)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.5)].shape[0] \
/ ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \
if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
if k not in scores:
scores[k] = {k_o: {} for k_o in others_dict.keys()}
scores[k][k_o]["ref_acc"] = np.mean(ref_accs)
scores[k][k_o]["[email protected]"] = np.mean(acc_025ious)
scores[k][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(
ref_acc[i][masks[i] == multiple_dict[k]]) if np.sum(
masks[i] == multiple_dict[k]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.25)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(masks[i] == multiple_dict[k]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.5)].shape[0] \
/ ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(masks[i] == multiple_dict[k]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
scores[k]["overall"] = {}
scores[k]["overall"]["ref_acc"] = np.mean(ref_accs)
scores[k]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores[k]["overall"]["[email protected]"] = np.mean(acc_05ious)
scores["overall"] = {}
for k_o in others_dict.keys():
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(
ref_acc[i][others[i] == others_dict[k_o]]) if np.sum(
others[i] == others_dict[k_o]) > 0 else 0
running_acc_025iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.25)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(others[i] == others_dict[k_o]) > 0 else 0
running_acc_05iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.5)].shape[0] \
/ ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(others[i] == others_dict[k_o]) > 0 else 0
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"][k_o] = {}
scores["overall"][k_o]["ref_acc"] = np.mean(ref_accs)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_025ious)
scores["overall"][k_o]["[email protected]"] = np.mean(acc_05ious)
ref_accs, acc_025ious, acc_05ious = [], [], []
for i in range(masks.shape[0]):
running_ref_acc = np.mean(ref_acc[i])
running_acc_025iou = ious[i][
ious[i] >= 0.25].shape[0] / ious[i].shape[0]
running_acc_05iou = ious[i][ious[i] >= 0.5].shape[0] / ious[i].shape[0]
# store
ref_accs.append(running_ref_acc)
acc_025ious.append(running_acc_025iou)
acc_05ious.append(running_acc_05iou)
# aggregate
scores["overall"]["overall"] = {}
scores["overall"]["overall"]["ref_acc"] = np.mean(ref_accs)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_025ious)
scores["overall"]["overall"]["[email protected]"] = np.mean(acc_05ious)
# report
print("\nstats:")
for k_s in stats.keys():
for k_o in stats[k_s].keys():
print("{} | {}: {}".format(k_s, k_o, stats[k_s][k_o]))
for k_s in scores.keys():
print("\n{}:".format(k_s))
for k_m in scores[k_s].keys():
for metric in scores[k_s][k_m].keys():
print("{} | {} | {}: {}".format(k_s, k_m, metric,
scores[k_s][k_m][metric]))
print("\nlanguage classification accuracy: {}".format(np.mean(lang_acc)))
def evaluate(args):
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC)
# model
model = get_model(args)
# config
POST_DICT = {
'remove_empty_box': True,
'use_3d_nms': True,
'nms_iou': 0.25,
'use_old_type_nms': False,
'cls_nms': True,
'per_class_proposal': True,
'conf_thresh': 0.05,
'dataset_config': DC
} if not args.no_nms else None
# evaluate
print("evaluating...")
ref_acc = []
objectness_precisions, objectness_recalls, objectness_f1s = [], [], []
ious = []
masks = []
maskrcnn_model = resnet_fpn_backbone('resnet18', True).fpn.cuda()
for data in tqdm(dataloader):
for key in data:
if key != "scan_name":
data[key] = data[key].cuda()
batch_size = len(data['scan_name'])
new_features = torch.zeros((batch_size, 40000, 32)).cuda()
print("start to project")
for idx, scene_id in enumerate(data['scan_name']):
intrinsics = get_intrinsics(scene_id, args)
projection = ProjectionHelper(intrinsics, args.depth_min,
args.depth_max, proj_image_dims)
features_2d = scannet_projection(
data['point_clouds'][idx].cpu().numpy(), intrinsics,
projection, scene_id, args, None, None, maskrcnn_model)
new_features[idx, :] = features_2d[:]
data['new_features'] = new_features
print("finish projection")
# feed
data = model(data)
_, data = get_loss(data, DC, True, True, POST_DICT)
ref_acc += data["ref_acc"]
objectness_precisions += data["objectness_precision"]
objectness_recalls += data["objectness_recall"]
objectness_f1s += data["objectness_f1"]
ious += data["ref_iou"]
masks += data["ref_multiple_mask"]
# aggregate scores
ref_acc = np.array(ref_acc)
objectness_precisions, objectness_recalls, objectness_f1s = np.array(
objectness_precisions), np.array(objectness_recalls), np.array(
objectness_f1s)
ious = np.array(ious)
masks = np.array(masks)
stats = {
"unique": np.sum(masks == 0),
"multiple": np.sum(masks == 1),
"overall": masks.shape[0]
}
scores = {"unique": {}, "multiple": {}, "overall": {}}
scores["unique"]["ref_acc"] = np.mean(
ref_acc[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["objn_prec"] = np.mean(
objectness_precisions[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["objn_recall"] = np.mean(
objectness_recalls[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["objn_f1"] = np.mean(
objectness_f1s[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["iou_rate_0.25"] = ious[masks == 0][ious[
masks == 0] >= 0.25].shape[0] / ious[masks == 0].shape[0] if np.sum(
masks == 0) > 0 else 0
scores["unique"]["iou_rate_0.5"] = ious[masks == 0][ious[
masks == 0] >= 0.5].shape[0] / ious[masks == 0].shape[0] if np.sum(
masks == 0) > 0 else 0
scores["multiple"]["ref_acc"] = np.mean(
ref_acc[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["objn_prec"] = np.mean(
objectness_precisions[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["objn_recall"] = np.mean(
objectness_recalls[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["objn_f1"] = np.mean(
objectness_f1s[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["iou_rate_0.25"] = ious[masks == 1][ious[
masks == 1] >= 0.25].shape[0] / ious[masks == 1].shape[0] if np.sum(
masks == 1) > 0 else 0
scores["multiple"]["iou_rate_0.5"] = ious[masks == 1][ious[
masks == 1] >= 0.5].shape[0] / ious[masks == 1].shape[0] if np.sum(
masks == 1) > 0 else 0
scores["overall"]["ref_acc"] = np.mean(ref_acc)
scores["overall"]["objn_prec"] = np.mean(objectness_precisions)
scores["overall"]["objn_recall"] = np.mean(objectness_recalls)
scores["overall"]["objn_f1"] = np.mean(objectness_f1s)
scores["overall"]["iou_rate_0.25"] = ious[
ious >= 0.25].shape[0] / ious.shape[0]
scores["overall"]["iou_rate_0.5"] = ious[
ious >= 0.5].shape[0] / ious.shape[0]
print("done!")
return stats, scores
def evaluate(args):
# init training dataset
print("preparing data...")
scanrefer, scene_list = get_scanrefer(args)
# dataloader
_, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC)
# model
model = get_model(args)
# config
POST_DICT = {
'remove_empty_box': True,
'use_3d_nms': True,
'nms_iou': 0.25,
'use_old_type_nms': False,
'cls_nms': True,
'per_class_proposal': True,
'conf_thresh': 0.05,
'dataset_config': DC
} if not args.no_nms else None
# evaluate
print("evaluating...")
ref_acc = []
objectness_precisions, objectness_recalls, objectness_f1s = [], [], []
ious = []
masks = []
for data in tqdm(dataloader):
for key in data:
data[key] = data[key].cuda()
# feed
data = model(data)
_, data = get_loss(data, DC, True, True, POST_DICT)
ref_acc += data["ref_acc"]
objectness_precisions += data["objectness_precision"]
objectness_recalls += data["objectness_recall"]
objectness_f1s += data["objectness_f1"]
ious += data["ref_iou"]
masks += data["ref_multiple_mask"]
# aggregate scores
ref_acc = np.array(ref_acc)
objectness_precisions, objectness_recalls, objectness_f1s = np.array(
objectness_precisions), np.array(objectness_recalls), np.array(
objectness_f1s)
ious = np.array(ious)
masks = np.array(masks)
stats = {
"unique": np.sum(masks == 0),
"multiple": np.sum(masks == 1),
"overall": masks.shape[0]
}
scores = {"unique": {}, "multiple": {}, "overall": {}}
scores["unique"]["ref_acc"] = np.mean(
ref_acc[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["objn_prec"] = np.mean(
objectness_precisions[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["objn_recall"] = np.mean(
objectness_recalls[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["objn_f1"] = np.mean(
objectness_f1s[masks == 0]) if np.sum(masks == 0) > 0 else 0
scores["unique"]["iou_rate_0.25"] = ious[masks == 0][ious[
masks == 0] >= 0.25].shape[0] / ious[masks == 0].shape[0] if np.sum(
masks == 0) > 0 else 0
scores["unique"]["iou_rate_0.5"] = ious[masks == 0][ious[
masks == 0] >= 0.5].shape[0] / ious[masks == 0].shape[0] if np.sum(
masks == 0) > 0 else 0
scores["multiple"]["ref_acc"] = np.mean(
ref_acc[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["objn_prec"] = np.mean(
objectness_precisions[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["objn_recall"] = np.mean(
objectness_recalls[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["objn_f1"] = np.mean(
objectness_f1s[masks == 1]) if np.sum(masks == 1) > 0 else 0
scores["multiple"]["iou_rate_0.25"] = ious[masks == 1][ious[
masks == 1] >= 0.25].shape[0] / ious[masks == 1].shape[0] if np.sum(
masks == 1) > 0 else 0
scores["multiple"]["iou_rate_0.5"] = ious[masks == 1][ious[
masks == 1] >= 0.5].shape[0] / ious[masks == 1].shape[0] if np.sum(
masks == 1) > 0 else 0
scores["overall"]["ref_acc"] = np.mean(ref_acc)
scores["overall"]["objn_prec"] = np.mean(objectness_precisions)
scores["overall"]["objn_recall"] = np.mean(objectness_recalls)
scores["overall"]["objn_f1"] = np.mean(objectness_f1s)
scores["overall"]["iou_rate_0.25"] = ious[
ious >= 0.25].shape[0] / ious.shape[0]
scores["overall"]["iou_rate_0.5"] = ious[
ious >= 0.5].shape[0] / ious.shape[0]
print("done!")
return stats, scores
def _compute_loss(self, data_dict):
_, data_dict = get_loss(data_dict=data_dict,
config=self.config,
detection=self.detection,
reference=self.reference,
use_lang_classifier=self.use_lang_classifier)
