def _get_raw2label(self): ''' ScanNet label classes in DC.type2class (18 classes): 0 'cabinet', 1 'bed', 2 'chair', 3 'sofa', 4 'table', 5 'door', 6 'window', 7 'bookshelf', 8 'picture', 9 'counter', 10 'desk', 11 'curtain', 12 'refrigerator', 13 'shower curtain', 14 'toilet', 15 'sink', 16 'bathtub', 17 'others' ''' ### Mapping NYU40 labeled classes to ScanNet labeled classes. DC = ScannetDatasetConfig() scannet_labels = DC.type2class.keys() scannet2label = {label: i for i, label in enumerate(scannet_labels)} raw2label = {} for line in self.classes_map: label_classes_set = set(scannet_labels) ele = line.split('\t') raw_name = ele[1] nyu40_name = ele[7] if nyu40_name not in label_classes_set: raw2label[raw_name] = scannet2label['others'] else: raw2label[raw_name] = scannet2label[nyu40_name] return raw2label
import torch import torch.nn as nn import torch.nn.functional as F import numpy as np import os import sys sys.path.append(os.path.join(os.getcwd(), "lib")) # HACK add the lib folder import lib.pointnet2.pointnet2_utils from data.scannet.model_util_scannet import ScannetDatasetConfig from lib.pointnet2.pointnet2_modules import PointnetSAModuleVotes from utils.box_util import get_3d_box_batch # constants DC = ScannetDatasetConfig() class ProposalModule(nn.Module): def __init__(self, num_class, num_heading_bin, num_size_cluster, mean_size_arr, num_proposal, sampling, seed_feat_dim=256): super().__init__() self.num_class = num_class self.num_heading_bin = num_heading_bin self.num_size_cluster = num_size_cluster self.mean_size_arr = mean_size_arr self.num_proposal = num_proposal self.sampling = sampling self.seed_feat_dim = seed_feat_dim # Vote clustering self.vote_aggregation = PointnetSAModuleVotes(
def predict(args): print("predict bounding boxes...") # constant DC = ScannetDatasetConfig() # init training dataset print("preparing data...") scanrefer, scene_list = get_scanrefer(args) # dataloader _, dataloader = get_dataloader(args, scanrefer, scene_list, "test", DC) # model model = get_model(args, DC) # config POST_DICT = { "remove_empty_box": True, "use_3d_nms": True, "nms_iou": 0.25, "use_old_type_nms": False, "cls_nms": True, "per_class_proposal": True, "conf_thresh": 0.05, "dataset_config": DC } if not args.no_nms else None # predict print("predicting...") pred_bboxes = [] for data_dict in tqdm(dataloader): for key in data_dict: data_dict[key] = data_dict[key].cuda() # feed data_dict = model(data_dict) _, data_dict = get_loss( data_dict=data_dict, config=DC, detection=False, reference=True ) objectness_preds_batch = torch.argmax(data_dict['objectness_scores'], 2).long() if POST_DICT: _ = parse_predictions(data_dict, POST_DICT) nms_masks = torch.LongTensor(data_dict['pred_mask']).cuda() # construct valid mask pred_masks = (nms_masks * objectness_preds_batch == 1).float() else: # construct valid mask pred_masks = (objectness_preds_batch == 1).float() pred_ref = torch.argmax(data_dict['cluster_ref'] * pred_masks, 1) # (B,) pred_center = data_dict['center'] # (B,K,3) pred_heading_class = torch.argmax(data_dict['heading_scores'], -1) # B,num_proposal pred_heading_residual = torch.gather(data_dict['heading_residuals'], 2, pred_heading_class.unsqueeze(-1)) # B,num_proposal,1 pred_heading_class = pred_heading_class # B,num_proposal pred_heading_residual = pred_heading_residual.squeeze(2) # B,num_proposal pred_size_class = torch.argmax(data_dict['size_scores'], -1) # B,num_proposal pred_size_residual = torch.gather(data_dict['size_residuals'], 2, pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(1,1,1,3)) # B,num_proposal,1,3 pred_size_class = pred_size_class pred_size_residual = pred_size_residual.squeeze(2) # B,num_proposal,3 for i in range(pred_ref.shape[0]): # compute the iou pred_ref_idx = pred_ref[i] pred_obb = DC.param2obb( pred_center[i, pred_ref_idx, 0:3].detach().cpu().numpy(), pred_heading_class[i, pred_ref_idx].detach().cpu().numpy(), pred_heading_residual[i, pred_ref_idx].detach().cpu().numpy(), pred_size_class[i, pred_ref_idx].detach().cpu().numpy(), pred_size_residual[i, pred_ref_idx].detach().cpu().numpy() ) pred_bbox = get_3d_box(pred_obb[3:6], pred_obb[6], pred_obb[0:3]) # construct the multiple mask multiple = data_dict["unique_multiple"][i].item() # construct the others mask others = 1 if data_dict["object_cat"][i] == 17 else 0 # store data scanrefer_idx = data_dict["scan_idx"][i].item() pred_data = { "scene_id": scanrefer[scanrefer_idx]["scene_id"], "object_id": scanrefer[scanrefer_idx]["object_id"], "ann_id": scanrefer[scanrefer_idx]["ann_id"], "bbox": pred_bbox.tolist(), "unique_multiple": multiple, "others": others } pred_bboxes.append(pred_data) # dump print("dumping...") pred_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "pred.json") with open(pred_path, "w") as f: json.dump(pred_bboxes, f, indent=4) print("done!")
def eval_ref(args): print("evaluate...") # constant DC = ScannetDatasetConfig() # init training dataset print("preparing data...") scanrefer, scene_list = get_scanrefer() # dataloader _, dataloader = get_dataloader(args, scanrefer, scene_list, "val") # model model = get_model(args) # random seeds seeds = [args.manual_seed] # evaluate print("evaluating...") score_path = os.path.join(args.use_checkpoint, "scores.p") pred_path = os.path.join(args.use_checkpoint, "predictions.p") if not os.path.exists(score_path): ref_acc_all = [] ious_all = [] masks_all = [] others_all = [] lang_acc_all = [] for seed in seeds: # reproducibility torch.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False np.random.seed(seed) print("generating the scores for seed {}...".format(seed)) ref_acc = [] ious = [] masks = [] others = [] lang_acc = [] predictions = {} for data in tqdm(dataloader): for key in data: if key in ['lang_feat', 'lang_len', 'object_cat', 'lidar', 'point_min', 'point_max', 'mlm_label', 'ref_center_label', 'ref_size_residual_label']: data[key] = data[key].cuda() # feed data = model(data) data = get_loss(data, DC) data = get_eval( data_dict=data, config=DC, ) ref_acc += data["ref_acc"] ious += data["ref_iou"] masks += data["ref_multiple_mask"] others += data["ref_others_mask"] lang_acc.append(data["lang_acc"].item()) # store predictions ids = data["scan_idx"].detach().cpu().numpy() for i in range(ids.shape[0]): idx = ids[i] scene_id = scanrefer[idx]["scene_id"] object_id = scanrefer[idx]["object_id"] ann_id = scanrefer[idx]["ann_id"] if scene_id not in predictions: predictions[scene_id] = {} if object_id not in predictions[scene_id]: predictions[scene_id][object_id] = {} if ann_id not in predictions[scene_id][object_id]: predictions[scene_id][object_id][ann_id] = {} predictions[scene_id][object_id][ann_id][""] = data["pred_bboxes"][i] predictions[scene_id][object_id][ann_id]["gt_bpred_bboxbox"] = data["gt_bboxes"][i] predictions[scene_id][object_id][ann_id]["iou"] = data["ref_iou"][i] # save the last predictions with open(pred_path, "wb") as f: pickle.dump(predictions, f) # save to global ref_acc_all.append(ref_acc) ious_all.append(ious) masks_all.append(masks) others_all.append(others) lang_acc_all.append(lang_acc) # convert to numpy array ref_acc = np.array(ref_acc_all) ious = np.array(ious_all) masks = np.array(masks_all) others = np.array(others_all) lang_acc = np.array(lang_acc_all) # save the global scores with open(score_path, "wb") as f: scores = { "ref_acc": ref_acc_all, "ious": ious_all, "masks": masks_all, "others": others_all, "lang_acc": lang_acc_all } pickle.dump(scores, f) else: print("loading the scores...") with open(score_path, "rb") as f: scores = pickle.load(f) # unpack ref_acc = np.array(scores["ref_acc"]) ious = np.array(scores["ious"]) masks = np.array(scores["masks"]) others = np.array(scores["others"]) lang_acc = np.array(scores["lang_acc"]) multiple_dict = { "unique": 0, "multiple": 1 } others_dict = { "not_in_others": 0, "in_others": 1 } # evaluation stats stats = {k: np.sum(masks[0] == v) for k, v in multiple_dict.items()} stats["overall"] = masks[0].shape[0] stats = {} for k, v in multiple_dict.items(): stats[k] = {} for k_o, v_o in others_dict.items(): stats[k][k_o] = np.sum(np.logical_and(masks[0] == v, others[0] == v_o)) stats[k]["overall"] = np.sum(masks[0] == v) stats["overall"] = {} for k_o, v_o in others_dict.items(): stats["overall"][k_o] = np.sum(others[0] == v_o) stats["overall"]["overall"] = masks[0].shape[0] # aggregate scores scores = {} for k, v in multiple_dict.items(): for k_o in others_dict.keys(): ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean( ref_acc[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])]) \ if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0 running_acc_025iou = ious[i][np.logical_and( np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.25)].shape[0] \ / ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \ if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0 running_acc_05iou = ious[i][np.logical_and( np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.5)].shape[0] \ / ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \ if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0 # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) if k not in scores: scores[k] = {k_o: {} for k_o in others_dict.keys()} scores[k][k_o]["ref_acc"] = np.mean(ref_accs) scores[k][k_o]["[email protected]"] = np.mean(acc_025ious) scores[k][k_o]["[email protected]"] = np.mean(acc_05ious) ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean(ref_acc[i][masks[i] == multiple_dict[k]]) if np.sum( masks[i] == multiple_dict[k]) > 0 else 0 running_acc_025iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.25)].shape[0] \ / ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum( masks[i] == multiple_dict[k]) > 0 else 0 running_acc_05iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.5)].shape[0] \ / ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum( masks[i] == multiple_dict[k]) > 0 else 0 # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) scores[k]["overall"] = {} scores[k]["overall"]["ref_acc"] = np.mean(ref_accs) scores[k]["overall"]["[email protected]"] = np.mean(acc_025ious) scores[k]["overall"]["[email protected]"] = np.mean(acc_05ious) scores["overall"] = {} for k_o in others_dict.keys(): ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean(ref_acc[i][others[i] == others_dict[k_o]]) if np.sum( others[i] == others_dict[k_o]) > 0 else 0 running_acc_025iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.25)].shape[0] \ / ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum( others[i] == others_dict[k_o]) > 0 else 0 running_acc_05iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.5)].shape[0] \ / ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum( others[i] == others_dict[k_o]) > 0 else 0 # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) # aggregate scores["overall"][k_o] = {} scores["overall"][k_o]["ref_acc"] = np.mean(ref_accs) scores["overall"][k_o]["[email protected]"] = np.mean(acc_025ious) scores["overall"][k_o]["[email protected]"] = np.mean(acc_05ious) ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean(ref_acc[i]) running_acc_025iou = ious[i][ious[i] >= 0.25].shape[0] / ious[i].shape[0] running_acc_05iou = ious[i][ious[i] >= 0.5].shape[0] / ious[i].shape[0] # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) # aggregate scores["overall"]["overall"] = {} scores["overall"]["overall"]["ref_acc"] = np.mean(ref_accs) scores["overall"]["overall"]["[email protected]"] = np.mean(acc_025ious) scores["overall"]["overall"]["[email protected]"] = np.mean(acc_05ious) # report print("\nstats:") for k_s in stats.keys(): for k_o in stats[k_s].keys(): print("{} | {}: {}".format(k_s, k_o, stats[k_s][k_o])) for k_s in scores.keys(): print("\n{}:".format(k_s)) for k_m in scores[k_s].keys(): for metric in scores[k_s][k_m].keys(): print("{} | {} | {}: {:.4f}".format(k_s, k_m, metric, scores[k_s][k_m][metric])) print("\nlanguage classification accuracy: {:.4f}".format(np.mean(lang_acc)))
def eval_detection(args): print("evaluate detection...") # constant DC = ScannetDatasetConfig() # init training dataset print("preparing data...") # get eval data scanrefer_eval, eval_scene_list = get_eval_data(args) # get dataloader dataset, dataloader = get_dataloader(args, scanrefer_eval, eval_scene_list, DC) # model print("initializing...") device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") root = CONF.PATH.PRETRAINED if args.eval_pretrained else CONF.PATH.OUTPUT model = get_model(args, dataset, device, root) # config POST_DICT = { "remove_empty_box": True, "use_3d_nms": True, "nms_iou": 0.25, "use_old_type_nms": False, "cls_nms": True, "per_class_proposal": True, "conf_thresh": 0.05, "dataset_config": DC } AP_IOU_THRESHOLDS = [0.25, 0.5] AP_CALCULATOR_LIST = [ APCalculator(iou_thresh, DC.class2type) for iou_thresh in AP_IOU_THRESHOLDS ] sem_acc = [] for data in tqdm(dataloader): for key in data: data[key] = data[key].cuda() # feed with torch.no_grad(): data = model(data, use_tf=False, is_eval=True) data = get_scene_cap_loss(data, device, DC, weights=dataset.weights, detection=True, caption=False) batch_pred_map_cls = parse_predictions(data, POST_DICT) batch_gt_map_cls = parse_groundtruths(data, POST_DICT) for ap_calculator in AP_CALCULATOR_LIST: ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls) # aggregate object detection results and report for i, ap_calculator in enumerate(AP_CALCULATOR_LIST): print() print("-" * 10, "iou_thresh: %f" % (AP_IOU_THRESHOLDS[i]), "-" * 10) metrics_dict = ap_calculator.compute_metrics() for key in metrics_dict: print("eval %s: %f" % (key, metrics_dict[key]))
def eval_det(args): print("evaluate detection...") # constant DC = ScannetDatasetConfig() # init training dataset print("preparing data...") scanrefer, scene_list = get_scanrefer(args) # dataloader _, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC) # model model = get_model(args, DC) # config POST_DICT = { "remove_empty_box": True, "use_3d_nms": True, "nms_iou": 0.25, "use_old_type_nms": False, "cls_nms": True, "per_class_proposal": True, "conf_thresh": 0.05, "dataset_config": DC } AP_IOU_THRESHOLDS = [0.25, 0.5] AP_CALCULATOR_LIST = [ APCalculator(iou_thresh, DC.class2type) for iou_thresh in AP_IOU_THRESHOLDS ] sem_acc = [] for data in tqdm(dataloader): for key in data: data[key] = data[key].cuda() # feed with torch.no_grad(): data = model(data) _, data = get_loss(data_dict=data, config=DC, detection=True, reference=False) data = get_eval(data_dict=data, config=DC, reference=False, post_processing=POST_DICT) sem_acc.append(data["sem_acc"].item()) batch_pred_map_cls = parse_predictions(data, POST_DICT) batch_gt_map_cls = parse_groundtruths(data, POST_DICT) for ap_calculator in AP_CALCULATOR_LIST: ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls) # aggregate object detection results and report print("\nobject detection sem_acc: {}".format(np.mean(sem_acc))) for i, ap_calculator in enumerate(AP_CALCULATOR_LIST): print() print("-" * 10, "iou_thresh: %f" % (AP_IOU_THRESHOLDS[i]), "-" * 10) metrics_dict = ap_calculator.compute_metrics() for key in metrics_dict: print("eval %s: %f" % (key, metrics_dict[key]))
def eval_ref(args): print("evaluate localization...") # constant DC = ScannetDatasetConfig() # init training dataset print("preparing data...") scanrefer, scene_list = get_scanrefer(args) # dataloader _, dataloader = get_dataloader(args, scanrefer, scene_list, "val", DC) # model model = get_model(args, DC) # config POST_DICT = { "remove_empty_box": True, "use_3d_nms": True, "nms_iou": 0.25, "use_old_type_nms": False, "cls_nms": True, "per_class_proposal": True, "conf_thresh": 0.05, "dataset_config": DC } if not args.no_nms else None # random seeds seeds = [args.seed] + [2 * i for i in range(args.repeat - 1)] # evaluate print("evaluating...") score_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "scores.p") pred_path = os.path.join(CONF.PATH.OUTPUT, args.folder, "predictions.p") gen_flag = ( not os.path.exists(score_path)) or args.force or args.repeat > 1 if gen_flag: ref_acc_all = [] ious_all = [] masks_all = [] others_all = [] lang_acc_all = [] for seed in seeds: # reproducibility torch.manual_seed(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False np.random.seed(seed) print("generating the scores for seed {}...".format(seed)) ref_acc = [] ious = [] masks = [] others = [] lang_acc = [] predictions = {} for data in tqdm(dataloader): print(data.keys()) print(data['lang_feat'].shape) assert (1 == 2) for key in data: data[key] = data[key].cuda() # feed data = model(data) _, data = get_loss(data_dict=data, config=DC, detection=True, reference=True, use_lang_classifier=not args.no_lang_cls) data = get_eval(data_dict=data, config=DC, reference=True, use_lang_classifier=not args.no_lang_cls, use_oracle=args.use_oracle, use_cat_rand=args.use_cat_rand, use_best=args.use_best, post_processing=POST_DICT) ref_acc += data["ref_acc"] ious += data["ref_iou"] masks += data["ref_multiple_mask"] others += data["ref_others_mask"] lang_acc.append(data["lang_acc"].item()) # store predictions ids = data["scan_idx"].detach().cpu().numpy() for i in range(ids.shape[0]): idx = ids[i] scene_id = scanrefer[idx]["scene_id"] object_id = scanrefer[idx]["object_id"] ann_id = scanrefer[idx]["ann_id"] if scene_id not in predictions: predictions[scene_id] = {} if object_id not in predictions[scene_id]: predictions[scene_id][object_id] = {} if ann_id not in predictions[scene_id][object_id]: predictions[scene_id][object_id][ann_id] = {} predictions[scene_id][object_id][ann_id][ "pred_bbox"] = data["pred_bboxes"][i] predictions[scene_id][object_id][ann_id]["gt_bbox"] = data[ "gt_bboxes"][i] predictions[scene_id][object_id][ann_id]["iou"] = data[ "ref_iou"][i] # save the last predictions with open(pred_path, "wb") as f: pickle.dump(predictions, f) # save to global ref_acc_all.append(ref_acc) ious_all.append(ious) masks_all.append(masks) others_all.append(others) lang_acc_all.append(lang_acc) # convert to numpy array ref_acc = np.array(ref_acc_all) ious = np.array(ious_all) masks = np.array(masks_all) others = np.array(others_all) lang_acc = np.array(lang_acc_all) # save the global scores with open(score_path, "wb") as f: scores = { "ref_acc": ref_acc_all, "ious": ious_all, "masks": masks_all, "others": others_all, "lang_acc": lang_acc_all } pickle.dump(scores, f) else: print("loading the scores...") with open(score_path, "rb") as f: scores = pickle.load(f) # unpack ref_acc = np.array(scores["ref_acc"]) ious = np.array(scores["ious"]) masks = np.array(scores["masks"]) others = np.array(scores["others"]) lang_acc = np.array(scores["lang_acc"]) multiple_dict = {"unique": 0, "multiple": 1} others_dict = {"not_in_others": 0, "in_others": 1} # evaluation stats stats = {k: np.sum(masks[0] == v) for k, v in multiple_dict.items()} stats["overall"] = masks[0].shape[0] stats = {} for k, v in multiple_dict.items(): stats[k] = {} for k_o, v_o in others_dict.items(): stats[k][k_o] = np.sum( np.logical_and(masks[0] == v, others[0] == v_o)) stats[k]["overall"] = np.sum(masks[0] == v) stats["overall"] = {} for k_o, v_o in others_dict.items(): stats["overall"][k_o] = np.sum(others[0] == v_o) stats["overall"]["overall"] = masks[0].shape[0] # aggregate scores scores = {} for k, v in multiple_dict.items(): for k_o in others_dict.keys(): ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean(ref_acc[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])]) \ if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0 running_acc_025iou = ious[i][np.logical_and(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.25)].shape[0] \ / ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \ if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0 running_acc_05iou = ious[i][np.logical_and(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o]), ious[i] >= 0.5)].shape[0] \ / ious[i][np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])].shape[0] \ if np.sum(np.logical_and(masks[i] == multiple_dict[k], others[i] == others_dict[k_o])) > 0 else 0 # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) if k not in scores: scores[k] = {k_o: {} for k_o in others_dict.keys()} scores[k][k_o]["ref_acc"] = np.mean(ref_accs) scores[k][k_o]["[email protected]"] = np.mean(acc_025ious) scores[k][k_o]["[email protected]"] = np.mean(acc_05ious) ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean( ref_acc[i][masks[i] == multiple_dict[k]]) if np.sum( masks[i] == multiple_dict[k]) > 0 else 0 running_acc_025iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.25)].shape[0] \ / ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(masks[i] == multiple_dict[k]) > 0 else 0 running_acc_05iou = ious[i][np.logical_and(masks[i] == multiple_dict[k], ious[i] >= 0.5)].shape[0] \ / ious[i][masks[i] == multiple_dict[k]].shape[0] if np.sum(masks[i] == multiple_dict[k]) > 0 else 0 # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) scores[k]["overall"] = {} scores[k]["overall"]["ref_acc"] = np.mean(ref_accs) scores[k]["overall"]["[email protected]"] = np.mean(acc_025ious) scores[k]["overall"]["[email protected]"] = np.mean(acc_05ious) scores["overall"] = {} for k_o in others_dict.keys(): ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean( ref_acc[i][others[i] == others_dict[k_o]]) if np.sum( others[i] == others_dict[k_o]) > 0 else 0 running_acc_025iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.25)].shape[0] \ / ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(others[i] == others_dict[k_o]) > 0 else 0 running_acc_05iou = ious[i][np.logical_and(others[i] == others_dict[k_o], ious[i] >= 0.5)].shape[0] \ / ious[i][others[i] == others_dict[k_o]].shape[0] if np.sum(others[i] == others_dict[k_o]) > 0 else 0 # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) # aggregate scores["overall"][k_o] = {} scores["overall"][k_o]["ref_acc"] = np.mean(ref_accs) scores["overall"][k_o]["[email protected]"] = np.mean(acc_025ious) scores["overall"][k_o]["[email protected]"] = np.mean(acc_05ious) ref_accs, acc_025ious, acc_05ious = [], [], [] for i in range(masks.shape[0]): running_ref_acc = np.mean(ref_acc[i]) running_acc_025iou = ious[i][ ious[i] >= 0.25].shape[0] / ious[i].shape[0] running_acc_05iou = ious[i][ious[i] >= 0.5].shape[0] / ious[i].shape[0] # store ref_accs.append(running_ref_acc) acc_025ious.append(running_acc_025iou) acc_05ious.append(running_acc_05iou) # aggregate scores["overall"]["overall"] = {} scores["overall"]["overall"]["ref_acc"] = np.mean(ref_accs) scores["overall"]["overall"]["[email protected]"] = np.mean(acc_025ious) scores["overall"]["overall"]["[email protected]"] = np.mean(acc_05ious) # report print("\nstats:") for k_s in stats.keys(): for k_o in stats[k_s].keys(): print("{} | {}: {}".format(k_s, k_o, stats[k_s][k_o])) for k_s in scores.keys(): print("\n{}:".format(k_s)) for k_m in scores[k_s].keys(): for metric in scores[k_s][k_m].keys(): print("{} | {} | {}: {}".format(k_s, k_m, metric, scores[k_s][k_m][metric])) print("\nlanguage classification accuracy: {}".format(np.mean(lang_acc)))