def train_daly_object(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) _set_defcfg_detectron(cfg) _set_defcfg_detectron_train(cfg) _set_defcfg_object_hacks(cfg) cf = cfg.parse() cf_add_d2 = cfg.without_prefix('d2.') cf_add_d2 = d2dict_gpu_scaling(cf, cf_add_d2, cf['num_gpus']) dataset = Dataset_daly_ocv() dataset.populate_from_folder(cf['dataset.cache_folder']) o100_objects, category_map = get_category_map_o100(dataset) assert len(o100_objects) == 16 split_label = cf['dataset.subset'] split_vids = get_daly_split_vids(dataset, split_label) datalist: Datalist = simplest_daly_to_datalist_v2(dataset, split_vids) cls_names, datalist_converter = \ _datalist_hacky_converter(cf, dataset) datalist = datalist_converter(datalist) TRAIN_DATASET_NAME = 'daly_objaction_train' _train_routine(cf, cf_add_d2, out, cls_names, TRAIN_DATASET_NAME, datalist, add_args)
def eval_daly_object(workfolder, cfg_dict, add_args): """ Evaluation code with hacks """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) _set_defcfg_detectron(cfg) _set_defcfg_detectron_test(cfg) _set_defcfg_object_hacks(cfg) cfg.set_deftype(""" what_to_eval: [~, str] eval_hacks: model_to_eval: ['what', ['what', 'what+foldname']] """) cf = cfg.parse() cf_add_d2 = cfg.without_prefix('d2.') # DALY Dataset dataset = Dataset_daly_ocv() dataset.populate_from_folder(cf['dataset.cache_folder']) split_label = cf['dataset.subset'] split_vids = get_daly_split_vids(dataset, split_label) datalist: Datalist = simplest_daly_to_datalist_v2(dataset, split_vids) cls_names, datalist_converter = \ _datalist_hacky_converter(cf, dataset) datalist = datalist_converter(datalist) TEST_DATASET_NAME = 'daly_objaction_test' model_to_eval = _eval_foldname_hack(cf) _eval_routine(cf, cf_add_d2, out, cls_names, TEST_DATASET_NAME, datalist, model_to_eval)
def eval_avstubes(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(raise_without_defaults=False) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tubes.set_defcfg(cfg) cfg.set_defaults(""" tubes_path: ~ """) Ncfg_tube_eval.set_defcfg(cfg) cf = cfg.parse() dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) tubes_to_eval: AV_dict[T_dwein_scored] = \ small.load_pkl(cf['tubes_path']) # Extended version of "Ncfg_tube_eval.evalprint_if" nms_thresh = cf['tube_eval.nms.thresh'] iou_thresholds: List[float] = cf['tube_eval.iou_thresholds'] minscore_cutoff = cf['tube_eval.minscore_cutoff'] tubes_to_eval = av_stubes_above_score(tubes_to_eval, minscore_cutoff) tubes_to_eval = \ compute_nms_for_av_stubes(tubes_to_eval, nms_thresh) dfdict = _compute_exhaustive_evaluation_stats(av_gt_tubes, tubes_to_eval, iou_thresholds) _print_exhaustive_evaluation_stats(dfdict)
def apply_pncaffe_rcnn_in_frames(workfolder, cfg_dict, add_args): """ Apply Phil-Nic rcnn model on tube boxes to extract per-action scores """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tubes.set_defcfg(cfg) Ncfg_nicphil_rcnn.set_defcfg(cfg) Ncfg_generic_rcnn_eval.set_defcfg(cfg) Ncfg_tube_eval.set_defcfg(cfg) cf = cfg.parse() # Preparation dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) tubes_dwein: Dict[I_dwein, T_dwein] = \ Ncfg_tubes.resolve_tubes_dwein(cf, split_vids) neth: Nicolas_net_helper = Ncfg_nicphil_rcnn.resolve_helper(cf) # Experiment logic if cf['demo_run.enabled']: Ncfg_generic_rcnn_eval.demo_run(cf, out, dataset, split_vids, tubes_dwein, neth) return vf_connections_dwti, vf_cls_probs = \ Ncfg_generic_rcnn_eval.evaluate_rcnn_boxes( cf, out, dataset, split_vids, tubes_dwein, neth) small.save_pkl(out / 'vf_connections_dwti.pkl', vf_connections_dwti) small.save_pkl(out / 'vf_cls_probs.pkl', vf_cls_probs) agg_kind = cf['score_agg_kind'] av_stubes = Ncfg_generic_rcnn_eval.aggregate_rcnn_scores( dataset, tubes_dwein, vf_connections_dwti, vf_cls_probs, agg_kind) small.save_pkl(out / 'av_stubes.pkl', av_stubes) # Post experiment Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def compare_data(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(raise_without_defaults=False) cfg.set_defaults(""" cache_folders: ~ """) cf = cfg.parse() dataset = Dataset_daly_ocv() dataset.precompute_to_folder(out)
def load_wein_tubes(workfolder, cfg_dict, add_args): """ Philippe tubes: tube: (one row per frame): index of the frame (starting at 1) x1 y1 x2 y2 score of the generic human detector score of the instance-specific detector """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_deftype(""" wein_tubes: [~, str] dataset: cache_folder: [~, str] mirror: ['uname', ~] wein_index: [0, int] # 0 = 510 cut tubes, 1 = 200 uncut tubes """) cf = cfg.parse() # Dataset dataset = Dataset_daly_ocv(cf['dataset.mirror']) dataset.populate_from_folder(cf['dataset.cache_folder']) # Tubes # 0 has 510 elements, 1 has 200 elements wein_package = small.load_py2_pkl(cf['wein_tubes'])[cf['wein_index']] # We got a dictionary of filenames (w .mp4 suffix) extracted_tubes: Dict[I_dwein, Tube_daly_wein_as_provided] = {} for vid_mp4, wein_bunches in wein_package.items(): vid = re.search(r'(.*)\.mp4', vid_mp4).group(1) rs = dataset.rstats[vid] for bunch_id, wein_tubes in enumerate(wein_bunches): for tube_id, wein_tube in enumerate(wein_tubes): frame_inds = wein_tube[:, 0].astype(np.int) - 1 assert max(frame_inds) < rs['max_pos_frames'] boxes_ltrd = wein_tube[:, 1:5] # ltrd human_scores = wein_tube[:, 5] instance_scores = wein_tube[:, 6] tube = { 'frame_inds': frame_inds, 'boxes': boxes_ltrd, 'hscores': human_scores, 'iscores': instance_scores } extracted_tubes[(vid, bunch_id, tube_id)] = tube small.save_pkl(out / 'extracted_tubes.pkl', extracted_tubes)
def gather_reapply_agg_rcnn_avstubes(workfolder, cfg_dict, add_args): """ Will apply aggregation again """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(['gather.paths']) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tubes.set_defcfg(cfg) cfg.set_deftype(""" gather: paths: [~, ~] score_agg_kind: ['mean', ['mean', 'max', 'sum']] """) Ncfg_tube_eval.set_defcfg(cfg) cf = cfg.parse() # Preparation dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) tubes_dwein: Dict[I_dwein, T_dwein] = \ Ncfg_tubes.resolve_tubes_dwein(cf, split_vids) # Experiment logic gather_paths = cf['gather.paths'] if not _gather_check_all_present( gather_paths, ['vf_cls_probs.pkl', 'vf_connections_dwti.pkl']): return vf_connections_dwti = {} vf_cls_probs = {} for path in gather_paths: path = Path(path) vf_cls_probs_ = small.load_pkl(path / 'vf_cls_probs.pkl') vf_connections_dwti_ = small.load_pkl(path / 'vf_connections_dwti.pkl') assert vf_cls_probs_.keys() == vf_connections_dwti_.keys() vf_cls_probs.update(vf_cls_probs_) vf_connections_dwti.update(vf_connections_dwti_) small.save_pkl(out / 'vf_connections_dwti.pkl', vf_connections_dwti) small.save_pkl(out / 'vf_cls_probs.pkl', vf_cls_probs) agg_kind = cf['score_agg_kind'] av_stubes: AV_dict[T_dwein_scored] = \ Ncfg_generic_rcnn_eval.aggregate_rcnn_scores( dataset, tubes_dwein, vf_connections_dwti, vf_cls_probs, agg_kind) small.save_pkl(out / 'av_stubes.pkl', av_stubes) # Post experiment Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def assign_objactions_to_tubes(workfolder, cfg_dict, add_args): """ Score tubes by assigning objactions to them and pooling the scores, then evaluate resulting scored tubes - Objactions: detecton evaluated datalist or gt objects (per frame) - Tubes: philippe tubes - Assignment: inner overlap or iou scores """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tubes.set_defcfg(cfg) cfg.set_deftype(""" actobjects: source: ['detected', ['detected', 'gt']] detected: path: [~, ~] obj_to_tube: overlap_type: ['inner_overlap', ['inner_overlap', 'iou']] overlap_cutoff: [0.2, float] score_cutoff: [0.2, float] """) Ncfg_tube_eval.set_defcfg(cfg) cf = cfg.parse() dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) # Inputs to the assignment routine ftubes: Dict[I_dwein, T_dwein] = \ Ncfg_tubes.resolve_tubes_dwein(cf, split_vids) objactions_vf: Dict[Vid_daly, Dict[int, Objaction_dets]] = \ _resolve_actobjects(cf, dataset, split_vids) # Assignment itself overlap_type = cf['obj_to_tube.overlap_type'] overlap_cutoff = cf['obj_to_tube.overlap_cutoff'] score_cutoff = cf['obj_to_tube.score_cutoff'] av_stubes: AV_dict[T_dwein_scored] = \ score_ftubes_via_objaction_overlap_aggregation( dataset, objactions_vf, ftubes, overlap_type, overlap_cutoff, score_cutoff) small.save_pkl(out / 'av_stubes.pkl', av_stubes) Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def train_daly_action(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) _set_defcfg_detectron(cfg) _set_defcfg_detectron_train(cfg) cf = cfg.parse() cf_add_d2 = cfg.without_prefix('d2.') cf_add_d2 = d2dict_gpu_scaling(cf, cf_add_d2, cf['num_gpus']) dataset = Dataset_daly_ocv() dataset.populate_from_folder(cf['dataset.cache_folder']) split_label = cf['dataset.subset'] split_vids = get_daly_split_vids(dataset, split_label) datalist: Datalist = daly_to_datalist_pfadet(dataset, split_vids) cls_names = dataset.action_names TRAIN_DATASET_NAME = 'daly_pfadet_train' _train_routine(cf, cf_add_d2, out, cls_names, TRAIN_DATASET_NAME, datalist, add_args)
def apply_pfadet_rcnn_in_frames(workfolder, cfg_dict, add_args): """ Apply trained d2 frcnn model on tube boxes to extract per-action scores - We dispense with the frcnn box predictions and only use per-roi scores """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(['d2.']) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tubes.set_defcfg(cfg) Ncfg_generic_rcnn_eval.set_defcfg(cfg) cfg.set_deftype(""" d2_rcnn: model: [~, ~] conf_thresh: [0.0, float] """) Ncfg_tube_eval.set_defcfg(cfg) cf = cfg.parse() cf_add_d2 = cfg.without_prefix('d2.') # Preparation dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) tubes_dwein: Dict[I_dwein, T_dwein] = \ Ncfg_tubes.resolve_tubes_dwein(cf, split_vids) neth = D2_rcnn_helper(cf, cf_add_d2, dataset, out) # Experiment logic if cf['demo_run.enabled']: Ncfg_generic_rcnn_eval.demo_run(cf, out, dataset, split_vids, tubes_dwein, neth) return vf_connections_dwti, vf_cls_probs = \ Ncfg_generic_rcnn_eval.evaluate_rcnn_boxes( cf, out, dataset, split_vids, tubes_dwein, neth) small.save_pkl(out / 'vf_connections_dwti.pkl', vf_connections_dwti) small.save_pkl(out / 'vf_cls_probs.pkl', vf_cls_probs) agg_kind = cf['score_agg_kind'] av_stubes = Ncfg_generic_rcnn_eval.aggregate_rcnn_scores( dataset, tubes_dwein, vf_connections_dwti, vf_cls_probs, agg_kind) small.save_pkl(out / 'av_stubes.pkl', av_stubes) # Post experiment Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def precompute_cache(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_deftype(""" dataset: [~, str] charades: mirror: ['gpuhost7', str] resolution: ['480', str] daly: mirror: [~, str] """) cf = cfg.parse() if cf['dataset'] == 'daly': dataset = Dataset_daly_ocv(cf['daly.mirror']) elif cf['dataset'] == 'charades': dataset = Dataset_charades_ocv(cf['charades.mirror'], cf['charades.resolution']) elif cf['dataset'] == 'voc2007': raise NotImplementedError() else: raise RuntimeError('Wrong dataset') dataset.precompute_to_folder(out)
def eval_stubes_experimental(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(raise_without_defaults=False) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tubes.set_defcfg(cfg) cfg.set_defaults(""" tubes_path: ~ """) Ncfg_tube_eval.set_defcfg(cfg) cf = cfg.parse() dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) tubes_to_eval: AV_dict[T_dwein_scored] = \ small.load_pkl(cf['tubes_path']) dgt_tubes: Dict[I_dgt, T_dgt] = _reindex_avd_to_dgt(av_gt_tubes) tubes_dwein: Dict[I_dwein, T_dwein] tubes_dwein = loadconvert_tubes_dwein('/home/vsydorov/projects/deployed/2019_12_Thesis/dervo/thes/gpuhost7/2020_03_31/10_clean_inputs/20_wein_tubes/thes.gpuhost7.2020_03_31.10_clean_inputs.20_wein_tubes/194e3a301c202bc5e818dca26953ddb268aa98b3/out/extracted_tubes.pkl') tubes_dwein = dtindex_filter_split(tubes_dwein, split_vids) wgi, gti = _get_weingroup_assignment(dgt_tubes, tubes_dwein) wgi_to_gti: Dict[I_weingroup, I_dgt] = dict(zip(wgi, gti)) # Evaluation keyframe_av_stubes = small.load_pkl('/home/vsydorov/projects/deployed/2019_12_Thesis/dervo/thes/gpuhost7/2020_03_31/30_whole_video/30_pncaffe_rcnn/05_develop/10_splits_keyframes/thes.gpuhost7.2020_03_31.30_whole_video.30_pncaffe_rcnn.05_develop.10_splits_keyframes/RAW/out/av_stubes.pkl') # Recall without NMS iou_thresholds = cf['tube_eval.iou_thresholds'] nms_thresh = 0.5 recall_norm = compute_recall_for_avtubes_as_dfs( av_gt_tubes, norm_av_stubes, iou_thresholds, False)[0] recall_kf = compute_recall_for_avtubes_as_dfs( av_gt_tubes, keyframe_av_stubes, iou_thresholds, False)[0] # Ap without NMS ap_norm = compute_ap_for_avtubes_as_df( av_gt_tubes, norm_av_stubes, iou_thresholds, False, False) ap_kf = compute_ap_for_avtubes_as_df( av_gt_tubes, keyframe_av_stubes, iou_thresholds, False, False) # # Apply "special NMS" # wnms_norm_av_stubes = _weingroup_nms(norm_av_stubes, nms_thresh) # wnms_keyframe_av_stubes = _weingroup_nms(keyframe_av_stubes, nms_thresh) # Apply "special NMS" wnms_norm_av_stubes = \ (small.stash2(out/'_temp_wnms_norm_av_stubes.pkl') (_weingroup_nms, norm_av_stubes, nms_thresh)) wnms_keyframe_av_stubes = \ (small.stash2(out/'_temp_wnms_keyframe_av_stubes.pkl') (_weingroup_nms, keyframe_av_stubes, nms_thresh)) # Recall with "special" NMS wnms_recall_norm = compute_recall_for_avtubes_as_dfs( av_gt_tubes, wnms_norm_av_stubes, iou_thresholds, False)[0] wnms_recall_kf = compute_recall_for_avtubes_as_dfs( av_gt_tubes, wnms_keyframe_av_stubes, iou_thresholds, False)[0] # AP with "special" NMS. wnms_ap_norm = compute_ap_for_avtubes_as_df( av_gt_tubes, wnms_norm_av_stubes, iou_thresholds, False, False) wnms_ap_kf = compute_ap_for_avtubes_as_df( av_gt_tubes, wnms_keyframe_av_stubes, iou_thresholds, False, False) # WG AP with "special" NMS wnms_wap_norm = compute_ap_for_avtubes_WG_as_df(wgi_to_gti, av_gt_tubes, wnms_norm_av_stubes, iou_thresholds, False, False) wnms_wap_kf = compute_ap_for_avtubes_WG_as_df(wgi_to_gti, av_gt_tubes, wnms_keyframe_av_stubes, iou_thresholds, False, False) import pudb; pudb.set_trace() # XXX BREAKPOINT
def vis_stubes(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(raise_without_defaults=False) Ncfg_dataset.set_dataset_seed(cfg) cfg.set_defaults(""" paths: av_stubes: ~ vf_connections_dwti: ~ vf_cls_probs: ~ """) cf = cfg.parse() dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) av_stubes: AV_dict[T_dwein_scored] = \ small.load_pkl(cf['paths.av_stubes']) vf_connections_dwti: Dict[Vid_daly, Dict[int, Box_connections_dwti]] = \ small.load_pkl(cf['paths.vf_connections_dwti']) vf_cls_probs: Dict[Vid_daly, Dict[int, np.ndarray]] = \ small.load_pkl(cf['paths.vf_cls_probs']) # WGI GTI dgt_tubes: Dict[I_dgt, T_dgt] = _reindex_avd_to_dgt(av_gt_tubes) tubes_dwein: Dict[I_dwein, T_dwein] = loadconvert_tubes_dwein('/home/vsydorov/projects/deployed/2019_12_Thesis/dervo/thes/gpuhost7/2020_03_31/10_clean_inputs/20_wein_tubes/thes.gpuhost7.2020_03_31.10_clean_inputs.20_wein_tubes/194e3a301c202bc5e818dca26953ddb268aa98b3/out/extracted_tubes.pkl') tubes_dwein = dtindex_filter_split(tubes_dwein, split_vids) wgi, gti = _get_weingroup_assignment(dgt_tubes, tubes_dwein) wgi_to_gti: Dict[I_weingroup, I_dgt] = dict(zip(wgi, gti)) nms_thresh = 0.5 wnms_av_stubes = \ (small.stash2(out/'_temp_wnms_norm_av_stubes.pkl') (_weingroup_nms, av_stubes, nms_thresh)) iou_thresholds = [0.3, 0.5] co_wnms_av_stubes = av_stubes_above_score( wnms_av_stubes, 0.05) wnms_ap = compute_ap_for_avtubes_as_df( av_gt_tubes, co_wnms_av_stubes, iou_thresholds, False, False) action = 'Drinking' action_dwti_ind = dataset.action_names.index(action) + 1 vfold = small.mkdir(out/'ap_emulate'/action) v_stubes = wnms_av_stubes[action] v_gt_tubes = av_gt_tubes[action] # ap-like preparations fgts, fdets = _convert_to_flat_representation(v_gt_tubes, v_stubes) det_to_eligible_gt = _compute_eligible_tubes_for_eval_weingroup( fgts, fdets, wgi_to_gti) gt_already_matched = np.zeros(len(fgts), dtype=bool) nd = len(fdets) tp = np.zeros(nd) fp = np.zeros(nd) npos = len([x for x in fgts if not x['diff']]) detection_scores = np.array([x['score'] for x in fdets]) detection_scores = detection_scores.round(3) sorted_inds = np.argsort(-detection_scores) iou_thresh = 0.5 use_diff = False # Provenance detection_matched_to_which_gt = np.ones(len(fdets), dtype=int)*-1 ifdet_to_iou_coverages = {} for d, ifdet in enumerate(sorted_inds): matchable_ifgts = list(det_to_eligible_gt.get(ifdet, {})) if not len(matchable_ifgts): fp[d] = 1 continue iou_coverages: List[float] = \ _compute_iou_coverages(fgts, fdets, det_to_eligible_gt, matchable_ifgts, ifdet, False) ifdet_to_iou_coverages[ifdet] = iou_coverages max_coverage_local_id: int = np.argmax(iou_coverages) max_coverage: float = iou_coverages[max_coverage_local_id] max_coverage_ifgt = matchable_ifgts[max_coverage_local_id] if max_coverage > iou_thresh: if (not use_diff) and fgts[max_coverage_ifgt]['diff']: continue if not gt_already_matched[max_coverage_ifgt]: tp[d] = 1 gt_already_matched[max_coverage_ifgt] = True detection_matched_to_which_gt[ifdet] = max_coverage_ifgt else: fp[d] = 1 else: fp[d] = 1 cumsum_fp = np.cumsum(fp) cumsum_tp = np.cumsum(tp) rec = cumsum_tp / float(npos) prec = cumsum_tp / np.maximum(cumsum_tp + cumsum_fp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, False) log.info(f'{ap=}') # Visualize this for d, ifdet in enumerate(sorted_inds): if not tp[d]: continue tube = fdets[ifdet]['obj'] score = tube['score'] sf, ef = tube['start_frame'], tube['end_frame'] vid = tube['index'][0] frame_inds = tube['frame_inds'] video_fold = small.mkdir(vfold/f'D{d:04d}_IFDET{ifdet:04d}_V({vid})_{sf}_to_{ef}_score{score:02f}') video_path = dataset.videos[vid]['path'] # Extract with vt_cv.video_capture_open(video_path) as vcap: frames_u8 = vt_cv.video_sample( vcap, frame_inds, debug_filename=video_path) # Draw drawn_frames_u8 = [] for i, (find, frame_BGR) in enumerate(zip(frame_inds, frames_u8)): image = frame_BGR.copy() box = tube['boxes'][i] connections_dwti = vf_connections_dwti[vid][find] scores_dwti = vf_cls_probs[vid][find] source_id = connections_dwti['dwti_sources'].index(tube['index']) dwti_box = connections_dwti['boxes'][source_id] dwti_score = scores_dwti[source_id][action_dwti_ind] assert np.allclose(dwti_box, box) snippets.cv_put_box_with_text(image, box, text='{}({}); {:.2f}/{:.2f}; {}'.format( i, find, dwti_score, score, action)) drawn_frames_u8.append(image) # # Save as images # for find, image in zip(frame_inds, drawn_frames_u8): # cv2.imwrite(str( # video_fold/'Fr{:05d}.png'.format(find)), image) # Save as video snippets.qsave_video(video_fold/'overlaid.mp4', drawn_frames_u8)
def charades_stats(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_deftype(""" dataset: cache_folder: [~, str] charades: mirror: ['gpuhost7', str] resolution: ['480', str] """) cf = cfg.parse() mirror = cf['dataset.charades.mirror'] resolution = cf['dataset.charades.resolution'] dataset = Dataset_charades_ocv(mirror, resolution) dataset.populate_from_folder(cf['dataset.cache_folder']) pd_vs = pd.DataFrame(dataset.videos_ocv).T s_split = pd.Series(dataset.split) pd_vs['split'] = s_split pd_vs['action_labels'] = pd_vs.actions.apply( lambda x: [y['name'] for y in x]) train_mask = pd_vs['split'] == 'train' val_mask = pd_vs['split'] == 'val' pd_vs_train = pd_vs[train_mask] pd_vs_val = pd_vs[val_mask] N_classes = len(dataset.action_names) log.info(f'N classes: {N_classes}') log.info('N videos (Total/Train/Val): {}/{}/{}'.format( len(pd_vs), len(pd_vs_train), len(pd_vs_val))) log.info('N frames (Total/Train/Val): {}/{}/{}'.format( pd_vs.nframes.sum(), pd_vs_train.nframes.sum(), pd_vs_val.nframes.sum())) l_ = [] for i, a in enumerate(dataset.action_names): s = pd_vs.action_labels.apply(lambda x: a in x) s.name = i l_.append(s) action_presence = pd.concat(l_, axis=1) l_ = [] for i, o in enumerate(dataset.object_names): s = pd_vs.objects.apply(lambda x: o in x) s.name = i l_.append(s) obj_presence = pd.concat(l_, axis=1) action_mapping_int = { dataset.action_names.index(a): dataset.object_names.index(o) for a, (o, v) in dataset.action_mapping.items() } column_mapped_obj = {} for i, column in action_presence.iteritems(): column_mapped_obj.setdefault(action_mapping_int[i], []).append(column) obj_presence_mapped = pd.DataFrame({ k: pd.concat(v, axis=1).any(axis=1) for k, v in column_mapped_obj.items() }).sort_index(axis=1) # 0/'None' object to be removed obj_presence0 = obj_presence.loc[:, 1:] obj_presence_mapped0 = obj_presence_mapped.loc[:, 1:] def frac_str(X, Y): return f'{X}/{Y}/{X/Y*100:.2f}%' def mean_frac(X): return frac_str(X.sum(), len(X)) X_extra_present = ((obj_presence0 > obj_presence_mapped0).any(axis=1)) log.info('Videos have extra objects present, beyond mapped: ' '[Total: {}] [Train: {}] [Val: {}]'.format( mean_frac(X_extra_present), mean_frac(X_extra_present[train_mask]), mean_frac(X_extra_present[val_mask]))) X_impobj_missing = (obj_presence0 < obj_presence_mapped0).any(axis=1) log.info( 'Videos have objects missing, though they are implied by mapping: ' '[Total: {}] [Train: {}] [Val: {}]'.format( mean_frac(X_impobj_missing), mean_frac(X_impobj_missing[train_mask]), mean_frac(X_impobj_missing[val_mask]))) def produce_map_annot(obj_presence_mapped0, obj_presence0): X = (obj_presence_mapped0 & obj_presence0).sum() XX = obj_presence_mapped0.sum() present_when_mapped = pd.concat((X / XX * 100, X, XX), axis=1) present_when_mapped.index = dataset.object_names[1:] present_when_mapped.columns = ['%', 'matched', 'total_mapped'] return present_when_mapped X = produce_map_annot(obj_presence_mapped0, obj_presence0) X_train = produce_map_annot(obj_presence_mapped0[train_mask], obj_presence0[train_mask]) X_val = produce_map_annot(obj_presence_mapped0[val_mask], obj_presence0[val_mask]) log.info('Videos per cls, where mapped object is annotated:\n{}'.format( pd.concat((X, X_train, X_val), keys=['total', 'train', 'val'], axis=1))) # Per-instance stats instances_ = {} for vid, video in dataset.videos_ocv.items(): for ii, instance in enumerate(video['actions']): instance_ = copy.copy(instance) instance_['split'] = dataset.split[vid] instances_[(vid, ii)] = instance_ pd_inst = pd.DataFrame(instances_).T pd_inst['nframes'] = pd_inst['end_frame'] - pd_inst['start_frame'] + 1 pd_inst['action_ind'] = pd_inst.name.apply( lambda x: dataset.action_names.index(x)) train_mask_i = pd_inst['split'] == 'train' val_mask_i = pd_inst['split'] == 'val' nframes_per_cls = pd_inst.groupby('action_ind').nframes.sum() log.info(f'N frames per cls, summed (total): {nframes_per_cls.sum()}') nframes_per_cls_train = ( pd_inst[train_mask_i].groupby('action_ind').nframes.sum()) nframes_per_cls_val = ( pd_inst[val_mask_i].groupby('action_ind').nframes.sum()) X = pd.concat( (nframes_per_cls, nframes_per_cls_train, nframes_per_cls_val), axis=1, keys=['total', 'train', 'val']) log.info('Describe frames per cls:\n{}'.format(X.describe())) # Frame coverage fcoverage = { vid: np.zeros(row['nframes'], dtype=np.int) for vid, row in pd_vs.iterrows() } for (vid, i), row in pd_inst.iterrows(): s = row['start_frame'] e = row['end_frame'] fcoverage[vid][s:e + 1] += 1 coverage_per_vid = {} for vid, cov in fcoverage.items(): d = { 'nframes': len(cov), 'covered': (cov > 0).sum(), 'annos': cov.sum() } coverage_per_vid[vid] = d df_coverage = pd.DataFrame(coverage_per_vid).T df_coverage['frac'] = df_coverage['covered'] / df_coverage['nframes'] def anno_frac(X): return frac_str(X.covered.sum(), X.nframes.sum()) def avg_anno_vf_frac(X): return frac_str(X.annos.sum(), X.nframes.sum()) def avg_anno_af_frac(X): return frac_str(X.annos.sum(), X.covered.sum()) log.info('Annotated fraction (per-vid): ' '[Total: {:.2f}%] [Train: {:.2f}%] [Val: {:.2f}%]'.format( df_coverage.frac.mean() * 100, df_coverage[train_mask].frac.mean() * 100, df_coverage[val_mask].frac.mean() * 100)) log.info('Annotated fraction: ' '[Total: {}] [Train: {}] [Val: {}]'.format( anno_frac(df_coverage), anno_frac(df_coverage[train_mask]), anno_frac(df_coverage[val_mask]))) log.info('Avg. N annotation per video frame: ' '[Total: {}] [Train: {}] [Val: {}]'.format( avg_anno_vf_frac(df_coverage), avg_anno_vf_frac(df_coverage[train_mask]), avg_anno_vf_frac(df_coverage[val_mask]))) log.info('Avg. N annotation per annotated video frame: ' '[Total: {}] [Train: {}] [Val: {}]'.format( avg_anno_af_frac(df_coverage), avg_anno_af_frac(df_coverage[train_mask]), avg_anno_af_frac(df_coverage[val_mask])))
def merge_scores_avstubes(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(raise_without_defaults=False) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tube_eval.set_defcfg(cfg) cfg.set_defaults(""" tube_dict: ~ combinations: enabled: False sizes: ~ """) cf = cfg.parse() dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) ts = {k: small.load_pkl(v) for k, v in cfg_dict['tube_dict'].items()} if not cf['combinations.enabled']: av_stubes = _meanpool_avstubes(list(ts.values())) small.save_pkl(out / 'merged_av_stubes.pkl', av_stubes) log.info('All combined score:') Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes) return sizes = cf['combinations.sizes'] combinations = [list(itertools.combinations(ts.keys(), r)) for r in sizes] combinations = list(itertools.chain(*combinations)) log.info('Combinations: {}'.format(combinations)) comb_dfdicts = {} for comb in combinations: comb_name = '+'.join(comb) comb_fold = small.mkdir(out / comb_name) def compute(): to_merge = [ts[k] for k in comb] av_stubes = _meanpool_avstubes(to_merge) small.save_pkl(comb_fold / 'av_stubes.pkl', av_stubes) dfdict = Ncfg_tube_eval.eval_as_df(cf, av_stubes, av_gt_tubes) return dfdict dfdict = small.stash2(comb_fold / 'stashed_dfdict.pkl')(compute) comb_dfdicts[comb_name] = dfdict log.info('Individual results:') for comb_name, dfdict in comb_dfdicts.items(): log.info(f'Results for {comb_name}:') _print_quick_evaluation_stats(dfdict) log.info('Combined tables:') big_ = {comb: pd.concat(dfdict) for comb, dfdict in comb_dfdicts.items()} big = pd.concat(big_, axis=1) for stat in big.index.levels[0]: log.info(f'=== {stat} ===') for thresh in big.columns.levels[1]: X = (big.loc['ap'].loc[:, pd.IndexSlice[:, thresh]].droplevel(1, axis=1)) table = snippets.df_to_table_v2((X * 100).round(2)) log.info(f'{stat} for IOU {thresh}:\n{table}') log.info('\n')