def rename_old_rundir(rundir): if len(list(rundir.iterdir())) > 0: timestamp = datetime.fromtimestamp( rundir.stat().st_mtime).strftime('%Y-%m-%d_%H:%M:%S') str_rnd = ''.join(random.choices(string.ascii_uppercase, k=3)) new_foldname = f'old_{timestamp}_{str_rnd}' log.info(f'Existing experiment moved to {new_foldname}') shutil.move(rundir, rundir.parent/new_foldname) small.mkdir(rundir)
def demo_run(cls, cf, out, dataset, split_vids, tubes_dwein, neth): vf_connections_dwti = cls._define_boxes_to_evaluate( cf, dataset, split_vids, tubes_dwein) vf_connections_dwti = sample_dict( vf_connections_dwti, N=5, NP_SEED=0) vfold = small.mkdir(out/'demovis') cls._demovis_apply(vfold, neth, dataset, vf_connections_dwti)
def _simple_gpu_compute( cls, out, dataset, neth, vf_connections_dwti ) -> Dict[Vid_daly, Dict[int, np.ndarray]]: """Progress saved on video-level scale""" def isaver_eval_func(vid): f_connections_dwti = vf_connections_dwti[vid] video_path = dataset.videos[vid]['path'] finds = list(f_connections_dwti) with vt_cv.video_capture_open(video_path) as vcap: frames_u8 = vt_cv.video_sample( vcap, finds, debug_filename=video_path) f_cls_probs = {} for find, frame_BGR in zip(finds, frames_u8): connections_dwti = f_connections_dwti[find] boxes = connections_dwti['boxes'] cls_probs = neth.score_boxes(frame_BGR, boxes) # N, (bcg+10) f_cls_probs[find] = cls_probs return f_cls_probs vids_to_eval = list(vf_connections_dwti.keys()) isaver = snippets.Isaver_simple( small.mkdir(out/'isave_rcnn_vid_eval'), vids_to_eval, isaver_eval_func, '::10', 120) isaver_items = isaver.run() vf_cls_probs: Dict[Vid_daly, Dict[int, np.ndarray]] vf_cls_probs = dict(zip(vids_to_eval, isaver_items)) return vf_cls_probs
def _train_routine(cf, cf_add_d2, out, cls_names, TRAIN_DATASET_NAME, datalist, add_args): num_classes = len(cls_names) d2_output_dir = str(small.mkdir(out/'d2_output')) d_cfg = set_detectron_cfg_base( d2_output_dir, num_classes, cf['seed']) d_cfg = set_detectron_cfg_train( d_cfg, TRAIN_DATASET_NAME, cf_add_d2) num_gpus = cf['num_gpus'] nargs = argparse.Namespace() nargs.datalist = datalist nargs.TRAIN_DATASET_NAME = TRAIN_DATASET_NAME nargs.cls_names = cls_names nargs.resume = True nargs.trainer = DalyVideoDatasetTrainer dist_url = _figure_out_disturl(add_args) launch_w_logging(_detectron_train_function, num_gpus, num_machines=1, machine_rank=0, dist_url=dist_url, args=(d_cfg, cf, nargs))
def _eval_routine(cf, cf_add_d2, out, cls_names, TEST_DATASET_NAME, datalist: Datalist, model_to_eval): num_classes = len(cls_names) d2_output_dir = str(small.mkdir(out/'d2_output')) d_cfg = set_detectron_cfg_base( d2_output_dir, num_classes, cf['seed']) d_cfg = set_detectron_cfg_test( d_cfg, TEST_DATASET_NAME, model_to_eval, cf['conf_thresh'], cf_add_d2) simple_d2_setup(d_cfg) predictor = DefaultPredictor(d_cfg) cpu_device = torch.device("cpu") def eval_func(dl_item: Dl_record): video_path: Path = dl_item['video_path'] frame_number: int = dl_item['video_frame_number'] frame_time: float = dl_item['video_frame_time'] frame_u8 = get_frame_without_crashing( video_path, frame_number, frame_time) predictions = predictor(frame_u8) cpu_instances = predictions["instances"].to(cpu_device) return cpu_instances df_isaver = snippets.Isaver_simple( small.mkdir(out/'isaver'), datalist, eval_func, '::50') predicted_datalist = df_isaver.run() if cf['nms.enable']: nms_thresh = cf['nms.thresh'] nmsed_predicted_datalist = [] for pred_item in predicted_datalist: if cf['nms.batched']: keep = batched_nms(pred_item.pred_boxes.tensor, pred_item.scores, pred_item.pred_classes, nms_thresh) else: keep = nms(pred_item.pred_boxes.tensor, pred_item.scores, nms_thresh) nmsed_item = pred_item[keep] nmsed_predicted_datalist.append(nmsed_item) predicted_datalist = nmsed_predicted_datalist log.info('AP v2:') computeprint_ap_for_video_datalist(cls_names, datalist, predicted_datalist)
def additional_logging(rundir, start_gstep): # Also log to rundir id_string = snippets.get_experiment_id_string() logfilename = small.mkdir(rundir) / '{}_{}.log'.format( start_gstep, id_string) out_filehandler = logging.FileHandler(str(logfilename)) LOG_FORMATTER = logging.Formatter("%(asctime)s %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S") out_filehandler.setFormatter(LOG_FORMATTER) out_filehandler.setLevel(logging.INFO) logging.getLogger().addHandler(out_filehandler) # Also print platform info log.info(snippets.platform_info())
def _involved_gpu_compute( cls, out, dataset, neth, vf_connections_dwti, size_video_chunk, ) -> Dict[Vid_daly, Dict[int, np.ndarray]]: frame_chunks = [] vids_to_eval = list(vf_connections_dwti.keys()) for vid in vids_to_eval: f_connections_dwti = vf_connections_dwti[vid] finds = np.array(list(f_connections_dwti)) finds_split = snippets.leqn_split( finds, size_video_chunk) for subset_finds in finds_split: frame_chunks.append((vid, subset_finds)) def isaver_eval_func(frame_chunk): vid, finds = frame_chunk f_connections_dwti = vf_connections_dwti[vid] video_path = dataset.videos[vid]['path'] with vt_cv.video_capture_open(video_path) as vcap: frames_u8 = vt_cv.video_sample( vcap, finds, debug_filename=video_path) f_cls_probs = {} for find, frame_BGR in zip(finds, frames_u8): connections_dwti = f_connections_dwti[find] boxes = connections_dwti['boxes'] cls_probs = neth.score_boxes( frame_BGR, boxes) # N, (bcg+10) f_cls_probs[find] = cls_probs return f_cls_probs isaver = snippets.Isaver_simple( small.mkdir(out/'isave_rcnn_vid_eval'), frame_chunks, isaver_eval_func, save_interval=60, log_interval=300) isaver_items = isaver.run() vf_cls_probs: Dict[Vid_daly, Dict[int, np.ndarray]] vf_cls_probs = {} for (vid, subset_finds), f_cls_probs in zip( frame_chunks, isaver_items): vf_cls_probs.setdefault(vid, {}).update(f_cls_probs) return vf_cls_probs
def __init__(self, cf, cf_add_d2, dataset, out): num_classes = len(dataset.action_names) TEST_DATASET_NAME = 'daly_objaction_test' # / Define d2 conf d2_output_dir = str(small.mkdir(out / 'd2_output')) d_cfg = set_detectron_cfg_base(d2_output_dir, num_classes, cf['seed']) d_cfg = set_detectron_cfg_test(d_cfg, TEST_DATASET_NAME, cf['d2_rcnn.model'], cf['d2_rcnn.conf_thresh'], cf_add_d2, freeze=False) d_cfg.MODEL.PROPOSAL_GENERATOR.NAME = "PrecomputedProposals" d_cfg.freeze() # / Start d2 simple_d2_setup(d_cfg) # Predictor without proposal generator model = build_model(d_cfg) model.eval() checkpointer = DetectionCheckpointer(model) checkpointer.load(d_cfg.MODEL.WEIGHTS) MIN_SIZE_TEST = d_cfg.INPUT.MIN_SIZE_TEST MAX_SIZE_TEST = d_cfg.INPUT.MAX_SIZE_TEST transform_gen = d2_transforms.ResizeShortestEdge( [MIN_SIZE_TEST, MIN_SIZE_TEST], MAX_SIZE_TEST) # Instance monkeypatching # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module/50600307#50600307 model.forward = MethodType(genrcnn_rcnn_roiscores_forward, model) self.d_cfg = d_cfg self.rcnn_roiscores_model = model self.cpu_device = torch.device("cpu") self.transform_gen = transform_gen
def _demovis_apply(cls, vfold, neth, dataset: Dataset_daly_ocv, vf_connections_dwti): nicolas_labels = ['background', ] + cast(List[str], dataset.action_names) for vid, f_connections_dwti in tqdm( vf_connections_dwti.items(), 'nicphil_demovis'): video_path = dataset.videos[vid]['path'] finds = list(f_connections_dwti) with vt_cv.video_capture_open(video_path) as vcap: frames_u8 = vt_cv.video_sample( vcap, finds, debug_filename=video_path) video_fold = small.mkdir(vfold/f'vid{vid}') for find, frame_BGR in zip(finds, frames_u8): connections_dwti = f_connections_dwti[find] boxes = connections_dwti['boxes'] box_cls_probs = neth.score_boxes(frame_BGR, boxes) # N, (bcg+10) # Draw and print txt_output = [] image = frame_BGR.copy() for i, cls_probs in enumerate(box_cls_probs): box = boxes[i] best_score_id = np.argmax(cls_probs) best_score = cls_probs[best_score_id] best_nicolas_label = nicolas_labels[best_score_id] snippets.cv_put_box_with_text(image, box, text='{} {} {:.2f}'.format( i, best_nicolas_label, best_score)) line = (' '.join([f'{y}: {x:.3f}' for x, y in zip(cls_probs, nicolas_labels)]) + str(box)) txt_output.append(line) cv2.imwrite(str( video_fold/'Fr{:05d}.png'.format(find)), image) with (video_fold/f'Fr{find:05d}_scores.txt').open('w') as f: f.write('\n'.join(txt_output))
def tubefeats_dist_train_mlp(workfolder, cfg_dict, add_args): """ Training of MLP trainfeats in the same way as we finetune stuff """ out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig_v2(cfg_dict) Ncfg_daly.set_defcfg_v2(cfg) cfg.set_defaults_yaml(""" inputs: keyframes: fold: ~ featname: ~ """) cfg.set_defaults_yaml(""" inputs: tubes_dwein: ~ tubes_dwein_feats: fold: ~ kind: !def ['roipooled', ['fullframe', 'roipooled']] ckpt: ~ seed: 42 split_assignment: !def ['train/val', ['train/val', 'trainval/test']] data_scaler: !def ['no', ['keyframes', 'no']] net: n_outputs: !def [~, [10, 11]] train: lr: 1.0e-05 weight_decay: 5.0e-2 start_epoch: 0 n_epochs: 120 batch_size: 32 tubes: top_n_matches: 1 stride: 4 frame_dist: 16 add_keyframes: True period: i_batch: loss_log: '::' i_epoch: loss_log: '0::1' q_eval: '::' full_eval: '0,1,2,3,4::5' eval: full_tubes: enabled: True detect_mode: !def ['roipooled', ['fullframe', 'roipooled']] nms: 0.3 field_nms: 'box_det_score' # hscore field_det: 'box_det_score' # hscore*frame_cls_score """) cf = cfg.parse() # Seeds initial_seed = cf['seed'] torch.manual_seed(initial_seed) # Data # General DALY level preparation dataset = Ncfg_daly.get_dataset(cf) vgroup = Ncfg_daly.get_vids(cf, dataset) sset_train, sset_eval = cf['split_assignment'].split('/') vids_train, vids_eval = vgroup[sset_train], vgroup[sset_eval] man_feats_kf = Manager_feats_keyframes( cf['inputs.keyframes.fold'], cf['inputs.keyframes.featname']) # wein tubes tubes_dwein_d, tubes_dgt_d = load_gt_and_wein_tubes( cf['inputs.tubes_dwein'], dataset, vgroup) tubes_dwein_prov: Dict[I_dwein, Tube_daly_wein_as_provided] = \ small.load_pkl(cf['inputs.tubes_dwein']) # synthetic tube labels _, dwti_to_label_eval = qload_synthetic_tube_labels( tubes_dgt_d[sset_eval], tubes_dwein_d[sset_eval], dataset) # Ssset tkfeats_train = man_feats_kf.split_off(vids_train, True) tkfeats_eval = man_feats_kf.split_off(vids_eval, True) tubes_dwein_train = tubes_dwein_d[sset_train] tubes_dwein_eval = tubes_dwein_d[sset_eval] tubes_dgt_train = tubes_dgt_d[sset_train] tubes_dgt_eval = tubes_dgt_d[sset_eval] # Interacting with big assert cf['eval.full_tubes.enabled'], 'We train on them anyway' top_n_matches = cf['train.tubes.top_n_matches'] stride = cf['train.tubes.stride'] detect_mode = cf['inputs.tubes_dwein_feats.kind'] man_feats_dwt = create_preextracted_feats_manager( cf, None, detect_mode) max_distance = cf['train.tubes.frame_dist'] output_dims = cf['net.n_outputs'] if detect_mode == 'fullframe': # fullframes labeled_frames: List[Frame_labeled] = \ prepare_label_fullframes_for_training( tubes_dgt_train, dataset, stride, max_distance) # associate to extracted feats labeled_linked_frames: List[Frame_labeled] = \ _link_lframes_to_exfeats(labeled_frames, man_feats_dwt) tdataset = TDataset_over_labeled_linked_frames( labeled_linked_frames, man_feats_dwt) assert output_dims == 10 D_in = man_feats_dwt.fullframe_feats.shape[-1] elif detect_mode == 'roipooled': # roitubes tkfeats_train_numpy = man_feats_kf.split_off(vids_train, False) keyframes_train = tkfeats_train_numpy['kf'] keyframe_feats_train = tkfeats_train_numpy['X'] labeled_boxes: List[Box_labeled] = \ prepare_label_roiboxes_for_training( tubes_dgt_train, dataset, stride, max_distance, tubes_dwein_train, keyframes_train, top_n_matches, cf['train.tubes.add_keyframes']) # associate roiboxes to extracted feats labeled_linked_boxes: List[Box_labeled_linked] = \ _link_lboxes_to_exfeats(labeled_boxes, man_feats_dwt) tdataset = TDataset_over_labeled_linked_boxes( labeled_linked_boxes, man_feats_dwt, keyframe_feats_train) assert output_dims == 11 D_in = man_feats_dwt.BIG.shape[-1] else: raise RuntimeError() # Model model = Net_mlp_onelayer( D_in, output_dims, 32, 0.5) loss_fn = torch.nn.CrossEntropyLoss(reduction='mean') optimizer = torch.optim.AdamW(model.parameters(), lr=cf['train.lr'], weight_decay=cf['train.weight_decay']) ckpt = Checkpointer(model, optimizer) # Restore previous run rundir = small.mkdir(out/'rundir') checkpoint_path = (Manager_checkpoint_name.find_last_checkpoint(rundir)) if '--new' in add_args: Manager_checkpoint_name.rename_old_rundir(rundir) checkpoint_path = None start_epoch = (ckpt.restore_model_magic(checkpoint_path, cf['inputs.ckpt'], cf['train.start_epoch'])) # Training n_epochs = cf['train.n_epochs'] for i_epoch in range(start_epoch, n_epochs): log.info(f'Started epoch {i_epoch=}') model.train() l_avg = snippets.misc.Averager() avg_bs = snippets.misc.Averager() # Loader reproducible even if we restore rgen = np.random.default_rng(initial_seed+i_epoch) loader = _get_trainloader_rnd_sampler(tdataset, cf['train.batch_size'], rgen) for i_batch, (data_input) in enumerate(loader): # inputs converter (meta,) = data_input flat_labels = np.hstack([m['labels'] for m in meta]) flat_feats = np.vstack([m['feats'] for m in meta]) flat_labels_t = torch.from_numpy(flat_labels) flat_feats_t = torch.from_numpy(flat_feats) result = model(flat_feats_t) pred_train = result['x_final'] loss = loss_fn(pred_train, flat_labels_t) optimizer.zero_grad() loss.backward() optimizer.step() avg_bs.update(len(flat_labels)) l_avg.update(loss.item()) if check_step(i_batch, cf['period.i_batch.loss_log']): Nb = len(loader) loss_str = (f'loss(all/last):{l_avg.avg:.4f}/{l_avg.last:.4f}') log.info(f'{i_epoch=}, {i_batch=}/{Nb}; {loss_str}') log.info('Epoch stats: avg_batchsize {}, loader_size {} '.format( avg_bs.avg, len(loader))) ckpt.save_epoch(rundir, i_epoch) if check_step(i_epoch, cf['period.i_epoch.loss_log']): log.info(f'Loss at {i_epoch=}: {l_avg.avg}') if check_step(i_epoch, cf['period.i_epoch.q_eval']): model.eval() kacc_train = quick_accuracy_over_kfeat( tkfeats_train, model, True)*100 kacc_eval = quick_accuracy_over_kfeat( tkfeats_eval, model, True)*100 model.train() log.info(f'Qperf at {i_epoch=}: ' f'{kacc_train=:.2f} {kacc_eval=:.2f}') if check_step(i_epoch, cf['period.i_epoch.full_eval']): model.eval() result = mlp_perf_kf_evaluate( model, tkfeats_train, tkfeats_eval, tubes_dwein_eval, tubes_dgt_eval, dataset, output_dims) result_fulltube = mlp_perf_fulltube_evaluate( model, man_feats_dwt, tubes_dwein_eval, tubes_dwein_prov, tubes_dgt_eval, dwti_to_label_eval, dataset, output_dims, cf['eval.full_tubes.detect_mode'], cf['eval.full_tubes.nms'], cf['eval.full_tubes.field_nms'], cf['eval.full_tubes.field_det']) result.update(result_fulltube) model.train() log.info(f'Evalset perf at {i_epoch=}') mlp_perf_display(result, sset_eval)
def get_subfolders(folder, subfolder_names=['out', 'temp']): return [small.mkdir(folder / name) for name in subfolder_names]
def merge_scores_avstubes(workfolder, cfg_dict, add_args): out, = snippets.get_subfolders(workfolder, ['out']) cfg = snippets.YConfig(cfg_dict) cfg.set_defaults_handling(raise_without_defaults=False) Ncfg_dataset.set_dataset_seed(cfg) Ncfg_tube_eval.set_defcfg(cfg) cfg.set_defaults(""" tube_dict: ~ combinations: enabled: False sizes: ~ """) cf = cfg.parse() dataset, split_vids, av_gt_tubes = \ Ncfg_dataset.resolve_dataset_tubes(cf) ts = {k: small.load_pkl(v) for k, v in cfg_dict['tube_dict'].items()} if not cf['combinations.enabled']: av_stubes = _meanpool_avstubes(list(ts.values())) small.save_pkl(out / 'merged_av_stubes.pkl', av_stubes) log.info('All combined score:') Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes) return sizes = cf['combinations.sizes'] combinations = [list(itertools.combinations(ts.keys(), r)) for r in sizes] combinations = list(itertools.chain(*combinations)) log.info('Combinations: {}'.format(combinations)) comb_dfdicts = {} for comb in combinations: comb_name = '+'.join(comb) comb_fold = small.mkdir(out / comb_name) def compute(): to_merge = [ts[k] for k in comb] av_stubes = _meanpool_avstubes(to_merge) small.save_pkl(comb_fold / 'av_stubes.pkl', av_stubes) dfdict = Ncfg_tube_eval.eval_as_df(cf, av_stubes, av_gt_tubes) return dfdict dfdict = small.stash2(comb_fold / 'stashed_dfdict.pkl')(compute) comb_dfdicts[comb_name] = dfdict log.info('Individual results:') for comb_name, dfdict in comb_dfdicts.items(): log.info(f'Results for {comb_name}:') _print_quick_evaluation_stats(dfdict) log.info('Combined tables:') big_ = {comb: pd.concat(dfdict) for comb, dfdict in comb_dfdicts.items()} big = pd.concat(big_, axis=1) for stat in big.index.levels[0]: log.info(f'=== {stat} ===') for thresh in big.columns.levels[1]: X = (big.loc['ap'].loc[:, pd.IndexSlice[:, thresh]].droplevel(1, axis=1)) table = snippets.df_to_table_v2((X * 100).round(2)) log.info(f'{stat} for IOU {thresh}:\n{table}') log.info('\n')