def train_daly_object(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
_set_defcfg_detectron(cfg)
_set_defcfg_detectron_train(cfg)
_set_defcfg_object_hacks(cfg)
cf = cfg.parse()
cf_add_d2 = cfg.without_prefix('d2.')
cf_add_d2 = d2dict_gpu_scaling(cf, cf_add_d2, cf['num_gpus'])
dataset = Dataset_daly_ocv()
dataset.populate_from_folder(cf['dataset.cache_folder'])
o100_objects, category_map = get_category_map_o100(dataset)
assert len(o100_objects) == 16
split_label = cf['dataset.subset']
split_vids = get_daly_split_vids(dataset, split_label)
datalist: Datalist = simplest_daly_to_datalist_v2(dataset, split_vids)
cls_names, datalist_converter = \
_datalist_hacky_converter(cf, dataset)
datalist = datalist_converter(datalist)
TRAIN_DATASET_NAME = 'daly_objaction_train'
_train_routine(cf, cf_add_d2, out,
cls_names, TRAIN_DATASET_NAME, datalist, add_args)
def eval_daly_object(workfolder, cfg_dict, add_args):
"""
Evaluation code with hacks
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
_set_defcfg_detectron(cfg)
_set_defcfg_detectron_test(cfg)
_set_defcfg_object_hacks(cfg)
cfg.set_deftype("""
what_to_eval: [~, str]
eval_hacks:
model_to_eval: ['what', ['what', 'what+foldname']]
""")
cf = cfg.parse()
cf_add_d2 = cfg.without_prefix('d2.')
# DALY Dataset
dataset = Dataset_daly_ocv()
dataset.populate_from_folder(cf['dataset.cache_folder'])
split_label = cf['dataset.subset']
split_vids = get_daly_split_vids(dataset, split_label)
datalist: Datalist = simplest_daly_to_datalist_v2(dataset, split_vids)
cls_names, datalist_converter = \
_datalist_hacky_converter(cf, dataset)
datalist = datalist_converter(datalist)
TEST_DATASET_NAME = 'daly_objaction_test'
model_to_eval = _eval_foldname_hack(cf)
_eval_routine(cf, cf_add_d2, out, cls_names,
TEST_DATASET_NAME, datalist, model_to_eval)
def eval_avstubes(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(raise_without_defaults=False)
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tubes.set_defcfg(cfg)
cfg.set_defaults("""
tubes_path: ~
""")
Ncfg_tube_eval.set_defcfg(cfg)
cf = cfg.parse()
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
tubes_to_eval: AV_dict[T_dwein_scored] = \
small.load_pkl(cf['tubes_path'])
# Extended version of "Ncfg_tube_eval.evalprint_if"
nms_thresh = cf['tube_eval.nms.thresh']
iou_thresholds: List[float] = cf['tube_eval.iou_thresholds']
minscore_cutoff = cf['tube_eval.minscore_cutoff']
tubes_to_eval = av_stubes_above_score(tubes_to_eval, minscore_cutoff)
tubes_to_eval = \
compute_nms_for_av_stubes(tubes_to_eval, nms_thresh)
dfdict = _compute_exhaustive_evaluation_stats(av_gt_tubes, tubes_to_eval,
iou_thresholds)
_print_exhaustive_evaluation_stats(dfdict)
def apply_pncaffe_rcnn_in_frames(workfolder, cfg_dict, add_args):
"""
Apply Phil-Nic rcnn model on tube boxes to extract per-action scores
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tubes.set_defcfg(cfg)
Ncfg_nicphil_rcnn.set_defcfg(cfg)
Ncfg_generic_rcnn_eval.set_defcfg(cfg)
Ncfg_tube_eval.set_defcfg(cfg)
cf = cfg.parse()
# Preparation
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
tubes_dwein: Dict[I_dwein, T_dwein] = \
Ncfg_tubes.resolve_tubes_dwein(cf, split_vids)
neth: Nicolas_net_helper = Ncfg_nicphil_rcnn.resolve_helper(cf)
# Experiment logic
if cf['demo_run.enabled']:
Ncfg_generic_rcnn_eval.demo_run(cf, out, dataset, split_vids,
tubes_dwein, neth)
return
vf_connections_dwti, vf_cls_probs = \
Ncfg_generic_rcnn_eval.evaluate_rcnn_boxes(
cf, out, dataset, split_vids, tubes_dwein, neth)
small.save_pkl(out / 'vf_connections_dwti.pkl', vf_connections_dwti)
small.save_pkl(out / 'vf_cls_probs.pkl', vf_cls_probs)
agg_kind = cf['score_agg_kind']
av_stubes = Ncfg_generic_rcnn_eval.aggregate_rcnn_scores(
dataset, tubes_dwein, vf_connections_dwti, vf_cls_probs, agg_kind)
small.save_pkl(out / 'av_stubes.pkl', av_stubes)
# Post experiment
Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def compare_data(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(raise_without_defaults=False)
cfg.set_defaults("""
cache_folders: ~
""")
cf = cfg.parse()
dataset = Dataset_daly_ocv()
dataset.precompute_to_folder(out)
def load_wein_tubes(workfolder, cfg_dict, add_args):
"""
Philippe tubes:
tube:
(one row per frame):
index of the frame (starting at 1)
x1 y1 x2 y2
score of the generic human detector
score of the instance-specific detector
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_deftype("""
wein_tubes: [~, str]
dataset:
cache_folder: [~, str]
mirror: ['uname', ~]
wein_index: [0, int] # 0 = 510 cut tubes, 1 = 200 uncut tubes
""")
cf = cfg.parse()
# Dataset
dataset = Dataset_daly_ocv(cf['dataset.mirror'])
dataset.populate_from_folder(cf['dataset.cache_folder'])
# Tubes
# 0 has 510 elements, 1 has 200 elements
wein_package = small.load_py2_pkl(cf['wein_tubes'])[cf['wein_index']]
# We got a dictionary of filenames (w .mp4 suffix)
extracted_tubes: Dict[I_dwein, Tube_daly_wein_as_provided] = {}
for vid_mp4, wein_bunches in wein_package.items():
vid = re.search(r'(.*)\.mp4', vid_mp4).group(1)
rs = dataset.rstats[vid]
for bunch_id, wein_tubes in enumerate(wein_bunches):
for tube_id, wein_tube in enumerate(wein_tubes):
frame_inds = wein_tube[:, 0].astype(np.int) - 1
assert max(frame_inds) < rs['max_pos_frames']
boxes_ltrd = wein_tube[:, 1:5] # ltrd
human_scores = wein_tube[:, 5]
instance_scores = wein_tube[:, 6]
tube = {
'frame_inds': frame_inds,
'boxes': boxes_ltrd,
'hscores': human_scores,
'iscores': instance_scores
}
extracted_tubes[(vid, bunch_id, tube_id)] = tube
small.save_pkl(out / 'extracted_tubes.pkl', extracted_tubes)
def gather_reapply_agg_rcnn_avstubes(workfolder, cfg_dict, add_args):
"""
Will apply aggregation again
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(['gather.paths'])
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tubes.set_defcfg(cfg)
cfg.set_deftype("""
gather:
paths: [~, ~]
score_agg_kind: ['mean', ['mean', 'max', 'sum']]
""")
Ncfg_tube_eval.set_defcfg(cfg)
cf = cfg.parse()
# Preparation
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
tubes_dwein: Dict[I_dwein, T_dwein] = \
Ncfg_tubes.resolve_tubes_dwein(cf, split_vids)
# Experiment logic
gather_paths = cf['gather.paths']
if not _gather_check_all_present(
gather_paths, ['vf_cls_probs.pkl', 'vf_connections_dwti.pkl']):
return
vf_connections_dwti = {}
vf_cls_probs = {}
for path in gather_paths:
path = Path(path)
vf_cls_probs_ = small.load_pkl(path / 'vf_cls_probs.pkl')
vf_connections_dwti_ = small.load_pkl(path / 'vf_connections_dwti.pkl')
assert vf_cls_probs_.keys() == vf_connections_dwti_.keys()
vf_cls_probs.update(vf_cls_probs_)
vf_connections_dwti.update(vf_connections_dwti_)
small.save_pkl(out / 'vf_connections_dwti.pkl', vf_connections_dwti)
small.save_pkl(out / 'vf_cls_probs.pkl', vf_cls_probs)
agg_kind = cf['score_agg_kind']
av_stubes: AV_dict[T_dwein_scored] = \
Ncfg_generic_rcnn_eval.aggregate_rcnn_scores(
dataset, tubes_dwein, vf_connections_dwti,
vf_cls_probs, agg_kind)
small.save_pkl(out / 'av_stubes.pkl', av_stubes)
# Post experiment
Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def assign_objactions_to_tubes(workfolder, cfg_dict, add_args):
"""
Score tubes by assigning objactions to them and pooling the scores,
then evaluate resulting scored tubes
- Objactions: detecton evaluated datalist or gt objects (per frame)
- Tubes: philippe tubes
- Assignment: inner overlap or iou scores
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tubes.set_defcfg(cfg)
cfg.set_deftype("""
actobjects:
source: ['detected', ['detected', 'gt']]
detected:
path: [~, ~]
obj_to_tube:
overlap_type: ['inner_overlap', ['inner_overlap', 'iou']]
overlap_cutoff: [0.2, float]
score_cutoff: [0.2, float]
""")
Ncfg_tube_eval.set_defcfg(cfg)
cf = cfg.parse()
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
# Inputs to the assignment routine
ftubes: Dict[I_dwein, T_dwein] = \
Ncfg_tubes.resolve_tubes_dwein(cf, split_vids)
objactions_vf: Dict[Vid_daly, Dict[int, Objaction_dets]] = \
_resolve_actobjects(cf, dataset, split_vids)
# Assignment itself
overlap_type = cf['obj_to_tube.overlap_type']
overlap_cutoff = cf['obj_to_tube.overlap_cutoff']
score_cutoff = cf['obj_to_tube.score_cutoff']
av_stubes: AV_dict[T_dwein_scored] = \
score_ftubes_via_objaction_overlap_aggregation(
dataset, objactions_vf, ftubes, overlap_type,
overlap_cutoff, score_cutoff)
small.save_pkl(out / 'av_stubes.pkl', av_stubes)
Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def train_daly_action(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
_set_defcfg_detectron(cfg)
_set_defcfg_detectron_train(cfg)
cf = cfg.parse()
cf_add_d2 = cfg.without_prefix('d2.')
cf_add_d2 = d2dict_gpu_scaling(cf, cf_add_d2, cf['num_gpus'])
dataset = Dataset_daly_ocv()
dataset.populate_from_folder(cf['dataset.cache_folder'])
split_label = cf['dataset.subset']
split_vids = get_daly_split_vids(dataset, split_label)
datalist: Datalist = daly_to_datalist_pfadet(dataset, split_vids)
cls_names = dataset.action_names
TRAIN_DATASET_NAME = 'daly_pfadet_train'
_train_routine(cf, cf_add_d2, out,
cls_names, TRAIN_DATASET_NAME, datalist, add_args)
def apply_pfadet_rcnn_in_frames(workfolder, cfg_dict, add_args):
"""
Apply trained d2 frcnn model on tube boxes to extract per-action scores
- We dispense with the frcnn box predictions and only use per-roi scores
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(['d2.'])
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tubes.set_defcfg(cfg)
Ncfg_generic_rcnn_eval.set_defcfg(cfg)
cfg.set_deftype("""
d2_rcnn:
model: [~, ~]
conf_thresh: [0.0, float]
""")
Ncfg_tube_eval.set_defcfg(cfg)
cf = cfg.parse()
cf_add_d2 = cfg.without_prefix('d2.')
# Preparation
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
tubes_dwein: Dict[I_dwein, T_dwein] = \
Ncfg_tubes.resolve_tubes_dwein(cf, split_vids)
neth = D2_rcnn_helper(cf, cf_add_d2, dataset, out)
# Experiment logic
if cf['demo_run.enabled']:
Ncfg_generic_rcnn_eval.demo_run(cf, out, dataset, split_vids,
tubes_dwein, neth)
return
vf_connections_dwti, vf_cls_probs = \
Ncfg_generic_rcnn_eval.evaluate_rcnn_boxes(
cf, out, dataset, split_vids, tubes_dwein, neth)
small.save_pkl(out / 'vf_connections_dwti.pkl', vf_connections_dwti)
small.save_pkl(out / 'vf_cls_probs.pkl', vf_cls_probs)
agg_kind = cf['score_agg_kind']
av_stubes = Ncfg_generic_rcnn_eval.aggregate_rcnn_scores(
dataset, tubes_dwein, vf_connections_dwti, vf_cls_probs, agg_kind)
small.save_pkl(out / 'av_stubes.pkl', av_stubes)
# Post experiment
Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
def precompute_cache(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_deftype("""
dataset: [~, str]
charades:
mirror: ['gpuhost7', str]
resolution: ['480', str]
daly:
mirror: [~, str]
""")
cf = cfg.parse()
if cf['dataset'] == 'daly':
dataset = Dataset_daly_ocv(cf['daly.mirror'])
elif cf['dataset'] == 'charades':
dataset = Dataset_charades_ocv(cf['charades.mirror'],
cf['charades.resolution'])
elif cf['dataset'] == 'voc2007':
raise NotImplementedError()
else:
raise RuntimeError('Wrong dataset')
dataset.precompute_to_folder(out)
def eval_stubes_experimental(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(raise_without_defaults=False)
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tubes.set_defcfg(cfg)
cfg.set_defaults("""
tubes_path: ~
""")
Ncfg_tube_eval.set_defcfg(cfg)
cf = cfg.parse()
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
tubes_to_eval: AV_dict[T_dwein_scored] = \
small.load_pkl(cf['tubes_path'])
dgt_tubes: Dict[I_dgt, T_dgt] = _reindex_avd_to_dgt(av_gt_tubes)
tubes_dwein: Dict[I_dwein, T_dwein]
tubes_dwein = loadconvert_tubes_dwein('/home/vsydorov/projects/deployed/2019_12_Thesis/dervo/thes/gpuhost7/2020_03_31/10_clean_inputs/20_wein_tubes/thes.gpuhost7.2020_03_31.10_clean_inputs.20_wein_tubes/194e3a301c202bc5e818dca26953ddb268aa98b3/out/extracted_tubes.pkl')
tubes_dwein = dtindex_filter_split(tubes_dwein, split_vids)
wgi, gti = _get_weingroup_assignment(dgt_tubes, tubes_dwein)
wgi_to_gti: Dict[I_weingroup, I_dgt] = dict(zip(wgi, gti))
# Evaluation
keyframe_av_stubes = small.load_pkl('/home/vsydorov/projects/deployed/2019_12_Thesis/dervo/thes/gpuhost7/2020_03_31/30_whole_video/30_pncaffe_rcnn/05_develop/10_splits_keyframes/thes.gpuhost7.2020_03_31.30_whole_video.30_pncaffe_rcnn.05_develop.10_splits_keyframes/RAW/out/av_stubes.pkl')
# Recall without NMS
iou_thresholds = cf['tube_eval.iou_thresholds']
nms_thresh = 0.5
recall_norm = compute_recall_for_avtubes_as_dfs(
av_gt_tubes, norm_av_stubes, iou_thresholds, False)[0]
recall_kf = compute_recall_for_avtubes_as_dfs(
av_gt_tubes, keyframe_av_stubes, iou_thresholds, False)[0]
# Ap without NMS
ap_norm = compute_ap_for_avtubes_as_df(
av_gt_tubes, norm_av_stubes, iou_thresholds, False, False)
ap_kf = compute_ap_for_avtubes_as_df(
av_gt_tubes, keyframe_av_stubes, iou_thresholds, False, False)
# # Apply "special NMS"
# wnms_norm_av_stubes = _weingroup_nms(norm_av_stubes, nms_thresh)
# wnms_keyframe_av_stubes = _weingroup_nms(keyframe_av_stubes, nms_thresh)
# Apply "special NMS"
wnms_norm_av_stubes = \
(small.stash2(out/'_temp_wnms_norm_av_stubes.pkl')
(_weingroup_nms, norm_av_stubes, nms_thresh))
wnms_keyframe_av_stubes = \
(small.stash2(out/'_temp_wnms_keyframe_av_stubes.pkl')
(_weingroup_nms, keyframe_av_stubes, nms_thresh))
# Recall with "special" NMS
wnms_recall_norm = compute_recall_for_avtubes_as_dfs(
av_gt_tubes, wnms_norm_av_stubes, iou_thresholds, False)[0]
wnms_recall_kf = compute_recall_for_avtubes_as_dfs(
av_gt_tubes, wnms_keyframe_av_stubes, iou_thresholds, False)[0]
# AP with "special" NMS.
wnms_ap_norm = compute_ap_for_avtubes_as_df(
av_gt_tubes, wnms_norm_av_stubes, iou_thresholds, False, False)
wnms_ap_kf = compute_ap_for_avtubes_as_df(
av_gt_tubes, wnms_keyframe_av_stubes, iou_thresholds, False, False)
# WG AP with "special" NMS
wnms_wap_norm = compute_ap_for_avtubes_WG_as_df(wgi_to_gti,
av_gt_tubes, wnms_norm_av_stubes, iou_thresholds, False, False)
wnms_wap_kf = compute_ap_for_avtubes_WG_as_df(wgi_to_gti,
av_gt_tubes, wnms_keyframe_av_stubes, iou_thresholds, False, False)
import pudb; pudb.set_trace() # XXX BREAKPOINT
def vis_stubes(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(raise_without_defaults=False)
Ncfg_dataset.set_dataset_seed(cfg)
cfg.set_defaults("""
paths:
av_stubes: ~
vf_connections_dwti: ~
vf_cls_probs: ~
""")
cf = cfg.parse()
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
av_stubes: AV_dict[T_dwein_scored] = \
small.load_pkl(cf['paths.av_stubes'])
vf_connections_dwti: Dict[Vid_daly, Dict[int, Box_connections_dwti]] = \
small.load_pkl(cf['paths.vf_connections_dwti'])
vf_cls_probs: Dict[Vid_daly, Dict[int, np.ndarray]] = \
small.load_pkl(cf['paths.vf_cls_probs'])
# WGI GTI
dgt_tubes: Dict[I_dgt, T_dgt] = _reindex_avd_to_dgt(av_gt_tubes)
tubes_dwein: Dict[I_dwein, T_dwein] = loadconvert_tubes_dwein('/home/vsydorov/projects/deployed/2019_12_Thesis/dervo/thes/gpuhost7/2020_03_31/10_clean_inputs/20_wein_tubes/thes.gpuhost7.2020_03_31.10_clean_inputs.20_wein_tubes/194e3a301c202bc5e818dca26953ddb268aa98b3/out/extracted_tubes.pkl')
tubes_dwein = dtindex_filter_split(tubes_dwein, split_vids)
wgi, gti = _get_weingroup_assignment(dgt_tubes, tubes_dwein)
wgi_to_gti: Dict[I_weingroup, I_dgt] = dict(zip(wgi, gti))
nms_thresh = 0.5
wnms_av_stubes = \
(small.stash2(out/'_temp_wnms_norm_av_stubes.pkl')
(_weingroup_nms, av_stubes, nms_thresh))
iou_thresholds = [0.3, 0.5]
co_wnms_av_stubes = av_stubes_above_score(
wnms_av_stubes, 0.05)
wnms_ap = compute_ap_for_avtubes_as_df(
av_gt_tubes, co_wnms_av_stubes, iou_thresholds, False, False)
action = 'Drinking'
action_dwti_ind = dataset.action_names.index(action) + 1
vfold = small.mkdir(out/'ap_emulate'/action)
v_stubes = wnms_av_stubes[action]
v_gt_tubes = av_gt_tubes[action]
# ap-like preparations
fgts, fdets = _convert_to_flat_representation(v_gt_tubes, v_stubes)
det_to_eligible_gt = _compute_eligible_tubes_for_eval_weingroup(
fgts, fdets, wgi_to_gti)
gt_already_matched = np.zeros(len(fgts), dtype=bool)
nd = len(fdets)
tp = np.zeros(nd)
fp = np.zeros(nd)
npos = len([x for x in fgts if not x['diff']])
detection_scores = np.array([x['score'] for x in fdets])
detection_scores = detection_scores.round(3)
sorted_inds = np.argsort(-detection_scores)
iou_thresh = 0.5
use_diff = False
# Provenance
detection_matched_to_which_gt = np.ones(len(fdets), dtype=int)*-1
ifdet_to_iou_coverages = {}
for d, ifdet in enumerate(sorted_inds):
matchable_ifgts = list(det_to_eligible_gt.get(ifdet, {}))
if not len(matchable_ifgts):
fp[d] = 1
continue
iou_coverages: List[float] = \
_compute_iou_coverages(fgts, fdets, det_to_eligible_gt,
matchable_ifgts, ifdet, False)
ifdet_to_iou_coverages[ifdet] = iou_coverages
max_coverage_local_id: int = np.argmax(iou_coverages)
max_coverage: float = iou_coverages[max_coverage_local_id]
max_coverage_ifgt = matchable_ifgts[max_coverage_local_id]
if max_coverage > iou_thresh:
if (not use_diff) and fgts[max_coverage_ifgt]['diff']:
continue
if not gt_already_matched[max_coverage_ifgt]:
tp[d] = 1
gt_already_matched[max_coverage_ifgt] = True
detection_matched_to_which_gt[ifdet] = max_coverage_ifgt
else:
fp[d] = 1
else:
fp[d] = 1
cumsum_fp = np.cumsum(fp)
cumsum_tp = np.cumsum(tp)
rec = cumsum_tp / float(npos)
prec = cumsum_tp / np.maximum(cumsum_tp + cumsum_fp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, False)
log.info(f'{ap=}')
# Visualize this
for d, ifdet in enumerate(sorted_inds):
if not tp[d]:
continue
tube = fdets[ifdet]['obj']
score = tube['score']
sf, ef = tube['start_frame'], tube['end_frame']
vid = tube['index'][0]
frame_inds = tube['frame_inds']
video_fold = small.mkdir(vfold/f'D{d:04d}_IFDET{ifdet:04d}_V({vid})_{sf}_to_{ef}_score{score:02f}')
video_path = dataset.videos[vid]['path']
# Extract
with vt_cv.video_capture_open(video_path) as vcap:
frames_u8 = vt_cv.video_sample(
vcap, frame_inds, debug_filename=video_path)
# Draw
drawn_frames_u8 = []
for i, (find, frame_BGR) in enumerate(zip(frame_inds, frames_u8)):
image = frame_BGR.copy()
box = tube['boxes'][i]
connections_dwti = vf_connections_dwti[vid][find]
scores_dwti = vf_cls_probs[vid][find]
source_id = connections_dwti['dwti_sources'].index(tube['index'])
dwti_box = connections_dwti['boxes'][source_id]
dwti_score = scores_dwti[source_id][action_dwti_ind]
assert np.allclose(dwti_box, box)
snippets.cv_put_box_with_text(image, box,
text='{}({}); {:.2f}/{:.2f}; {}'.format(
i, find, dwti_score, score, action))
drawn_frames_u8.append(image)
# # Save as images
# for find, image in zip(frame_inds, drawn_frames_u8):
# cv2.imwrite(str(
# video_fold/'Fr{:05d}.png'.format(find)), image)
# Save as video
snippets.qsave_video(video_fold/'overlaid.mp4', drawn_frames_u8)
def charades_stats(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_deftype("""
dataset:
cache_folder: [~, str]
charades:
mirror: ['gpuhost7', str]
resolution: ['480', str]
""")
cf = cfg.parse()
mirror = cf['dataset.charades.mirror']
resolution = cf['dataset.charades.resolution']
dataset = Dataset_charades_ocv(mirror, resolution)
dataset.populate_from_folder(cf['dataset.cache_folder'])
pd_vs = pd.DataFrame(dataset.videos_ocv).T
s_split = pd.Series(dataset.split)
pd_vs['split'] = s_split
pd_vs['action_labels'] = pd_vs.actions.apply(
lambda x: [y['name'] for y in x])
train_mask = pd_vs['split'] == 'train'
val_mask = pd_vs['split'] == 'val'
pd_vs_train = pd_vs[train_mask]
pd_vs_val = pd_vs[val_mask]
N_classes = len(dataset.action_names)
log.info(f'N classes: {N_classes}')
log.info('N videos (Total/Train/Val): {}/{}/{}'.format(
len(pd_vs), len(pd_vs_train), len(pd_vs_val)))
log.info('N frames (Total/Train/Val): {}/{}/{}'.format(
pd_vs.nframes.sum(), pd_vs_train.nframes.sum(),
pd_vs_val.nframes.sum()))
l_ = []
for i, a in enumerate(dataset.action_names):
s = pd_vs.action_labels.apply(lambda x: a in x)
s.name = i
l_.append(s)
action_presence = pd.concat(l_, axis=1)
l_ = []
for i, o in enumerate(dataset.object_names):
s = pd_vs.objects.apply(lambda x: o in x)
s.name = i
l_.append(s)
obj_presence = pd.concat(l_, axis=1)
action_mapping_int = {
dataset.action_names.index(a): dataset.object_names.index(o)
for a, (o, v) in dataset.action_mapping.items()
}
column_mapped_obj = {}
for i, column in action_presence.iteritems():
column_mapped_obj.setdefault(action_mapping_int[i], []).append(column)
obj_presence_mapped = pd.DataFrame({
k: pd.concat(v, axis=1).any(axis=1)
for k, v in column_mapped_obj.items()
}).sort_index(axis=1)
# 0/'None' object to be removed
obj_presence0 = obj_presence.loc[:, 1:]
obj_presence_mapped0 = obj_presence_mapped.loc[:, 1:]
def frac_str(X, Y):
return f'{X}/{Y}/{X/Y*100:.2f}%'
def mean_frac(X):
return frac_str(X.sum(), len(X))
X_extra_present = ((obj_presence0 > obj_presence_mapped0).any(axis=1))
log.info('Videos have extra objects present, beyond mapped: '
'[Total: {}] [Train: {}] [Val: {}]'.format(
mean_frac(X_extra_present),
mean_frac(X_extra_present[train_mask]),
mean_frac(X_extra_present[val_mask])))
X_impobj_missing = (obj_presence0 < obj_presence_mapped0).any(axis=1)
log.info(
'Videos have objects missing, though they are implied by mapping: '
'[Total: {}] [Train: {}] [Val: {}]'.format(
mean_frac(X_impobj_missing),
mean_frac(X_impobj_missing[train_mask]),
mean_frac(X_impobj_missing[val_mask])))
def produce_map_annot(obj_presence_mapped0, obj_presence0):
X = (obj_presence_mapped0 & obj_presence0).sum()
XX = obj_presence_mapped0.sum()
present_when_mapped = pd.concat((X / XX * 100, X, XX), axis=1)
present_when_mapped.index = dataset.object_names[1:]
present_when_mapped.columns = ['%', 'matched', 'total_mapped']
return present_when_mapped
X = produce_map_annot(obj_presence_mapped0, obj_presence0)
X_train = produce_map_annot(obj_presence_mapped0[train_mask],
obj_presence0[train_mask])
X_val = produce_map_annot(obj_presence_mapped0[val_mask],
obj_presence0[val_mask])
log.info('Videos per cls, where mapped object is annotated:\n{}'.format(
pd.concat((X, X_train, X_val), keys=['total', 'train', 'val'],
axis=1)))
# Per-instance stats
instances_ = {}
for vid, video in dataset.videos_ocv.items():
for ii, instance in enumerate(video['actions']):
instance_ = copy.copy(instance)
instance_['split'] = dataset.split[vid]
instances_[(vid, ii)] = instance_
pd_inst = pd.DataFrame(instances_).T
pd_inst['nframes'] = pd_inst['end_frame'] - pd_inst['start_frame'] + 1
pd_inst['action_ind'] = pd_inst.name.apply(
lambda x: dataset.action_names.index(x))
train_mask_i = pd_inst['split'] == 'train'
val_mask_i = pd_inst['split'] == 'val'
nframes_per_cls = pd_inst.groupby('action_ind').nframes.sum()
log.info(f'N frames per cls, summed (total): {nframes_per_cls.sum()}')
nframes_per_cls_train = (
pd_inst[train_mask_i].groupby('action_ind').nframes.sum())
nframes_per_cls_val = (
pd_inst[val_mask_i].groupby('action_ind').nframes.sum())
X = pd.concat(
(nframes_per_cls, nframes_per_cls_train, nframes_per_cls_val),
axis=1,
keys=['total', 'train', 'val'])
log.info('Describe frames per cls:\n{}'.format(X.describe()))
# Frame coverage
fcoverage = {
vid: np.zeros(row['nframes'], dtype=np.int)
for vid, row in pd_vs.iterrows()
}
for (vid, i), row in pd_inst.iterrows():
s = row['start_frame']
e = row['end_frame']
fcoverage[vid][s:e + 1] += 1
coverage_per_vid = {}
for vid, cov in fcoverage.items():
d = {
'nframes': len(cov),
'covered': (cov > 0).sum(),
'annos': cov.sum()
}
coverage_per_vid[vid] = d
df_coverage = pd.DataFrame(coverage_per_vid).T
df_coverage['frac'] = df_coverage['covered'] / df_coverage['nframes']
def anno_frac(X):
return frac_str(X.covered.sum(), X.nframes.sum())
def avg_anno_vf_frac(X):
return frac_str(X.annos.sum(), X.nframes.sum())
def avg_anno_af_frac(X):
return frac_str(X.annos.sum(), X.covered.sum())
log.info('Annotated fraction (per-vid): '
'[Total: {:.2f}%] [Train: {:.2f}%] [Val: {:.2f}%]'.format(
df_coverage.frac.mean() * 100,
df_coverage[train_mask].frac.mean() * 100,
df_coverage[val_mask].frac.mean() * 100))
log.info('Annotated fraction: '
'[Total: {}] [Train: {}] [Val: {}]'.format(
anno_frac(df_coverage), anno_frac(df_coverage[train_mask]),
anno_frac(df_coverage[val_mask])))
log.info('Avg. N annotation per video frame: '
'[Total: {}] [Train: {}] [Val: {}]'.format(
avg_anno_vf_frac(df_coverage),
avg_anno_vf_frac(df_coverage[train_mask]),
avg_anno_vf_frac(df_coverage[val_mask])))
log.info('Avg. N annotation per annotated video frame: '
'[Total: {}] [Train: {}] [Val: {}]'.format(
avg_anno_af_frac(df_coverage),
avg_anno_af_frac(df_coverage[train_mask]),
avg_anno_af_frac(df_coverage[val_mask])))
def merge_scores_avstubes(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(raise_without_defaults=False)
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tube_eval.set_defcfg(cfg)
cfg.set_defaults("""
tube_dict: ~
combinations:
enabled: False
sizes: ~
""")
cf = cfg.parse()
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
ts = {k: small.load_pkl(v) for k, v in cfg_dict['tube_dict'].items()}
if not cf['combinations.enabled']:
av_stubes = _meanpool_avstubes(list(ts.values()))
small.save_pkl(out / 'merged_av_stubes.pkl', av_stubes)
log.info('All combined score:')
Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
return
sizes = cf['combinations.sizes']
combinations = [list(itertools.combinations(ts.keys(), r)) for r in sizes]
combinations = list(itertools.chain(*combinations))
log.info('Combinations: {}'.format(combinations))
comb_dfdicts = {}
for comb in combinations:
comb_name = '+'.join(comb)
comb_fold = small.mkdir(out / comb_name)
def compute():
to_merge = [ts[k] for k in comb]
av_stubes = _meanpool_avstubes(to_merge)
small.save_pkl(comb_fold / 'av_stubes.pkl', av_stubes)
dfdict = Ncfg_tube_eval.eval_as_df(cf, av_stubes, av_gt_tubes)
return dfdict
dfdict = small.stash2(comb_fold / 'stashed_dfdict.pkl')(compute)
comb_dfdicts[comb_name] = dfdict
log.info('Individual results:')
for comb_name, dfdict in comb_dfdicts.items():
log.info(f'Results for {comb_name}:')
_print_quick_evaluation_stats(dfdict)
log.info('Combined tables:')
big_ = {comb: pd.concat(dfdict) for comb, dfdict in comb_dfdicts.items()}
big = pd.concat(big_, axis=1)
for stat in big.index.levels[0]:
log.info(f'=== {stat} ===')
for thresh in big.columns.levels[1]:
X = (big.loc['ap'].loc[:, pd.IndexSlice[:,
thresh]].droplevel(1,
axis=1))
table = snippets.df_to_table_v2((X * 100).round(2))
log.info(f'{stat} for IOU {thresh}:\n{table}')
log.info('\n')
