def load_relation_feature():
"""
Test loading precomputed relation features
"""
dataset = VidVRD('../vidvrd-dataset', '../vidvrd-dataset/videos',
['train', 'test'])
extractor = feature.FeatureExtractor(dataset, prefetch_count=0)
video_indices = dataset.get_index(split='train')
for vid in video_indices:
durations = set(
rel_inst['duration']
for rel_inst in dataset.get_relation_insts(vid, no_traj=True))
for duration in durations:
segs = segment_video(*duration)
for fstart, fend in segs:
extractor.extract_feature(dataset,
vid,
fstart,
fend,
verbose=True)
video_indices = dataset.get_index(split='test')
for vid in video_indices:
anno = dataset.get_anno(vid)
segs = segment_video(0, anno['frame_count'])
for fstart, fend in segs:
extractor.extract_feature(dataset, vid, fstart, fend, verbose=True)
def load_object_trajectory_proposal():
"""
Test loading precomputed object trajectory proposals
"""
dataset = VidVRD('../vidvrd-dataset', '../vidvrd-dataset/videos',
['train', 'test'])
video_indices = dataset.get_index(split='train')
for vid in video_indices:
durations = set(
rel_inst['duration']
for rel_inst in dataset.get_relation_insts(vid, no_traj=True))
for duration in durations:
segs = segment_video(*duration)
for fstart, fend in segs:
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=False,
verbose=True)
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=True,
verbose=True)
video_indices = dataset.get_index(split='test')
for vid in video_indices:
anno = dataset.get_anno(vid)
segs = segment_video(0, anno['frame_count'])
for fstart, fend in segs:
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=False,
verbose=True)
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=True,
verbose=True)
prediction_out = 'test_out_{}.json'.format(config)
if os.path.exists(prediction_out):
print('Loading prediction from {}'.format(prediction_out))
with open(prediction_out, 'r') as fin:
result = json.load(fin)
else:
with open('test.json', 'r') as test_st_rela_f:
test_st_rela = json.load(test_st_rela_f)
result = origin_mht_relational_association(test_st_rela['results'])
with open(prediction_out, 'w+') as out_f:
result = {"results": {test_vid: result}}
out_f.write(json.dumps(result))
print('Number of videos in prediction: {}'.format(len(result['results'])))
evaluate_relation(dataset, 'test', result['results'])
compare_result(dataset.get_relation_insts(test_vid, no_traj=True),
result['results'][test_vid], config)
# prediction_out = 'test.json'
# with open(prediction_out, 'r') as gt_f:
# result = json.load(gt_f)
# result = {
# test_vid: result['results']
# }
# evaluate_relation(dataset, 'test', result)
def eval_short_term_relation():
"""
Evaluate short-term relation prediction
"""
anno_rpath = 'baseline/vidvrd-dataset'
video_rpath = 'baseline/vidvrd-dataset/videos'
splits = ['train', 'test']
st_prediction = 'baseline/vidvrd-dataset/vidvrd-baseline-output/short-term-predication.json'
test_st_pred = '/home/daivd/Downloads/pad_result_24000_test_predicate_-1_pair_nms_0.4_rpn_nms_0.7_0.255_union.json'
res_path = test_st_pred
dataset = VidVRD(anno_rpath=anno_rpath,
video_rpath=video_rpath,
splits=splits)
with open(os.path.join(get_model_path(), 'baseline_setting.json'), 'r') as fin:
param = json.load(fin)
if os.path.exists(res_path):
with open(res_path, 'r') as fin:
short_term_relations = json.load(fin)
else:
short_term_relations = model.predict(dataset, param)
with open(res_path, 'w') as fout:
json.dump(short_term_relations, fout)
short_term_gt = dict()
short_term_pred = dict()
video_indices = dataset.get_index(split='test')
for vid in video_indices:
anno = dataset.get_anno(vid)
segs = segment_video(0, anno['frame_count'])
video_gts = dataset.get_relation_insts(vid)
if 'results' in short_term_relations.keys():
video_preds = short_term_relations['results'][vid]
else:
video_preds = short_term_relations[vid]
for fstart, fend in segs:
vsig = get_segment_signature(vid, fstart, fend)
segment_gts = []
for r in video_gts:
s = max(r['duration'][0], fstart)
e = min(r['duration'][1], fend)
if s < e:
sub_trac = r['sub_traj'][s - r['duration'][0]: e - r['duration'][0]]
obj_trac = r['obj_traj'][s - r['duration'][0]: e - r['duration'][0]]
segment_gts.append({
"triplet": r['triplet'],
"subject_tid": r['subject_tid'],
"object_tid": r['object_tid'],
"duration": [s, e],
"sub_traj": sub_trac,
"obj_traj": obj_trac
})
short_term_gt[vsig] = segment_gts
segment_preds = []
for r in video_preds:
if fstart <= r['duration'][0] and r['duration'][1] <= fend:
s = max(r['duration'][0], fstart)
e = min(r['duration'][1], fend)
sub_trac = r['sub_traj'][s - r['duration'][0]: e - r['duration'][0]]
obj_trac = r['obj_traj'][s - r['duration'][0]: e - r['duration'][0]]
segment_preds.append({
"triplet": r['triplet'],
"score": r['score'],
"duration": [s, e],
"sub_traj": sub_trac,
"obj_traj": obj_trac
})
short_term_pred[vsig] = segment_preds
for each_vsig in short_term_gt.keys():
if each_vsig not in short_term_pred.keys():
short_term_pred[each_vsig] = []
mean_ap, rec_at_n, mprec_at_n = eval_visual_relation(short_term_gt, short_term_pred)
print('detection mean AP (used in challenge): {}'.format(mean_ap))
print('detection recall@50: {}'.format(rec_at_n[50]))
print('detection recall@100: {}'.format(rec_at_n[100]))
print('tagging precision@1: {}'.format(mprec_at_n[1]))
print('tagging precision@5: {}'.format(mprec_at_n[5]))
print('tagging precision@10: {}'.format(mprec_at_n[10]))
splits = ['train', 'test']
prediction = 'baseline/vidvrd-dataset/vidvrd-baseline-output/models/baseline_relation_prediction.json'
st_prediction = 'baseline/vidvrd-dataset/vidvrd-baseline-output/short-term-predication.json'
dataset = VidVRD(anno_rpath=anno_rpath, video_rpath=video_rpath, splits=splits)
video_indices = dataset.get_index(split='test')
with open(st_prediction, 'r') as st_pre_f:
pred_segs = json.load(st_pre_f)
short_term_gt = dict()
short_term_pred = dict()
for vid in video_indices:
gt = dataset.get_relation_insts(vid)
pred = pred_segs[vid]
gt_segs = separate_vid_2_seg(gt)
for each_gt_seg in gt_segs:
if len(each_gt_seg) < 1:
continue
fstart, fend = each_gt_seg[0]['duration']
vsig = get_segment_signature(vid, fstart, fend)
short_term_gt[vsig] = each_gt_seg
for each_pred_seg in pred:
if each_pred_seg['duration'] == each_gt_seg[0]['duration']:
if vsig in short_term_pred.keys():
short_term_pred[vsig].append(each_pred_seg)
else: