def predict(dataset, param):
param['phase'] = 'test'
data_generator = DataGenerator(dataset, param)
# load model
with h5py.File(os.path.join(get_model_path(), param['model_dump_file']),
'r') as fin:
w = fin['/dense_1/dense_1/kernel:0'][:]
b = fin['/dense_1/dense_1/bias:0'][:]
print('predicting short-term visual relation...')
pbar = tqdm(total=len(data_generator.index))
short_term_relations = defaultdict(list)
# do not support prefetching mode in test phase
data = data_generator.get_data()
while data:
# get all possible pairs and the respective features and annos
index, pairs, feats, iou, trackid = data
vid, fstart, fend = index
# load predicted object tracklets
tracklets = object_trajectory_proposal(dataset, vid, fstart, fend)
for trac in tracklets:
trac.pstart = fstart
trac.pend = fend
trac.vsig = get_segment_signature(vid, fstart, fend)
# make prediction
p = feats.dot(w) + b
s = feats[:, :35]
o = feats[:, 35:70]
predictions = []
for i in range(len(pairs)):
top_s_ind = np.argsort(s[i])[-param['pair_topk']:]
top_p_ind = np.argsort(p[i])[-param['pair_topk']:]
top_o_ind = np.argsort(o[i])[-param['pair_topk']:]
score = s[i][top_s_ind, None, None] * p[i][None, top_p_ind,
None] * o[i][None, None,
top_o_ind]
top_flat_ind = np.argsort(score, axis=None)[-param['pair_topk']:]
top_score = score.ravel()[top_flat_ind]
top_s, top_p, top_o = np.unravel_index(top_flat_ind, score.shape)
predictions.extend(
(top_score[j], (top_s_ind[top_s[j]], top_p_ind[top_p[j]],
top_o_ind[top_o[j]]), pairs[i])
for j in range(top_score.size))
predictions = sorted(predictions, key=lambda x: x[0],
reverse=True)[:param['seg_topk']]
for x in predictions:
sub = dataset.get_object_name(x[1][0])
pred = dataset.get_predicate_name(x[1][1])
obj = dataset.get_object_name(x[1][2])
r = VideoRelation(sub, pred, obj, tracklets[x[2][0]],
tracklets[x[2][1]], x[0])
short_term_relations[vid].append(r.serialize())
data = data_generator.get_data()
pbar.update(1)
pbar.close()
return short_term_relations
def load_object_trajectory_proposal():
"""
Test loading precomputed object trajectory proposals
"""
dataset = VidVRD('../vidvrd-dataset', '../vidvrd-dataset/videos',
['train', 'test'])
video_indices = dataset.get_index(split='train')
for vid in video_indices:
durations = set(
rel_inst['duration']
for rel_inst in dataset.get_relation_insts(vid, no_traj=True))
for duration in durations:
segs = segment_video(*duration)
for fstart, fend in segs:
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=False,
verbose=True)
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=True,
verbose=True)
video_indices = dataset.get_index(split='test')
for vid in video_indices:
anno = dataset.get_anno(vid)
segs = segment_video(0, anno['frame_count'])
for fstart, fend in segs:
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=False,
verbose=True)
trajs = trajectory.object_trajectory_proposal(dataset,
vid,
fstart,
fend,
gt=True,
verbose=True)