def data_partition(cfg, imdbidlist_json, annos_dict):
assert (cfg.seq_len % 2 == 0)
seq_len_half = cfg.seq_len // 2
idxs = []
for mode in ['train', 'test', 'val']:
one_mode_idxs = []
for imdbid in imdbidlist_json[mode]:
anno_dict = annos_dict[imdbid]
shotid_list = sorted(anno_dict.keys())
shotid_tmp = 0
for shotid in shotid_list:
if int(shotid) < shotid_tmp + seq_len_half:
continue
shotid_tmp = int(shotid) + seq_len_half
one_idxs = []
for idx in range(-seq_len_half + 1, seq_len_half + 1):
one_idxs.append({
'imdbid': imdbid,
'shotid': strcal(shotid, idx)
})
one_mode_idxs.append(one_idxs)
idxs.append(one_mode_idxs)
partition = {}
partition['train'] = idxs[0]
partition['test'] = idxs[1]
partition['val'] = idxs[2]
return partition
def data_partition(cfg, valid_shotids):
assert (cfg.seq_len % 2 == 0)
seq_len_half = cfg.seq_len // 2
idxs = []
one_mode_idxs = []
shotid_tmp = 0
for shotid in valid_shotids:
if int(shotid) < shotid_tmp + seq_len_half:
continue
shotid_tmp = int(shotid) + seq_len_half
one_idxs = []
for idx in range(-seq_len_half + 1, seq_len_half + 1):
one_idxs.append({
'imdbid': cfg.video_name,
'shotid': strcal(shotid, idx)
})
one_mode_idxs.append(one_idxs)
idxs.append(one_mode_idxs)
partition = {}
partition['train'] = idxs[0]
partition['test'] = idxs[0]
partition['val'] = idxs[0]
return partition
def _get_single_item(self, ID):
imdbid = ID['imdbid']
shotid = ID['shotid']
label = self.data_dict["annos_dict"].get(imdbid).get(shotid)
aud_feats, place_feats = [], []
cast_feats, act_feats = [], []
if 'place' in self.mode:
for ind in self.shot_boundary_range:
name = 'shot_{}.npy'.format(strcal(shotid, ind))
path = osp.join(self.data_root, 'place_feat/{}'.format(imdbid),
name)
place_feat = np.load(path)
place_feats.append(torch.from_numpy(place_feat).float())
if 'cast' in self.mode:
for ind in self.shot_boundary_range:
cast_feat_raw = self.data_dict["casts_dict"].get(imdbid).get(
strcal(shotid, ind))
cast_feat = np.mean(cast_feat_raw, axis=0)
cast_feats.append(torch.from_numpy(cast_feat).float())
if 'act' in self.mode:
for ind in self.shot_boundary_range:
act_feat = self.data_dict["acts_dict"].get(imdbid).get(
strcal(shotid, ind))
act_feats.append(torch.from_numpy(act_feat).float())
if 'aud' in self.mode:
for ind in self.shot_boundary_range:
name = 'shot_{}.npy'.format(strcal(shotid, ind))
path = osp.join(self.data_root, 'aud_feat/{}'.format(imdbid),
name)
aud_feat = np.load(path)
aud_feats.append(torch.from_numpy(aud_feat).float())
if len(place_feats) > 0:
place_feats = torch.stack(place_feats)
if len(cast_feats) > 0:
cast_feats = torch.stack(cast_feats)
if len(act_feats) > 0:
act_feats = torch.stack(act_feats)
if len(aud_feats) > 0:
aud_feats = torch.stack(aud_feats)
return place_feats, cast_feats, act_feats, aud_feats, label
def cal_Recall_time(cfg, recall_time=3, threshold=0.3):
metric_dict = {
"pred_path":
osp.join(cfg.logger.logs_dir,
'pred/pred_{:.2f}.txt'.format(threshold)),
"shot_path":
cfg.shot_frm_path,
"recall_duration":
5,
"recall_time":
recall_time
}
result_dict = get_result_dict(metric_dict)
recall = []
for imdbid, result_dict_one in result_dict.items():
shot_fn = "{}/{}.txt".format(metric_dict.get('shot_path'), imdbid)
with open(shot_fn, "r") as f:
shot_list = f.read().splitlines()
cont_one, total_one = 0, 0
for shotid, item in result_dict_one.items():
gt = item.get('gt')
shot_time = int(shot_list[int(shotid)].split(' ')[1])
if gt != '1':
continue
total_one += 1
for ind in range(0 - metric_dict.get('recall_duration'),
1 + metric_dict.get('recall_duration')):
shotid_cp = strcal(shotid, ind)
if int(shotid_cp) < 0 or int(shotid_cp) >= len(shot_list):
continue
shot_time_cp = int(shot_list[int(shotid_cp)].split(' ')[1])
item_cp = result_dict_one.get(shotid_cp)
if item_cp is None:
continue
else:
pred = item_cp.get('pred')
gap_time = np.abs(shot_time_cp - shot_time) / 24
if gt == pred and gap_time < metric_dict.get(
'recall_time'):
cont_one += 1
break
recall_one = cont_one / (total_one + 1e-5)
recall.append(recall_one)
print('Recall_at_{}: '.format(metric_dict.get('recall_time')),
np.mean(recall))
return np.mean(recall)
def _get_single_item(self, ID):
shotid = ID["shotid"]
imgs = []
label = 1 ## this is a pesudo label
if 'image' in self.mode:
for ind in self.shot_boundary_range:
name = 'shot_{}_img_1.jpg'.format(strcal(shotid,ind))
path = osp.join(self.cfg.data_root, 'shot_keyf', self.cfg.video_name, name)
img = Image.open(path).convert('RGB')
if self.transform:
img = self.transform(img)
imgs.append(img)
if len(imgs)>0:
imgs = torch.stack(imgs)
return imgs, label
def _get_single_item(self, ID):
imdbid = ID['imdbid']
shotid = ID['shotid']
label = self.data_dict["annos_dict"].get(imdbid).get(shotid)
imgs = []
if 'image' in self.mode:
for ind in self.shot_boundary_range:
name = 'shot_{}.jpg'.format(strcal(shotid, ind))
path = osp.join(self.data_root, 'keyf_240p/{}'.format(imdbid),
name)
img = Image.open(path).convert('RGB')
if self.transform:
img = self.transform(img)
imgs.append(img)
if len(imgs) > 0:
imgs = torch.stack(imgs)
return imgs, label