def get_training_data(self, index):
video = self.video_list[index]
props = self._video_centric_sampling(video)
out_prop_ind = []
out_prop_type = []
out_prop_labels = []
out_prop_reg_targets = []
for idx, p in enumerate(props):
prop_indices, prop_label, reg_targets, prop_type = self._load_prop_data(
p)
out_prop_ind.append(prop_indices)
out_prop_labels.append(prop_label)
out_prop_reg_targets.append(reg_targets)
out_prop_type.append(prop_type)
out_prop_labels = torch.from_numpy(np.array(out_prop_labels))
out_prop_reg_targets = torch.from_numpy(
np.array(out_prop_reg_targets, dtype=np.float32))
out_prop_type = torch.from_numpy(np.array(out_prop_type))
#load prop fts
vid_full_name = video.id
vid = vid_full_name.split('/')[-1]
act_prop_ft, comp_prop_ft = I3D_Pooling(out_prop_ind, vid,
self.ft_path, video.num_frames)
return (
act_prop_ft,
comp_prop_ft), out_prop_type, out_prop_labels, out_prop_reg_targets
def runner_func(dataset, state_dict, stats, gpu_id, index_queue, result_queue,
iou_dict, dis_dict):
torch.cuda.set_device(gpu_id)
net = PGCN(model_configs, graph_configs, dataset_configs, test_mode=True)
net.load_state_dict(state_dict)
# net.prepare_test_fc()
net.eval()
net.cuda()
while True:
index = index_queue.get()
rel_props, prop_ticks, video_id, n_frames = dataset[index]
# calculate scores
# n_out = prop_ticks.size(0)
# act_scores = torch.zeros((n_out, num_class + 1)).cuda()
# comp_scores = torch.zeros((n_out, num_class)).cuda()
# if not args.no_regression:
# reg_scores = torch.zeros((n_out, num_class * 2)).cuda()
# else:
# reg_scores = None
# load prop fts
vid_full_name = video_id
vid = vid_full_name.split('/')[-1]
act_all_fts, comp_all_fts = I3D_Pooling(
prop_ticks, vid, dataset_configs['test_ft_path'], n_frames)
with torch.no_grad():
vid_iou_dict = torch.from_numpy(iou_dict[vid]).cuda()
vid_dis_dict = torch.from_numpy(dis_dict[vid]).cuda()
act_scores, comp_scores, reg_scores = net(act_all_fts.cuda(),
comp_all_fts.cuda(),
iou_dict=vid_iou_dict,
dis_dict=vid_dis_dict)
if reg_scores is not None:
reg_scores = reg_scores.view(-1, num_class, 2)
reg_scores[:, :,
0] = reg_scores[:, :, 0] * stats[1, 0] + stats[0, 0]
reg_scores[:, :,
1] = reg_scores[:, :, 1] * stats[1, 1] + stats[0, 1]
result_queue.put((dataset.video_list[index].id, rel_props.numpy(),
act_scores.squeeze().cpu().numpy(),
comp_scores.squeeze().cpu().numpy(),
reg_scores.cpu().numpy(), 0))
def get_training_data(self, index,r_index):
video = self.video_list[index]
props = self._video_centric_sampling(video)
out_prop_ind = []
out_prop_type = []
out_prop_labels = []
out_prop_reg_targets = []
for idx, p in enumerate(props):
prop_indices, prop_label, reg_targets, prop_type = self._load_prop_data(p)
out_prop_ind.append(prop_indices)
out_prop_labels.append(prop_label)
out_prop_reg_targets.append(reg_targets)
out_prop_type.append(prop_type)
out_prop_labels = torch.from_numpy(np.array(out_prop_labels))
out_prop_reg_targets = torch.from_numpy(np.array(out_prop_reg_targets, dtype=np.float32))
out_prop_type = torch.from_numpy(np.array(out_prop_type))
#load prop fts
vid_full_name = video.id
vid = vid_full_name.split('/')[-1]
# act_prop_ft, comp_prop_ft, out_prop_type,out_prop_labels,out_prop_reg_targets= My_Pooling(out_prop_ind, vid, self.ft_path, video.num_frames,index=r_index,val_mode=self.val_mode)
# act_prop_ft, comp_prop_ft= My_Pooling_full(out_prop_ind,vid,index=r_index,val_mode=self.val_mode)
# print(act_prop_ft.size())
act_prop_ft, comp_prop_ft ,comp_prop_len= I3D_Pooling(out_prop_ind, vid, self.ft_path, video.num_frames)
# print(act_prop_ft)
# print(comp_prop_ft)
# print('%%%%%%%%%%%%%%%%\n')
# print(out_prop_ind[12],index)
# exit()
return (act_prop_ft, comp_prop_ft,comp_prop_len), out_prop_type, out_prop_labels, out_prop_reg_targets,out_prop_ind,vid,r_index
def runner_func(dataset, state_dict, stats, gpu_id, index_queue, result_queue,
iou_dict, dis_dict):
torch.cuda.set_device(gpu_id)
net = PGCN(model_configs, graph_configs, test_mode=True)
net.load_state_dict(state_dict)
# net.prepare_test_fc()
net.eval()
net.cuda()
while True:
index = index_queue.get()
rel_props, prop_ticks, video_id, n_frames = dataset[index]
# calculate scores
n_out = prop_ticks.size(0)
act_scores = torch.zeros((n_out, num_class + 1)).cuda()
comp_scores = torch.zeros((n_out, num_class)).cuda()
if not args.no_regression:
reg_scores = torch.zeros((n_out, num_class * 2)).cuda()
else:
reg_scores = None
# load prop fts
vid_full_name = video_id
vid = vid_full_name.split('/')[-1]
act_all_fts, comp_all_fts = I3D_Pooling(
prop_ticks, vid, dataset_configs['test_ft_path'], n_frames)
for prop_idx, prop in enumerate(prop_ticks):
if prop_idx >= n_out:
break
with torch.no_grad():
vid_iou_dict = iou_dict[vid]
vid_dis_dict = dis_dict[vid]
# print(len(vid_iou_dict), len(vid_dis_dict), len(prop_ticks))
selected_idx = get_adjacent_batch(prop_idx, vid_iou_dict,
vid_dis_dict)
selected_idx = torch.from_numpy(np.array(selected_idx))
act_ft = act_all_fts[selected_idx, :]
comp_ft = comp_all_fts[selected_idx, :]
act_batch_var = act_ft.unsqueeze(0).cuda()
comp_batch_var = comp_ft.unsqueeze(0).cuda()
act_scores[prop_idx, :], comp_scores[prop_idx, :], \
reg_scores[prop_idx, :] = net((act_batch_var, comp_batch_var), None, None, None)
if reg_scores is not None:
reg_scores = reg_scores.view(-1, num_class, 2)
reg_scores[:, :,
0] = reg_scores[:, :, 0] * stats[1, 0] + stats[0, 0]
reg_scores[:, :,
1] = reg_scores[:, :, 1] * stats[1, 1] + stats[0, 1]
# perform stpp on scores
result_queue.put((dataset.video_list[index].id, rel_props.numpy(),
act_scores.cpu().numpy(), comp_scores.cpu().numpy(),
reg_scores.cpu().numpy(), 0))