def get_training_data(self, index): video = self.video_list[index] props = self._video_centric_sampling(video) out_prop_ind = [] out_prop_type = [] out_prop_labels = [] out_prop_reg_targets = [] for idx, p in enumerate(props): prop_indices, prop_label, reg_targets, prop_type = self._load_prop_data( p) out_prop_ind.append(prop_indices) out_prop_labels.append(prop_label) out_prop_reg_targets.append(reg_targets) out_prop_type.append(prop_type) out_prop_labels = torch.from_numpy(np.array(out_prop_labels)) out_prop_reg_targets = torch.from_numpy( np.array(out_prop_reg_targets, dtype=np.float32)) out_prop_type = torch.from_numpy(np.array(out_prop_type)) #load prop fts vid_full_name = video.id vid = vid_full_name.split('/')[-1] act_prop_ft, comp_prop_ft = I3D_Pooling(out_prop_ind, vid, self.ft_path, video.num_frames) return ( act_prop_ft, comp_prop_ft), out_prop_type, out_prop_labels, out_prop_reg_targets
def runner_func(dataset, state_dict, stats, gpu_id, index_queue, result_queue, iou_dict, dis_dict): torch.cuda.set_device(gpu_id) net = PGCN(model_configs, graph_configs, dataset_configs, test_mode=True) net.load_state_dict(state_dict) # net.prepare_test_fc() net.eval() net.cuda() while True: index = index_queue.get() rel_props, prop_ticks, video_id, n_frames = dataset[index] # calculate scores # n_out = prop_ticks.size(0) # act_scores = torch.zeros((n_out, num_class + 1)).cuda() # comp_scores = torch.zeros((n_out, num_class)).cuda() # if not args.no_regression: # reg_scores = torch.zeros((n_out, num_class * 2)).cuda() # else: # reg_scores = None # load prop fts vid_full_name = video_id vid = vid_full_name.split('/')[-1] act_all_fts, comp_all_fts = I3D_Pooling( prop_ticks, vid, dataset_configs['test_ft_path'], n_frames) with torch.no_grad(): vid_iou_dict = torch.from_numpy(iou_dict[vid]).cuda() vid_dis_dict = torch.from_numpy(dis_dict[vid]).cuda() act_scores, comp_scores, reg_scores = net(act_all_fts.cuda(), comp_all_fts.cuda(), iou_dict=vid_iou_dict, dis_dict=vid_dis_dict) if reg_scores is not None: reg_scores = reg_scores.view(-1, num_class, 2) reg_scores[:, :, 0] = reg_scores[:, :, 0] * stats[1, 0] + stats[0, 0] reg_scores[:, :, 1] = reg_scores[:, :, 1] * stats[1, 1] + stats[0, 1] result_queue.put((dataset.video_list[index].id, rel_props.numpy(), act_scores.squeeze().cpu().numpy(), comp_scores.squeeze().cpu().numpy(), reg_scores.cpu().numpy(), 0))
def get_training_data(self, index,r_index): video = self.video_list[index] props = self._video_centric_sampling(video) out_prop_ind = [] out_prop_type = [] out_prop_labels = [] out_prop_reg_targets = [] for idx, p in enumerate(props): prop_indices, prop_label, reg_targets, prop_type = self._load_prop_data(p) out_prop_ind.append(prop_indices) out_prop_labels.append(prop_label) out_prop_reg_targets.append(reg_targets) out_prop_type.append(prop_type) out_prop_labels = torch.from_numpy(np.array(out_prop_labels)) out_prop_reg_targets = torch.from_numpy(np.array(out_prop_reg_targets, dtype=np.float32)) out_prop_type = torch.from_numpy(np.array(out_prop_type)) #load prop fts vid_full_name = video.id vid = vid_full_name.split('/')[-1] # act_prop_ft, comp_prop_ft, out_prop_type,out_prop_labels,out_prop_reg_targets= My_Pooling(out_prop_ind, vid, self.ft_path, video.num_frames,index=r_index,val_mode=self.val_mode) # act_prop_ft, comp_prop_ft= My_Pooling_full(out_prop_ind,vid,index=r_index,val_mode=self.val_mode) # print(act_prop_ft.size()) act_prop_ft, comp_prop_ft ,comp_prop_len= I3D_Pooling(out_prop_ind, vid, self.ft_path, video.num_frames) # print(act_prop_ft) # print(comp_prop_ft) # print('%%%%%%%%%%%%%%%%\n') # print(out_prop_ind[12],index) # exit() return (act_prop_ft, comp_prop_ft,comp_prop_len), out_prop_type, out_prop_labels, out_prop_reg_targets,out_prop_ind,vid,r_index
def runner_func(dataset, state_dict, stats, gpu_id, index_queue, result_queue, iou_dict, dis_dict): torch.cuda.set_device(gpu_id) net = PGCN(model_configs, graph_configs, test_mode=True) net.load_state_dict(state_dict) # net.prepare_test_fc() net.eval() net.cuda() while True: index = index_queue.get() rel_props, prop_ticks, video_id, n_frames = dataset[index] # calculate scores n_out = prop_ticks.size(0) act_scores = torch.zeros((n_out, num_class + 1)).cuda() comp_scores = torch.zeros((n_out, num_class)).cuda() if not args.no_regression: reg_scores = torch.zeros((n_out, num_class * 2)).cuda() else: reg_scores = None # load prop fts vid_full_name = video_id vid = vid_full_name.split('/')[-1] act_all_fts, comp_all_fts = I3D_Pooling( prop_ticks, vid, dataset_configs['test_ft_path'], n_frames) for prop_idx, prop in enumerate(prop_ticks): if prop_idx >= n_out: break with torch.no_grad(): vid_iou_dict = iou_dict[vid] vid_dis_dict = dis_dict[vid] # print(len(vid_iou_dict), len(vid_dis_dict), len(prop_ticks)) selected_idx = get_adjacent_batch(prop_idx, vid_iou_dict, vid_dis_dict) selected_idx = torch.from_numpy(np.array(selected_idx)) act_ft = act_all_fts[selected_idx, :] comp_ft = comp_all_fts[selected_idx, :] act_batch_var = act_ft.unsqueeze(0).cuda() comp_batch_var = comp_ft.unsqueeze(0).cuda() act_scores[prop_idx, :], comp_scores[prop_idx, :], \ reg_scores[prop_idx, :] = net((act_batch_var, comp_batch_var), None, None, None) if reg_scores is not None: reg_scores = reg_scores.view(-1, num_class, 2) reg_scores[:, :, 0] = reg_scores[:, :, 0] * stats[1, 0] + stats[0, 0] reg_scores[:, :, 1] = reg_scores[:, :, 1] * stats[1, 1] + stats[0, 1] # perform stpp on scores result_queue.put((dataset.video_list[index].id, rel_props.numpy(), act_scores.cpu().numpy(), comp_scores.cpu().numpy(), reg_scores.cpu().numpy(), 0))