def BSN_inference_TEM(opt): model = TEM(opt) checkpoint = torch.load(opt["checkpoint_path"]+"/tem_best.pth.tar") base_dict = {'.'.join(k.split('.')[1:]): v for k,v in list(checkpoint['state_dict'].items())} model.load_state_dict(base_dict) model = torch.nn.DataParallel(model, device_ids=[0]).cuda() model.eval() test_loader = torch.utils.data.DataLoader(VideoDataSet(opt,subset="full"), batch_size=model.module.batch_size, shuffle=False, num_workers=8, pin_memory=True,drop_last=False) columns=["action","start","end","xmin","xmax"] for index_list,input_data,anchor_xmin,anchor_xmax in test_loader: TEM_output = model(input_data).detach().cpu().numpy() batch_action = TEM_output[:,0,:] batch_start = TEM_output[:,1,:] batch_end = TEM_output[:,2,:] index_list = index_list.numpy() anchor_xmin = np.array([x.numpy()[0] for x in anchor_xmin]) anchor_xmax = np.array([x.numpy()[0] for x in anchor_xmax]) for batch_idx,full_idx in enumerate(index_list): video = test_loader.dataset.video_list[full_idx] video_action = batch_action[batch_idx] video_start = batch_start[batch_idx] video_end = batch_end[batch_idx] video_result = np.stack((video_action,video_start,video_end,anchor_xmin,anchor_xmax),axis=1) video_df = pd.DataFrame(video_result,columns=columns) video_df.to_csv("./output/TEM_results/"+video+".csv",index=False)
def BSN_inference_TEM(opt): model = TEM(opt) checkpoint = torch.load(opt["checkpoint_path"]+"/"+opt["arch"]+"_tem_best.pth.tar") base_dict = {'.'.join(k.split('.')[1:]): v for k,v in list(checkpoint['state_dict'].items())} model.load_state_dict(base_dict) model = torch.nn.DataParallel(model, device_ids=GPU_IDs).cuda() model.eval() test_loader = torch.utils.data.DataLoader(VideoDataSet(opt,subset="full"), batch_size=model.module.batch_size, shuffle=False, num_workers=8, pin_memory=True,drop_last=False) # test_loader = torch.utils.data.DataLoader(VideoDataSet(opt,subset="trainval"), # batch_size=model.module.batch_size, shuffle=False, # num_workers=8, pin_memory=True,drop_last=False) columns=["action","start","end","xmin","xmax"] count = 0 for index_list,input_data,anchor_xmin,anchor_xmax in test_loader: #for video with different length # if opt['fix_scale'] is False: if opt['fix_scale'] == 'nonrescale': if len(anchor_xmin) != input_data.shape[2]: temporal_scale = input_data.shape[2] temporal_gap = 1. / temporal_scale anchor_xmin=[temporal_gap*i for i in range(temporal_scale)] anchor_xmin = [torch.tensor([x]) for x in anchor_xmin] anchor_xmax=[temporal_gap*i for i in range(1,temporal_scale+1)] anchor_xmax = [torch.tensor([x]) for x in anchor_xmax] ############################################################# TEM_output = model(input_data).detach().cpu().numpy() batch_action = TEM_output[:,0,:] batch_start = TEM_output[:,1,:] batch_end = TEM_output[:,2,:] index_list = index_list.numpy() anchor_xmin = np.array([x.numpy()[0] for x in anchor_xmin]) anchor_xmax = np.array([x.numpy()[0] for x in anchor_xmax]) for batch_idx,full_idx in enumerate(index_list): video = test_loader.dataset.video_list[full_idx] video_action = batch_action[batch_idx] video_start = batch_start[batch_idx] video_end = batch_end[batch_idx] video_result = np.stack((video_action,video_start,video_end,anchor_xmin,anchor_xmax),axis=1) video_df = pd.DataFrame(video_result,columns=columns) video_df.to_csv("./output/"+opt["arch"]+opt["fix_scale"]+"_TEM_results/"+video+".csv",index=False) count += 1 if count % 100 == 0: print('finish', count) sys.stdout.flush()
def BSN_inference_TEM(opt): ''' Inference of TEM step - 1. load the best_model step - 2. the output of TEM is three pdf-curve for each scaled-video ''' # step - 1 model = TEM(opt) checkpoint = torch.load(opt["checkpoint_path"]+"/tem_best.pth.tar") base_dict = {'.'.join(k.split('.')[1:]): v for k,v in list(checkpoint['state_dict'].items())} model.load_state_dict(base_dict) model = torch.nn.DataParallel(model, device_ids=[0]).cuda() model.eval() # step - 2 # set subset = 'full' to generate the pdf of all video test_loader = torch.utils.data.DataLoader(VideoDataSet(opt,subset="full"), batch_size=model.module.batch_size, shuffle=False, num_workers=8, pin_memory=True,drop_last=False) columns = ['action', 'start', 'end', 'xmin', 'xmax'] for index_list, input_data, anchor_xmin, anchor_xmax in test_loader: TEM_output = model(input_data).detach().cpu().numpy() batch_action = TEM_output[:,0,:] batch_start = TEM_output[:,1,:] batch_end = TEM_output[:,2,:] index_list = index_list.numpy() anchor_xmin = np.array([x.numpy()[0] for x in anchor_xmin]) anchor_xmax = np.array([x.numpy()[0] for x in anchor_xmax]) for batch_idx, full_idx in enumerate(index_list): video_name = test_loader.dataset.video_list[full_idx] video_action = batch_action[batch_idx] video_start = batch_start[batch_idx] video_end = batch_end[batch_idx] video_result = np.stack((video_action, video_start, video_end, anchor_xmin, anchor_xmax),axis=1) video_df = pd.DataFrame(video_result, columns=columns) video_df.to_csv('./output/TEM_results/' + video_name + '.csv', index=False)
def BSN_Train_TEM(opt): writer = SummaryWriter() model = TEM(opt) model = torch.nn.DataParallel(model, device_ids=[0]).cuda() state_dict = torch.load('checkpoint/tem_best.pth.tar')['state_dict'] model.load_state_dict(state_dict) optimizer = optim.Adam(model.parameters(), lr=opt["tem_training_lr"], weight_decay=opt["tem_weight_decay"]) train_loader = torch.utils.data.DataLoader( VideoDataSet(opt, subset="train"), batch_size=model.module.batch_size, shuffle=True, num_workers=4, pin_memory=True, drop_last=True) test_loader = torch.utils.data.DataLoader( VideoDataSet(opt, subset="validation"), batch_size=model.module.batch_size, shuffle=False, num_workers=4, pin_memory=True, drop_last=True) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=opt["tem_step_size"], gamma=opt["tem_step_gamma"]) for epoch in range(opt["tem_epoch"]): train_TEM(train_loader, model, optimizer, epoch, writer, opt) scheduler.step() if (epoch + 1) % 3 == 0: test_TEM(test_loader, model, epoch, writer, opt) writer.close()
def BSN_inference_TEM(opt): output_dir = os.path.join(opt['tem_results_dir'], opt['checkpoint_path'].split('/')[-1]) print(sorted(opt.items()), flush=True) model = TEM(opt) checkpoint_epoch = opt['checkpoint_epoch'] if checkpoint_epoch is not None: checkpoint_path = os.path.join( opt['checkpoint_path'], 'tem_checkpoint.%d.pth' % checkpoint_epoch) output_dir = os.path.join(output_dir, 'ckpt.%d' % checkpoint_epoch) else: checkpoint_path = os.path.join(opt['checkpoint_path'], 'tem_best.pth') output_dir = os.path.join(output_dir, 'ckpt.best') if not os.path.exists(output_dir): os.makedirs(output_dir) print('Checkpoint path is ', checkpoint_path, flush=True) checkpoint = torch.load(checkpoint_path) base_dict = { '.'.join(k.split('.')[1:]): v for k, v in list(checkpoint['state_dict'].items()) } model.load_state_dict(base_dict) model = torch.nn.DataParallel(model).cuda() model.eval() if opt['dataset'] == 'gymnastics': img_loading_func = get_img_loader(opt) dataset = GymnasticsImages(opt, subset=opt['tem_results_subset'].title(), img_loading_func=img_loading_func, image_dir=opt['gym_image_dir'], video_info_path=os.path.join( opt['video_info'], 'Full_Annotation.csv')) elif opt['dataset'] == 'gymnasticsfeatures': # feature_dirs should roughly look like: # /checkpoint/cinjon/spaceofmotion/sep052019/tsn.1024.426x240.12.no-oversample/csv/rgb,/checkpoint/cinjon/spaceofmotion/sep052019/tsn.1024.426x240.12.no-oversample/csv/flow feature_dirs = opt['feature_dirs'].split(',') dataset = GymnasticsFeatures(opt, subset=opt['tem_results_subset'].title(), feature_dirs=feature_dirs, video_info_path=os.path.join( opt['video_info'], 'Full_Annotation.csv')) elif opt['dataset'] == 'thumosfeatures': feature_dirs = opt['feature_dirs'].split(',') dataset = ThumosFeatures(opt, subset=opt['tem_results_subset'].title(), feature_dirs=feature_dirs, video_info_path=os.path.join( opt['video_info'], 'Full_Annotation.csv')) elif opt['dataset'] == 'thumosimages': img_loading_func = get_img_loader(opt) dataset = ThumosImages( opt, subset=opt['tem_results_subset'].title(), img_loading_func=img_loading_func, image_dir='/checkpoint/cinjon/thumos/rawframes.TH14_%s_tal.30' % opt['tem_results_subset'], video_info_path=os.path.join(opt['video_info'], 'Full_Annotation.csv')) elif opt['dataset'] == 'activitynet': representation_module = opt['representation_module'] test_transforms = get_video_transforms(representation_module, False) dataset = VideoDataset(opt, test_transforms, subset='full', fraction=1.0) test_loader = torch.utils.data.DataLoader( dataset, batch_size=model.module.batch_size, shuffle=False, num_workers=opt['data_workers'], pin_memory=True, drop_last=False) columns = ["action", "start", "end", "frames"] all_vids = defaultdict(int) current_video = None current_start = defaultdict(float) current_end = defaultdict(float) current_action = defaultdict(float) calc_time_list = defaultdict(int) num_videoframes = opt['num_videoframes'] skip_videoframes = opt['skip_videoframes'] print('About to start enumerating', flush=True) for test_idx, (index_list, input_data, video_name, snippets) in enumerate(test_loader): if test_idx == 0: print('Started enumerating!', flush=True) # The data should be coming back s.t. consecutive data are from the same video. # until there is a breakpoint and it starts a new video. TEM_output = model(input_data).detach().cpu().numpy() batch_action = TEM_output[:, 0, :] batch_start = TEM_output[:, 1, :] batch_end = TEM_output[:, 2, :] index_list = index_list.numpy() for batch_idx, full_idx in enumerate(index_list): item_video = video_name[batch_idx] all_vids[item_video] += 1 item_snippets = snippets[batch_idx] if not current_video: print('First video: ', item_video, flush=True) current_video = item_video current_start = defaultdict(float) current_end = defaultdict(float) current_action = defaultdict(float) calc_time_list = defaultdict(int) elif item_video != current_video: print('Next video: ', item_video, full_idx, flush=True) column_frames = sorted(calc_time_list.keys()) column_action = [ current_action[k] * 1. / calc_time_list[k] for k in column_frames ] column_start = [ current_start[k] * 1. / calc_time_list[k] for k in column_frames ] column_end = [ current_end[k] * 1. / calc_time_list[k] for k in column_frames ] video_result = np.stack( [column_action, column_start, column_end], axis=1) column_frames = np.reshape(column_frames, [-1, 1]) video_result = np.concatenate([video_result, column_frames], axis=1) video_df = pd.DataFrame(video_result, columns=columns) path = os.path.join(output_dir, '%s.csv' % current_video) video_df.to_csv(path, index=False) current_video = item_video current_start = defaultdict(float) current_end = defaultdict(float) current_action = defaultdict(float) calc_time_list = defaultdict(int) for snippet_, action_, start_, end_ in zip(item_snippets, batch_action[batch_idx], batch_start[batch_idx], batch_end[batch_idx]): frame = snippet_.item() calc_time_list[frame] += 1 current_action[frame] += action_ current_start[frame] += start_ current_end[frame] += end_ if len(calc_time_list): column_frames = sorted(calc_time_list.keys()) column_action = [ current_action[k] * 1. / calc_time_list[k] for k in column_frames ] column_start = [ current_start[k] * 1. / calc_time_list[k] for k in column_frames ] column_end = [ current_end[k] * 1. / calc_time_list[k] for k in column_frames ] video_result = np.stack([column_action, column_start, column_end], axis=1) print(video_result.shape, flush=True) video_result = np.concatenate( [video_result, np.reshape(column_frames, [-1, 1])], axis=1) video_df = pd.DataFrame(video_result, columns=columns) path = os.path.join(output_dir, '%s.csv' % current_video) video_df.to_csv(path, index=False) print(len(all_vids))