def main(opt): dataset = VideoDataset(opt, "test") opt["vocab_size"] = dataset.get_vocab_size() opt["seq_length"] = dataset.max_len if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], rnn_dropout_p=opt["rnn_dropout_p"]).cuda() elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, decoder).cuda() model = nn.DataParallel(model) # Setup the model model.load_state_dict(torch.load(opt["saved_model"])) crit = utils.LanguageModelCriterion() get_caption(model, crit, dataset, dataset.get_vocab(), opt)
def main(opt): dataset = VideoDataset(opt, 'inference') opt["vocab_size"] = dataset.get_vocab_size() opt["seq_length"] = dataset.max_len if opt['beam_size'] != 1: assert opt["batch_size"] == 1 if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], n_layers=opt['num_layers'], rnn_cell=opt['rnn_type'], bidirectional=opt["bidirectional"], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], n_layers=opt['num_layers'], rnn_cell=opt['rnn_type'], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], n_layers=opt['num_layers'], rnn_cell=opt['rnn_type'], input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, decoder) else: return # if torch.cuda.device_count() > 1: # print("{} devices detected, switch to parallel model.".format(torch.cuda.device_count())) # model = nn.DataParallel(model) convnet = 'nasnetalarge' vocab = dataset.get_vocab() full_decoder = ConvS2VT(convnet, model, opt) tf_img_fn = ptm_utils.TransformImage(full_decoder.conv) load_img_fn = PIL.Image.fromarray for video_path in opt['videos']: print(video_path) with torch.no_grad(): frames = skvideo.io.vread(video_path) # bp --- batches = create_batches(frames, load_img_fn, tf_img_fn) seq_prob, seq_preds = full_decoder(batches, mode='inference') sents = utils.decode_sequence(vocab, seq_preds) for sent in sents: print(sent)
def main(opt): dataset = VideoDataset(opt, 'train') dataloader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True, num_workers=0, pin_memory=True) global dataset_val global dataloader_val dataset_val = VideoDataset(opt, 'val') dataloader_val = DataLoader(dataset_val, batch_size=opt["batch_size"], shuffle=True, num_workers=0, pin_memory=True) opt["vocab_size"] = dataset.get_vocab_size() encoder = EncoderRNN( opt["dim_vid"], opt["dim_hidden"], bidirectional=bool(opt["bidirectional"]), input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN( opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=bool(opt["bidirectional"])) model = EncoderDecoderModel(encoder, decoder) model = model.cuda() model = nn.DataParallel(model) model.load_state_dict(torch.load('data/save_vatex_batch_noc3d/model_500.pth')) crit = utils.LanguageModelCriterion() optimizer = optim.Adam(model.parameters(),lr=opt["learning_rate"],weight_decay=opt["weight_decay"]) exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer,step_size=opt["learning_rate_decay_every"],gamma=opt["learning_rate_decay_rate"]) print("Data Loaded") train(dataloader, model, crit, optimizer, exp_lr_scheduler, opt, rl_crit)
def main(opt): dataset = VideoDataset(opt, 'test') opt.vocab_size = dataset.get_vocab_size() opt.seq_length = dataset.seq_length if opt.model == 'S2VTModel': model = S2VTModel(opt.vocab_size, opt.seq_length, opt.dim_hidden, opt.dim_word, rnn_dropout_p=opt.rnn_dropout_p).cuda() elif opt.model == "S2VTAttModel": encoder = EncoderRNN(opt.dim_vid, opt.dim_hidden) decoder = DecoderRNN(opt.vocab_size, opt.seq_length, opt.dim_hidden, opt.dim_word, rnn_dropout_p=0.2) model = S2VTAttModel(encoder, decoder).cuda() model = nn.DataParallel(model) # Setup the model model.load_state_dict(torch.load(opt.saved_model)) model.eval() crit = utils.LanguageModelCriterion() test(model, crit, dataset, dataset.get_vocab(), opt)
def main(opt): device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') if opt["model"] == "S2VTModel": dataset = VideoDataset(opt, "test") elif opt["model"] == "S2VTACTModel": dataset = VideoActDataset(opt, "test") else: print('Currently not supported: {}'.format(opt["model"])) raise ValueError opt["vocab_size"] = dataset.get_vocab_size() opt["seq_length"] = dataset.max_len if opt["model"] == "S2VTModel": model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], rnn_dropout_p=opt["rnn_dropout_p"]).to(device) elif opt["model"] == "S2VTACTModel": model = S2VTACTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], rnn_dropout_p=opt["rnn_dropout_p"]).to(device) elif opt["model"] == "S2VTAttModel": print('Currently Not Supported: {}'.format(opt["model"])) raise ValueError # model = nn.DataParallel(model) # Setup the model model.load_state_dict(torch.load(opt["saved_model"])) crit = utils.LanguageModelCriterion() test(model, crit, dataset, dataset.get_vocab(), device, opt)
def main(opt): dataset = VideoDataset(opt, 'train') dataloader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True) opt["vocab_size"] = dataset.get_vocab_size() encoder = EncoderRNN( opt["dim_vid"], opt["dim_hidden"], bidirectional=bool(opt["bidirectional"]), input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN( opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=bool(opt["bidirectional"])) model = S2VTAttModel(encoder, decoder) #model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], rnn_dropout_p=opt["rnn_dropout_p"]) #model = model.cuda() crit = utils.LanguageModelCriterion() rl_crit = utils.RewardCriterion() optimizer = optim.Adam( model.parameters(), lr=opt["learning_rate"], weight_decay=opt["weight_decay"]) exp_lr_scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=opt["learning_rate_decay_every"], gamma=opt["learning_rate_decay_rate"]) train(dataloader, model, crit, optimizer, exp_lr_scheduler, opt, rl_crit)
def main(opt): dataset = VideoDataset(opt, 'val', 'chinese') opt["vocab_size"] = 13491 #dataset.get_vocab_size() + chinDataset.get_vocab_size() opt["seq_length"] = dataset.max_len encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], bidirectional=bool(opt["bidirectional"]), input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=bool(opt["bidirectional"])) model = S2VTAttModel(encoder, decoder) # Setup the model model.load_state_dict( torch.load(opt["saved_model"], map_location=torch.device('cpu'))) crit = utils.LanguageModelCriterion() test(model, crit, dataset, dataset.get_vocab(), opt)
def main(opt): # DataLoader if opt["model"] == 'S2VTModel': dataset = VideoDataset(opt, 'train') elif opt["model"] == 'S2VTACTModel': dataset = VideoActDataset(opt, 'train') else: print('Currently Not Support this model: {}'.format(opt["model"])) raise ValueError dataloader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True) opt["vocab_size"] = dataset.get_vocab_size() if opt["model"] == 'S2VTModel': print(opt) model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == 'S2VTACTModel': print(opt) model = S2VTACTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == "S2VTAttModel": print('Currently not supported.') raise ValueError # Load model device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') model = model.to(device) # Criterions # LMCriterion = utils.LanguageModelCriterion() # rl_crit = utils.RewardCriterion() optimizer = optim.Adam(model.parameters(), lr=opt["learning_rate"], weight_decay=opt["weight_decay"]) exp_lr_scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=opt["learning_rate_decay_every"], gamma=opt["learning_rate_decay_rate"]) train(dataloader, model, optimizer, exp_lr_scheduler, opt, device, LMCriterion)
def get_loader(opt, mode, print_info=False, specific=-1, target_ratio=-1): dataset = VideoDataset(opt, mode, print_info, specific=specific, target_ratio=target_ratio) if opt.get('splits_redefine_path', ''): dataset.set_splits_by_json_path(opt['splits_redefine_path']) return DataLoader( dataset, batch_size=opt["batch_size"], shuffle=True if mode=='train' else False #shuffle=False )
def __init__(self, ckp_path, epoch_nums, batch_size, lr, lr_step_size=10, resnet_model='resnet50', num_classes=num_classes_train, pre_model_rgb=None): # get params self.ckp_path = ckp_path self.epoch_nums = epoch_nums self.batch_size = batch_size self.lr = lr self.lr_step_size = lr_step_size self.resnet_model = resnet_model self.num_classes = num_classes self.pre_model_rgb = pre_model_rgb # mkdir ckg_path if not os.path.exists(self.ckp_path): os.makedirs(self.ckp_path) # load model self.mymodel = model_rgb(batch_size=self.batch_size, num_classes=self.num_classes) self.mymodel.train() if (pre_model_rgb): checkpoints = torch.load(pre_model_rgb, map_location=lambda storage, loc: storage) self.mymodel.load_state_dict(checkpoints) print('submodel_rgb loaded from {}'.format(pre_model_rgb)) #self.mymodel = torch.nn.DataParallel(self.mymodel) #self.mymodel.cuda() print('model loaded.') # define video_dataloader self.myDataset = VideoDataset(KINETICS_FRAME_DIR, TRAIN_LIST, mode='train') self.myDataloader = DataLoader(self.myDataset, batch_size=self.batch_size, shuffle=True, num_workers=0) self.testDataset = VideoDataset(KINETICS_FRAME_DIR, TEST_LIST, mode='test') self.testDataloader = DataLoader(self.testDataset, batch_size=self.batch_size, shuffle=False, num_workers=0)
def main(opt): dataset = VideoDataset(opt, "test") opt["vocab_size"] = dataset.get_vocab_size() opt["seq_length"] = dataset.max_len encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], bidirectional=bool(opt["bidirectional"]),input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"],input_dropout_p=opt["input_dropout_p"],rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=bool(opt["bidirectional"])) model = EncoderDecoderModel(encoder, decoder).cuda() model = nn.DataParallel(model) model.load_state_dict(torch.load(opt["saved_model"])) crit = utils.LanguageModelCriterion() test(model, crit, dataset, dataset.get_vocab(), opt)
def main(opt): dataset = VideoDataset(opt, 'train') dataloader = DataLoader(dataset, batch_size=opt["batch_size"], num_workers=8, shuffle=True) opt["vocab_size"] = dataset.get_vocab_size() if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], bidirectional=opt["bidirectional"], rnn_dropout_p=opt["rnn_dropout_p"]).cuda() elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], n_layers=opt['num_layers'], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], n_layers=opt['num_layers'], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, decoder).cuda() crit = utils.LanguageModelCriterion() rl_crit = utils.RewardCriterion() optimizer = optim.Adam(model.parameters(), lr=opt["learning_rate"], weight_decay=opt["weight_decay"]) exp_lr_scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=opt["learning_rate_decay_every"], gamma=opt["learning_rate_decay_rate"]) model.load_state_dict( torch.load( "C:\\Users\\Shumpu\\VideoCaptioningAttack\\video_caption_pytorch\\save\\vgg16_model_460.pth" )) train(dataloader, model, crit, optimizer, exp_lr_scheduler, opt, rl_crit)
def eval(): opt = Opt() # prepare data validset = VideoDataset(opt.caption_file, opt.feats_path, mode='test') valid_loader = torch.utils.data.DataLoader(validset, batch_size=opt.batch_size, shuffle=False) word2ix = validset.word2ix ix2word = validset.ix2word vocab_size = len(word2ix) # load model model = torch.load(opt.model_path).to(device) ### ### start test ### pred_dict = {} for index, (feats, targets, IDs, masks) in enumerate(tqdm(valid_loader, desc="test")): # get prediction and cal loss model.eval() with torch.no_grad(): preds = model(feats, mode='test') # preds [B, L] # save result for ID, pred in zip(IDs, preds): word_preds = [ix2word[str(i.item())] for i in pred] if '<eos>' in word_preds: word_preds = word_preds[:word_preds.index('<eos>')] pred_dict[ID] = ' '.join(word_preds) return pred_dict
def get_loader(opt, mode, print_info=False, specific=-1, **kwargs): dataset = VideoDataset(opt, mode, print_info, specific=specific, **kwargs) batch_size = kwargs.get('batch_size', opt['batch_size']) return DataLoader( dataset, batch_size=batch_size, shuffle=True if mode=='train' else False )
def main(opt): train_dataset = VideoDataset(opt, 'train') train_dataloader = DataLoader(train_dataset, batch_size=opt.batch_size, shuffle=True) opt.vocab_size = train_dataset.vocab_size opt.seq_length = train_dataset.seq_length val_dataset = VideoDataset(opt, 'val') val_dataloader = DataLoader(val_dataset, batch_size=opt.batch_size, shuffle=True) if opt.model == 'S2VTModel': model = S2VTModel(opt.vocab_size, opt.seq_length, opt.dim_hidden, opt.dim_word, rnn_dropout_p=opt.rnn_dropout_p).cuda() elif opt.model == "Vid2seq": encoder = EncoderRNN(opt.dim_vid, opt.dim_hidden) decoder = DecoderRNN(opt.vocab_size, opt.seq_length, opt.dim_hidden, use_attention=True, rnn_dropout_p=opt.rnn_dropout_p) model = Vid2seq(encoder, decoder).cuda() crit = utils.LanguageModelCriterion() rl_crit = utils.RewardCriterion() optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate, weight_decay=opt.weight_decay) exp_lr_scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=opt.learning_rate_decay_every, gamma=opt.learning_rate_decay_rate) if not os.path.isdir(opt.checkpoint_path): os.mkdir(opt.checkpoint_path) train(train_dataloader, val_dataloader, model, crit, optimizer, exp_lr_scheduler, opt, rl_crit)
def main(opt): dataset = VideoDataset(opt, "test") opt["vocab_size"] = dataset.get_vocab_size() opt["seq_length"] = dataset.max_len if opt['beam_size'] != 1: assert opt["batch_size"] == 1 if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], n_layers=opt['num_layers'], rnn_cell=opt['rnn_type'], bidirectional=opt["bidirectional"], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], n_layers=opt['num_layers'], rnn_cell=opt['rnn_type'], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], n_layers=opt['num_layers'], rnn_cell=opt['rnn_type'], input_dropout_p=opt["input_dropout_p"], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, decoder) else: return if torch.cuda.device_count() > 1: print("{} devices detected, switch to parallel model.".format(torch.cuda.device_count())) model = nn.DataParallel(model) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) # Setup the model model.load_state_dict(torch.load(opt["saved_model"])) crit = utils.LanguageModelCriterion() test(model, crit, dataset, dataset.get_vocab(), opt)
def main(opt): dataset_test = VideoDataset(opt, 'test') dataloader_test = DataLoader(dataset_test, batch_size=opt["batch_size"], shuffle=False) opt["obj_vocab_size"] = dataset_test.get_obj_vocab_size() opt["rel_vocab_size"] = dataset_test.get_rel_vocab_size() if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["obj_vocab_size"], opt["rel_vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, decoder) model = model.cuda() model.load_state_dict(torch.load(opt['ckpt_path'])) crit = utils.ObjRelCriterion() test(model, crit, opt, dataloader_test)
def main(opt): opt_test = opt test_dataset = VideoDataset(opt_test, 'test') opt_test["vocab_size"] = test_dataset.get_vocab_size() opt_test["seq_length"] = test_dataset.max_len dataset = VideoDataset(opt, 'train') dataloader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True) opt["vocab_size"] = dataset.get_vocab_size() if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN( opt["dim_vid"], opt["dim_hidden"], # bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) second_lstm = Two_Lstm( opt["dim_vid"], opt["dim_hidden"], # bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) # bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, second_lstm, decoder) model = model.cuda() crit = utils.LanguageModelCriterion() rl_crit = utils.RewardCriterion() optimizer = optim.Adam(model.parameters(), lr=opt["learning_rate"], weight_decay=opt["weight_decay"]) exp_lr_scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=opt["learning_rate_decay_every"], gamma=opt["learning_rate_decay_rate"]) train(dataloader, model, crit, optimizer, exp_lr_scheduler, opt, rl_crit, opt_test, test_dataset)
def train(opt,EncoderRNN,DecoderCNN,Convcap,itow): ''' training initialize the models pass the arg through the Convcap model output: /checkpoint/model.pth ----- trained model ''' t_start = time.time() train_data=VideoDataset(opt, 'train') #####DataLODER##### collate_fn=PadSequence() train_loader=DataLoader(train_data, batch_size=opt["batch_size"],collate_fn=collate_fn,num_workers=opt['num_workers'], shuffle=True) print('[DEBUG] Loading train data ... %f secs' % (time.time() - t_start)) #,word_embed,masks,gts for c3d_feat,lengths,labels,ids,word_embed,masks,gts in train_loader: #for data in train_loader: print("came here")
def main(): # 画像群のパス作成 root_path = './data/kinetics_videos/' video_list = make_datapath_list(root_path) # 動画クラスラベルとIDの辞書作成 label_dicitionary_path = './video_download/kinetics_400_label_dicitionary.csv' label_id_dict, id_label_dict = get_label_id_dictionary( label_dicitionary_path) # 前処理の設定 resize, crop_size = 224, 224 mean, std = [104, 117, 123], [1, 1, 1] video_transform = VideoTransform(resize, crop_size, mean, std) # Datasetの作成 val_dataset = VideoDataset(video_list, label_id_dict, num_segments=16, phase='val', transform=video_transform, img_template='image_{:05d}.jpg') # DataLoaderを作成して,データをロード batch_size = 8 val_dataloader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False) batch_iterator = iter(val_dataloader) # イテレータに変換 imgs_transformeds, labels, label_ids, dir_path = next(batch_iterator) # モデルをインスタンス化し,推論モードに変更後,学習済みモデルをロード net = ECO_Lite() net.eval() net.load_state_dict(torch.load('./models/pretrained.pth')) # ECOで推論 with torch.set_grad_enabled(False): outputs = net(imgs_transformeds) # 予測を実施 idx = 0 show_eco_inference_result(dir_path, outputs, id_label_dict, idx)
def train(opt,EncoderRNN,DecoderCNN,Convcap,itow): ''' training initialize the models pass the arg through the Convcap model output: /checkpoint/model.pth ----- trained model ''' t_start = time.time() train_data=VideoDataset(opt, 'train') #####DataLODER##### #collate_fn=PadSequence(),collate_fn=collate_fn, train_loader=DataLoader(train_data, batch_size=opt["batch_size"],num_workers=opt['num_workers'], shuffle=True) print('[DEBUG] Loading train data ... %f secs' % (time.time() - t_start)) ##initialize encoder,decoder,model encoder=EncoderRNN.EncoderRNN(opt['dim_vid'],opt['dim_hidden'],bidirectional=opt['bidirectional'],rnn_cell=opt['rnn_type']).cuda() decoder=DecoderCNN.DecoderCNN(train_data.get_vocab_size()).cuda() convcap=Convcap.Convcap(encoder,decoder).cuda() ####initialize hyper params optimizer = optim.RMSprop(convcap.parameters(), lr=opt["learning_rate"]) scheduler = lr_scheduler.StepLR(optimizer, step_size=opt["learning_rate_decay_every"],gamma=opt["learning_rate_decay_rate"]) batchsize =opt['batch_size'] cap_size= opt['max_len'] nbatches = np.int_(np.floor((len(train_data)*1.)/batchsize)) bestscore = .0 batchsize_cap = batchsize*1 max_tokens= opt['max_len'] # print(batchsize,cap_size,batchsize_cap,max_tokens) # print("nbatches"+str(nbatches)) itr = 0 loss_graph=[] graph_x=[] for epoch in range(opt['epochs']): loss_train = 0. scheduler.step() for data in train_loader: #for c3d_feat,lengths,labels,word_embed,masks,gts in train_loader: #print("came here") print("iteration"+str(itr)) itr+=1 vid_feat=Variable(data['c3d_feats']).cuda() labels = Variable(data['labels'].type(torch.LongTensor)).cuda() mask = Variable(data['masks']).cpu() word_embed=Variable(data['word_embed']).cuda() cap=data['cap'] # vid_feat=Variable(c3d_feat).cuda() # print(vid_feat.dtype) # print(vid_feat.shape) # labels = Variable(torch.FloatTensor(labels)).cuda() # mask = Variable(torch.FloatTensor(masks)).cpu() # word_embed= torch.stack([x for x in word_embed],dim=0) # print(word_embed.shape) # word_embed=Variable(word_embed).cuda() # lengths=lengths.type(torch.FloatTensor) # print(c3d_feat,word_embed) optimizer.zero_grad() wordact = convcap(vid_feat,labels,word_embed,'train') # print("//////////////////////////////////////////////") #print("1.wordact.shape"+str(wordact.shape)) wordact = wordact[:,:,:-1] #print("2.wordact.shape"+str(wordact.shape)) # print("////////////////////////////////////") labels = labels[:,1:] mask = mask[:,1:].contiguous() # print (wordact.shape) # print(batchsize_cap,max_tokens) wordact_t = wordact.permute(0, 2, 1).contiguous().view(\ batchsize*(max_tokens-1), -1) #print(wordact_t.shape) wordclass_t = labels.contiguous().view(\ batchsize*(max_tokens-1), 1) maskids = torch.nonzero(mask.view(-1)).numpy().reshape(-1) #print("mask Ids \t"+str(maskids)) loss = F.cross_entropy(wordact_t[maskids, ...], \ wordclass_t[maskids, ...].contiguous().view(maskids.shape[0])) ########### for visual############################## wordprobs = F.softmax(wordact_t).cpu().data.numpy() # print("word_class \t"+str(wordclass_t.shape)+"\t"+str(wordclass_t.dtype)) # print(wordclass_t[maskids, ...].contiguous().view(maskids.shape[0])) # print(wordact_t[maskids, ...]) wordids = np.argmax(wordprobs, axis=1) for i in wordids: print(itow[i]) # for i in wordclass_t[maskids, ...].contiguous().view(maskids.shape[0]): # print(itow[i]) print(cap) ############################ ##################### if itr%500 == 0: graph_x.append(itr) loss_graph.append(loss) loss_train = loss_train + loss.item() loss.backward() optimizer.step() print("loss"+str(loss_train)) loss_train = (loss_train*1.)/(nbatches) print('[DEBUG] Training epoch %d has loss %f' % (epoch, loss_train)) modelfn = osp.join(opt['checkpoint_path'], 'model_j_19_'+str(itr)+'.pth') torch.save({ 'epoch': epoch, 'state_dict': convcap.state_dict(), 'optimizer' : optimizer.state_dict(), 'loss':loss_train }, modelfn) print('time for epoch %f' % (time.time() - t_start)) plt.plot(graph_x,loss_graph,'ro') plt.show()
def train(): opt = Opt() # write log save_opt(opt) # prepare data trainset = VideoDataset(opt.caption_file, opt.feats_path) train_loader = torch.utils.data.DataLoader(trainset, batch_size=opt.batch_size, shuffle=True) testset = VideoDataset(opt.caption_file, opt.feats_path, mode='valid') test_loader = torch.utils.data.DataLoader(testset, batch_size=opt.batch_size, shuffle=False) word2ix = trainset.word2ix ix2word = trainset.ix2word vocab_size = len(word2ix) # build model # model = S2VT_Att( # vocab_size, # opt.feat_dim, # dim_hid=opt.dim_hidden, # dim_embed=opt.dim_embed, # length=opt.train_length, # feat_dropout=opt.feat_dropout, # rnn_dropout=opt.rnn_dropout, # out_dropout=opt.out_dropout, # num_layers=opt.num_layers, # bidirectional=opt.bidirectional, # rnn_type=opt.rnn_type, # sos_ix=word2ix['<sos>'], # eos_ix=word2ix['<eos>'], # ).to(device) model = Att_Baseline(vocab_size, opt.feat_dim, length=opt.train_length, dim_hid=opt.dim_hidden, dim_embed=opt.dim_embed, feat_dropout=opt.feat_dropout, out_dropout=opt.out_dropout, sos_ix=3, eos_ix=4).to(device) # model.load_glove_weights('./data/glove.6B.300d.txt', 300, trainset.ix2word) optimizer = optim.Adam( model.parameters(), lr=opt.lr, # weight_decay=opt.weight_decay ) # dynamic learning rate lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau( optimizer, verbose=True, patience=opt.learning_rate_patience) early_stopping = EarlyStopping(patience=opt.early_stopping_patience, verbose=True, path=os.path.join( opt.save_path, opt.start_time + 'stop.pth')) criterion = MaskCriterion() ### ### start training ### try: for epoch in range(opt.EPOCHS): # **************************** # train # **************************** train_running_loss = 0.0 loss_count = 0 for index, (feats, targets, IDs, masks) in enumerate( tqdm(train_loader, desc="epoch:{}".format(epoch))): optimizer.zero_grad() model.train() # probs [B, L, vocab_size] probs = model(feats, targets=targets[:, :-1], mode='train') loss = criterion(probs, targets, masks) loss.backward() optimizer.step() train_running_loss += loss.item() loss_count += 1 train_running_loss /= loss_count writer.add_scalar('train_loss', train_running_loss, global_step=epoch) # **************************** # validate # **************************** valid_running_loss = 0.0 loss_count = 0 for index, (feats, targets, IDs, masks) in enumerate(test_loader): model.eval() with torch.no_grad(): probs = model(feats, targets=targets[:, :-1], mode='train') loss = criterion(probs, targets, masks) valid_running_loss += loss.item() loss_count += 1 valid_running_loss /= loss_count writer.add_scalar('valid_loss', valid_running_loss, global_step=epoch) writer.add_scalar('lr', optimizer.state_dict()['param_groups'][0]['lr'], global_step=epoch) if epoch % opt.histogram_freq == 0: for i, (name, param) in enumerate(model.named_parameters()): writer.add_histogram(name, param, epoch) print("train loss:{} valid loss: {}".format( train_running_loss, valid_running_loss)) lr_scheduler.step(valid_running_loss) # early stop early_stopping(valid_running_loss, model) if early_stopping.early_stop: print("Early stopping") break # save checkpoint if epoch % opt.save_freq == 0: print('epoch:{}, saving checkpoint'.format(epoch)) torch.save( model, os.path.join(opt.save_path, opt.start_time + str(epoch) + '.pth')) except KeyboardInterrupt as e: print(e) print("Training interruption, save tensorboard log...") writer.close() # save model torch.save(model, os.path.join(opt.save_path, opt.start_time + 'final.pth'))
def main(): timestamp = str(time.asctime(time.localtime(time.time()))) print('__pyTorch VERSION:', torch.__version__) print('__CUDA VERSION') print('__CUDNN VERSION:', torch.backends.cudnn.version()) print('__Number CUDA Devices:', torch.cuda.device_count()) print('__Devices') use_cuda = False # use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") # Training settings print("DEVICE ", device) max_epochs = 100 N_CL = 47 dim = (224, 224) seq_length = 300 n_channels = 3 test_params = {'batch_size': 1, 'shuffle': False, 'num_workers': 2} train_params = {'batch_size': 1, 'shuffle': True, 'num_workers': 4} parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--modality', type=str, default='full_image', metavar='rc', help='hands or full image') parser.add_argument('--batch-size', type=int, default=1, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=max_epochs, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.00001, metavar='LR', help='learning rate (default: 0.00001)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--dataloader', type=str, default='rand_crop', metavar='rc', help='data augmentation') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=100, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') parser.add_argument('--seq-length', type=int, default=seq_length, metavar='num', help='squence length') parser.add_argument('--hidden_size', type=int, default=1024, metavar='num', help='lstm units') parser.add_argument('--optim', type=int, default=1, metavar='optim number', help='optimizer sgd or adam') parser.add_argument('--n_layers', type=int, default=2, metavar='num', help='hidden size') parser.add_argument('--dropout', type=float, default=0.7, metavar='num', help='hidden size') parser.add_argument('--bidirectional', action='store_true', default=True, help='hidden size') args = parser.parse_args() # Helper: Save the model. train_partition, train_labels = load_csv_file(train_filepath) test_partition, test_labels = load_csv_file(test_filepath) classes = count_classes(train_labels) id2w = list(range(len(classes))) id2w = dict(zip(id2w, classes)) #print(id2w) print('Number of Classes {} \n \n '.format(len(classes))) # DATA GENERATORS training_set = VideoDataset(train_prefix, train_partition, train_labels, classes, seq_length, dim, False, False) training_generator = data.DataLoader(training_set, **train_params) validation_set = VideoDataset(test_prefix, test_partition, test_labels, classes, seq_length, dim, False, False) validation_generator = data.DataLoader(validation_set, **test_params) # MODEL ILITIALIZE model = RNNModel(args.hidden_size, args.n_layers, args.dropout, len(classes)).to(device) # model.load_state_dict( # torch.load('/home/hatzis/Desktop/stim_ctc/checkpoints/stim_ctc_firt_try/best_wer46.35898383147232.pth', map_location=device)[ # 'model_dict']) if (args.optim == 1): opt_name = 'ADAMlr ' print(" use optim ", opt_name, args.lr) optimizer = optim.Adam(model.parameters(), lr=args.lr) elif (args.optim == 2): opt_name = 'SGD lr ' print(" use optim", opt_name, args.lr) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.8, weight_decay=0.0000001) max_epochs = 100 scheduler = StepLR(optimizer, step_size=20, gamma=0.5) # TRAINING LOOP for epoch in range(1, max_epochs): #scheduler.step() train(args, model, device, training_generator, optimizer, epoch, timestamp, ngrams, id2w) print("!!!!!!!! VALIDATION !!!!!!!!!!!!!!!!!!") validate(args, model, device, validation_generator, optimizer, epoch, timestamp, ngrams, id2w)
def cross_validate(model, crit, opt): dataset = VideoDataset(opt, 'val') print(len(dataset)) _, _, seq_probs, seq_preds, labels, masks = test(model, crit, dataset, dataset.get_vocab(), opt)
import numpy as np import csv import time import argparse import opt from dataloader import VideoDataset import json import os import os.path as osp import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.optim import lr_scheduler from torch.autograd import Variable from torch.utils.data import DataLoader from model import EncoderRNN, DecoderCNN, Convcap train_data = VideoDataset(opt, 'train') train_loader = DataLoader(train_data, batch_size=opt["batch_size"], num_workers=3, shuffle=False) for data in train_loader: vid_feat = data['c3d_feats'] labels = data['labels'] mask = data['masks'] word_embed = data['word_embed'] vid_id = data['video_id']
def prepare_data(self): self.train_dataset = VideoDataset(self.hparams.train_dir, transform=self.hparams.augmentation) self.test_dataset = ImageDataset(self.hparams.test_dir)
def test(opt,EncoderRNN,DecoderCNN,Convcap,itow,wtoi,modelfn=None): ''' input : option, model, checkpoint/model.pth output :scores ''' t_start = time.time() t_start = time.time() test_data=VideoDataset(opt, 'test') test_loader=DataLoader(test_data, batch_size=opt["batch_size"],num_workers=30, shuffle=False) print('[DEBUG] Loading test data ... %f secs' % (time.time() - t_start)) batchsize =opt['batch_size'] cap_size= opt['max_len'] nbatches = np.int_(np.floor((len(test_data)*1.)/batchsize)) bestscore = .0 batchsize_cap = batchsize*1 max_tokens= opt['max_len'] if(modelfn is not None): encoder=EncoderRNN.EncoderRNN(opt['dim_vid'],opt['dim_hidden'],bidirectional=opt['bidirectional'],rnn_cell=opt['rnn_type']).cuda() decoder=DecoderCNN.DecoderCNN(test_data.get_vocab_size()).cuda() convcap=Convcap.Convcap(encoder,decoder).cuda() print('[DEBUG] Loading checkpoint %s' % modelfn) checkpoint = torch.load(modelfn) convcap.load_state_dict(checkpoint['state_dict']) convcap.train(False) pred_captions = [] itr=0 for data in test_loader: print("iteration"+str(itr)) #print(data['labels'].data[0]) print("\n") print("gt\n") for i in data['labels'].data[0]: print(itow[int(i)]) itr+=1 vid_feat=Variable(data['c3d_feats']).cuda() labels=Variable(data['labels'].type(torch.LongTensor)).cuda() mask = Variable(data['masks']).cpu() word_embed=Variable(data['word_embed']).cuda() vid_id=data['video_ids'] #print(vid_id[0]) wordclass_feed = np.zeros((batchsize, max_tokens), dtype='int64') wordclass_feed[:,0] =wtoi['<sos>'] #1 #index of <sos> # print('wordclass_feed shape') # print(wordclass_feed.shape) outcaps = np.empty((batchsize, 0)).tolist() x_outcaps=np.empty((batchsize, 0)).tolist() for j in range(max_tokens-1): wordclass = Variable(torch.from_numpy(wordclass_feed)).cuda() wordact = convcap(vid_feat,wordclass,word_embed,'test') x=convcap(vid_feat,labels,word_embed,'test') x=x[:,:,:-1] x_t=x.permute(0, 2, 1).contiguous().view(batchsize*(max_tokens-1), -1) x_prob=F.softmax(x_t).cpu().data.numpy() wordact = wordact[:,:,:-1] wordact_t = wordact.permute(0, 2, 1).contiguous().view(batchsize*(max_tokens-1), -1) # print("convcap output"+str(wordact_t.shape)) wordprobs = F.softmax(wordact_t).cpu().data.numpy() x_id=np.argmax(x_prob, axis=1) wordids = np.argmax(wordprobs, axis=1) probs=np.max(wordprobs,axis=1) for k in range(batchsize): word = itow[wordids[j+k*(max_tokens-1)]] x_word=itow[x_id[j+k*(max_tokens-1)]] outcaps[k].append(word) x_outcaps[k].append(x_word) if(j < max_tokens-1): wordclass_feed[k, j+1] = wordids[j+k*(max_tokens-1)] # print("word ids"+str(wordids[j+k*(max_tokens-1)])) for j in range(batchsize): num_words = len(outcaps[j]) x_n_words=len(x_outcaps[j]) if 'eos' in x_outcaps[j]: x_n_words=x_outcaps[j].index('eos') x_outcap=' '.join(x_outcaps[j][:num_words]) if 'eos' in outcaps[j]: num_words = outcaps[j].index('eos') outcap = ' '.join(outcaps[j][:num_words]) pred_captions.append({'vid_id': vid_id[0][5:], 'caption': outcap}) print("------------------------------------------------------------------------------") print("videoID \t"+str(vid_id)) print("caption \n") print(x_outcap) #print(itow[int(i)] for i in data['labels'].data[0]) print("------------------------------------------------------------------------------") scores = language_eval(pred_captions, '/home/sanjay/Documents/Video_convcap/output', 'test') return scores
import opts from dataloader import VideoDataset from torch.utils.data import DataLoader if __name__ == '__main__': opt = opts.parse_opt() opt = vars(opt) dataset = VideoDataset(opt, 'train') loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True) # data['fc_feats'] = torch.from_numpy(fc_feat).type(torch.FloatTensor) # data['labels'] = label # data['masks'] = mask # data['gts'] = gts # data['video_ids'] = vid_id for data in loader: for key in ('fc_feats', 'labels', 'masks', 'video_ids'): print("{} shape: {}".format(key, data[key].shape)) break
def main(opt): dataset = VideoDataset(opt, 'train') dataloader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True) opt["vocab_size"] = dataset.get_vocab_size() if opt["model"] == 'S2VTModel': model = S2VTModel(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], opt['dim_vid'], rnn_cell=opt['rnn_type'], n_layers=opt['num_layers'], rnn_dropout_p=opt["rnn_dropout_p"]) elif opt["model"] == "S2VTAttModel": encoder = EncoderRNN(opt["dim_vid"], opt["dim_hidden"], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) # # 声音encoder # encoder_voice = EncoderRNN( # opt["dim_voice"], # opt["dim_hidden"], # bidirectional=opt["bidirectional"], # input_dropout_p=opt["input_dropout_p"], # rnn_cell=opt['rnn_type'], # rnn_dropout_p=opt["rnn_dropout_p"]) # 手语encoder if opt['with_hand'] == 1: encoder_hand = EncoderRNN(opt["dim_hand"], opt["dim_hand_hidden"], bidirectional=opt["bidirectional"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"]) decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"] + opt["dim_hand_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, encoder_hand, decoder) else: decoder = DecoderRNN(opt["vocab_size"], opt["max_len"], opt["dim_hidden"], opt["dim_word"], input_dropout_p=opt["input_dropout_p"], rnn_cell=opt['rnn_type'], rnn_dropout_p=opt["rnn_dropout_p"], bidirectional=opt["bidirectional"]) model = S2VTAttModel(encoder, None, decoder) # model = S2VTAttModel(encoder, encoder_voice, encoder_hand, decoder) model = model.cuda() crit = utils.LanguageModelCriterion() rl_crit = utils.RewardCriterion() optimizer = optim.Adam(model.parameters(), lr=opt["learning_rate"], weight_decay=opt["weight_decay"]) exp_lr_scheduler = optim.lr_scheduler.StepLR( optimizer, step_size=opt["learning_rate_decay_every"], gamma=opt["learning_rate_decay_rate"]) # print(dataloader) # print(crit) # print(optimizer) train(dataloader, model, crit, optimizer, exp_lr_scheduler, opt, rl_crit)
device = 'cuda:0' transform = transforms.Compose( [ transforms.Resize(args.size), transforms.CenterCrop(args.size), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]), ] ) #dataset = datasets.ImageFolder(args.path, transform=transform) #loader = DataLoader(dataset, batch_size=128, shuffle=True, num_workers=4) train_data = VideoDataset(args.path, split='train', clip_len=3, preprocess=False,transform=transform) loader = DataLoader(train_data, batch_size=args.bs, shuffle=True, num_workers=4) test_data = VideoDataset(args.path, split='val', clip_len=3, clip_jump=1,preprocess=False,transform=transform) test_loader = DataLoader(test_data, batch_size=args.bs, shuffle=True, num_workers=4) #raw_input('Enter') model = DQLR() if not args.pretrained == None: print('Loading pretrained weights...') pre_w = torch.load(args.pretrained) for key in pre_w.keys(): model.state_dict()[key] = pre_w[key] if 'dec' in key: key2 = key.replace('dec','dec1') model.state_dict()[key2] = pre_w[key] key2 = key.replace('dec','dec2')