def main(): if not args.evaluate: sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt')) else: sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt')) print("==========\nArgs:{}\n==========".format(args)) if use_gpu: print("Currently using GPU {}".format(args.gpu)) cudnn.benchmark = True torch.cuda.manual_seed_all(args.seed) else: print("Currently using CPU") print("Initialize dataset {}".format(args.dataset)) dataset = h5py.File(args.dataset, 'r') num_videos = len(dataset.keys()) splits = read_json(args.split) assert args.split_id < len(splits), "split_id (got {}) exceeds {}".format(args.split_id, len(splits)) split = splits[args.split_id] train_keys = split['train_keys'] test_keys = split['test_keys'] print("# total videos {}. # train videos {}. # test videos {}".format(num_videos, len(train_keys), len(test_keys))) print("Initialize model") model = DSN(in_dim=args.input_dim, hid_dim=args.hidden_dim, num_layers=args.num_layers, cell=args.rnn_cell) print("Model size: {:.5f}M".format(sum(p.numel() for p in model.parameters())/1000000.0)) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.stepsize > 0: scheduler = lr_scheduler.StepLR(optimizer, step_size=args.stepsize, gamma=args.gamma) if args.resume: print("Loading checkpoint from '{}'".format(args.resume)) checkpoint = torch.load(args.resume) model.load_state_dict(checkpoint) else: start_epoch = 0 if use_gpu: model = nn.DataParallel(model).cuda() if args.evaluate: print("Evaluate only") evaluate(model, dataset, test_keys, use_gpu) return print("==> Start training") start_time = time.time() model.train() baselines = {key: 0. for key in train_keys} # baseline rewards for videos reward_writers = {key: [] for key in train_keys} # record reward changes for each video for epoch in range(start_epoch, args.max_epoch): idxs = np.arange(len(train_keys)) np.random.shuffle(idxs) # shuffle indices for idx in idxs: key = train_keys[idx] seq = dataset[key]['features'][...] # sequence of features, (seq_len, dim) seq = torch.from_numpy(seq).unsqueeze(0) # input shape (1, seq_len, dim) if use_gpu: seq = seq.cuda() probs = model(seq) # output shape (1, seq_len, 1) cost = args.beta * (probs.mean() - 0.5)**2 # minimize summary length penalty term [Eq.11] m = Bernoulli(probs) epis_rewards = [] for _ in range(args.num_episode): actions = m.sample() log_probs = m.log_prob(actions) reward = compute_reward(seq, actions, use_gpu=use_gpu) expected_reward = log_probs.mean() * (reward - baselines[key]) cost -= expected_reward # minimize negative expected reward epis_rewards.append(reward.item()) optimizer.zero_grad() cost.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0) optimizer.step() baselines[key] = 0.9 * baselines[key] + 0.1 * np.mean(epis_rewards) # update baseline reward via moving average reward_writers[key].append(np.mean(epis_rewards)) epoch_reward = np.mean([reward_writers[key][epoch] for key in train_keys]) print("epoch {}/{}\t reward {}\t".format(epoch+1, args.max_epoch, epoch_reward)) write_json(reward_writers, osp.join(args.save_dir, 'rewards.json')) evaluate(model, dataset, test_keys, use_gpu) elapsed = round(time.time() - start_time) elapsed = str(datetime.timedelta(seconds=elapsed)) print("Finished. Total elapsed time (h:m:s): {}".format(elapsed)) model_state_dict = model.module.state_dict() if use_gpu else model.state_dict() model_save_path = osp.join(args.save_dir, 'model_epoch' + str(args.max_epoch) + '.pth.tar') save_checkpoint(model_state_dict, model_save_path) print("Model saved to {}".format(model_save_path)) dataset.close()
def main(): if not args.evaluate: sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt')) else: sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt')) print("==========\nArgs:{}\n==========".format(args)) # if use_gpu: # print("Currently using GPU {}".format(args.gpu)) # cudnn.benchmark = True # torch.cuda.manual_seed_all(args.seed) # else: # print("Currently using CPU") print("Initialize dataset {}".format(args.dataset)) dataset = h5py.File(args.dataset, 'r') num_videos = len(dataset.keys()) splits = read_json(args.split) assert args.split_id < len(splits), "split_id (got {}) exceeds {}".format( args.split_id, len(splits)) split = splits[args.split_id] train_keys = split['train_keys'] test_keys = split['test_keys'] print("# total videos {}. # train videos {}. # test videos {}".format( num_videos, len(train_keys), len(test_keys))) print("Initialize model") model = Net() # model._set_inputs(np.array([1, 1024, 1024])) model.build((1, 1024, 1024)) print("\n", model.summary(), "\n") print("Model size: {:.5f}M".format( np.sum([np.prod(v.shape) for v in model.trainable_weights]) / 1000000.0)) if args.stepsize > 0: lr = tf.keras.optimizers.schedules.ExponentialDecay( args.lr, decay_rate=args.decay_rate, decay_steps=args.stepsize) else: lr = args.lr optimizer = tf.keras.optimizers.Adam(learning_rate=lr) if args.resume: print("Loading checkpoint from '{}'".format(args.resume)) model = tf.keras.models.load_model(args.resume) else: start_epoch = 0 # if use_gpu: # model = nn.DataParallel(model).cuda() if args.evaluate: print("Evaluate only") evaluate(model, dataset, test_keys) return print("==> Start training") start_time = time.time() # model.train() baselines = {key: 0. for key in train_keys} # baseline rewards for videos reward_writers = {key: [] for key in train_keys } # record reward changes for each video for epoch in range(start_epoch, args.max_epoch): idxs = np.arange(len(train_keys)) np.random.shuffle(idxs) # shuffle indices for idx in idxs: key = train_keys[idx] seq = dataset[key]['features'][ ...] # sequence of features, (seq_len, dim) seq = tf.expand_dims(tf.convert_to_tensor(seq), axis=0) # input shape (1, seq_len, dim) # print("seq shape:{}".format(seq.shape)) seq_len = seq.shape[1] # if use_gpu: seq = seq.cuda() with tf.GradientTape() as tape: probs = model(seq, training=True) # output shape (1, seq_len) # print("probs shape:{}".format(probs.shape)) cost = args.beta * ( tf.math.reduce_mean(probs) - 0.5)**2 # minimize summary length penalty term [Eq.11] m = tfp.distributions.Bernoulli(probs=probs) epis_rewards = [] for _ in range(args.num_episode): actions = m.sample() log_probs = tf.squeeze(m.log_prob(actions)) log_probs_mean = tf.math.reduce_mean(log_probs) reward = tf.stop_gradient(compute_reward(seq, actions)) reward_lowvar = tf.stop_gradient(reward - baselines[key]) expected_reward = log_probs_mean * reward_lowvar cost = cost - expected_reward # minimize negative expected reward epis_rewards.append(float(reward)) cost = cost / args.num_episode grads = tape.gradient(cost, model.trainable_weights) grads, _ = tf.clip_by_global_norm(grads, 5.0) optimizer.apply_gradients(zip(grads, model.trainable_weights)) baselines[key] = 0.9 * baselines[key] + 0.1 * np.mean( epis_rewards) # update baseline reward via moving average reward_writers[key].append(np.mean(epis_rewards)) epoch_reward = np.mean( [reward_writers[key][epoch] for key in train_keys]) print("epoch {}/{}\t reward {}\t".format(epoch + 1, args.max_epoch, epoch_reward)) write_json(reward_writers, osp.join(args.save_dir, 'rewards.json')) evaluate(model, dataset, test_keys) elapsed = round(time.time() - start_time) elapsed = str(datetime.timedelta(seconds=elapsed)) print("Finished. Total elapsed time (h:m:s): {}".format(elapsed)) # model_state_dict = model.module.state_dict() if use_gpu else model.state_dict() model_save_path = osp.join(args.save_dir, 'model_epoch' + str(args.max_epoch)) save_checkpoint(model, model_save_path) print("Model saved to {}".format(model_save_path)) dataset.close()
def main(): if not args.evaluate: sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt')) else: sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt')) print("==========\nArgs:{}\n==========".format(args)) if use_gpu: print("Currently using GPU {}".format(args.gpu)) cudnn.benchmark = True torch.cuda.manual_seed_all(args.seed) else: print("Currently using CPU") print("Initialize dataset") fpath = args.path_to_features + '*' features = [] fpaths = [] for f in sorted(glob.glob(fpath)): fpaths.append(f) f1 = np.load(f, allow_pickle=True) features.append(f1) features_all = (np.stack(features)) dist = features_all.shape[0] number_of_picks = args.number_of_picks start = args.start_idx features = features_all[start:start + dist, :] model = DSN(in_dim=args.classes, hid_dim=args.hidden_dim) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.stepsize > 0: scheduler = lr_scheduler.StepLR(optimizer, step_size=args.stepsize, gamma=args.gamma) if args.resume: print("Loading checkpoint from '{}'".format(args.resume)) checkpoint = torch.load(args.resume) model.load_state_dict(checkpoint) else: start_epoch = 0 if use_gpu: model = nn.DataParallel(model).cuda() print("==> Start training") start_time = time.time() model.train() baseline = 0. best_reward = 0.0 best_pi = [] os.system('rm -r ./selection/') os.system('mkdir selection') for epoch in range(start_epoch, args.max_epoch): seq = features seq = torch.from_numpy(seq).unsqueeze( 0) # input shape (1, seq_len, dim) if use_gpu: seq = seq.cuda() probs = model(seq) # output shape (1, seq_len, 1) cost = args.beta * (probs.mean() - 0.5)**2 m = Bernoulli(probs) epis_rewards = [] for _ in range(args.num_episode): actions = m.sample() log_probs = m.log_prob(actions) reward, pick_idxs = compute_reward(seq, actions, probs, nc=args.classes, picks=number_of_picks, use_gpu=use_gpu) if (reward > best_reward): best_reward = reward best_pi = pick_idxs expected_reward = log_probs.mean() * (reward - baseline) cost -= expected_reward # minimize negative expected reward epis_rewards.append(reward.item()) optimizer.zero_grad() cost.backward() optimizer.step() baseline = 0.9 * baseline + 0.1 * np.mean( epis_rewards) # update baseline reward via moving average print("epoch {}/{}\t reward {}\t".format(epoch + 1, args.max_epoch, np.mean(epis_rewards))) f = open('selection/' + str(start) + '.txt', 'w') for idx in best_pi: f.write(fpaths[start + idx] + '\n') f.close() elapsed = round(time.time() - start_time) elapsed = str(datetime.timedelta(seconds=elapsed)) print("Finished. Total elapsed time (h:m:s): {}".format(elapsed)) model_state_dict = model.module.state_dict( ) if use_gpu else model.state_dict() model_save_path = osp.join( args.save_dir, 'model_epoch' + str(args.max_epoch) + '.pth.tar') save_checkpoint(model_state_dict, model_save_path) print("Model saved to {}".format(model_save_path))
for epoch in range(start_epoch, args_max_epoch): probs = model(seq) # output shape (1, seq_len, 1) #cost is a torch.Tensor #cost = args_beta * (probs.mean() - 0.5)**2 # minimize summary length penalty term [Eq.11] cost = args_beta * (probs.mean() - 8.0 / number_of_images )**2 # minimize summary length penalty term [Eq.11] #0.5 means the percentage of frames to be selected. #in my case. it is m = Bernoulli(probs) #Bernoulli model epis_rewards = [] for _ in range(args_num_episode): #default value as 5 actions = m.sample() #This is a cuda tensor, cannot be detach() log_probs = m.log_prob(actions) #construct a loss function reward = compute_reward(seq, actions, use_gpu=use_gpu) expected_reward = log_probs.mean() * ( reward - baselines[key]) #What is the meaning of the baselines? cost -= expected_reward # minimize negative expected reward # cost -= reward epis_rewards.append(reward.item()) # print('cost',cost) optimizer.zero_grad() cost.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0) optimizer.step() #update the model baselines[key] = 0.9 * baselines[key] + 0.1 * np.mean( epis_rewards) # update baseline reward via moving average # print('baselines[key]',baselines[key]) #it increases # reward_writers[key].append(np.mean(epis_rewards)) #it is the mean of epis_rewards
def main(): if not args.evaluate: sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt')) else: sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt')) print("==========\nArgs:{}\n==========".format(args)) if use_gpu: print("Currently using GPU {}".format(args.gpu)) cudnn.benchmark = True torch.cuda.manual_seed_all(args.seed) else: print("Currently using CPU") print("Initialize dataset {}".format(args.dataset)) if args.dataset is None: datasets = [ 'datasets/eccv16_dataset_summe_google_pool5.h5', 'datasets/eccv16_dataset_tvsum_google_pool5.h5', 'datasets/eccv16_dataset_ovp_google_pool5.h5', 'datasets/eccv16_dataset_youtube_google_pool5.h5' ] dataset = {} for name in datasets: _, base_filename = os.path.split(name) base_filename = os.path.splitext(base_filename) dataset[base_filename[0]] = h5py.File(name, 'r') # Load split file splits = read_json(args.split) assert args.split_id < len( splits), "split_id (got {}) exceeds {}".format( args.split_id, len(splits)) split = splits[args.split_id] train_keys = split['train_keys'] test_keys = split['test_keys'] print("# train videos {}. # test videos {}".format( len(train_keys), len(test_keys))) else: dataset = h5py.File(args.dataset, 'r') num_videos = len(dataset.keys()) splits = read_json(args.split) assert args.split_id < len( splits), "split_id (got {}) exceeds {}".format( args.split_id, len(splits)) split = splits[args.split_id] train_keys = split['train_keys'] test_keys = split['test_keys'] print("# total videos {}. # train videos {}. # test videos {}".format( num_videos, len(train_keys), len(test_keys))) #### Set User Score Dataset #### userscoreset = h5py.File(args.userscore, 'r') print("Initialize model") model = DSRRL(in_dim=args.input_dim, hid_dim=args.hidden_dim, num_layers=args.num_layers, cell=args.rnn_cell) optimizer = torch.optim.Adam(model.parameters(), betas=(0.5, 0.999), lr=args.lr, weight_decay=args.weight_decay) if args.stepsize > 0: scheduler = lr_scheduler.StepLR(optimizer, step_size=args.stepsize, gamma=args.gamma) if args.resume: print("Loading checkpoint from '{}'".format(args.resume)) checkpoint = torch.load(args.resume) model.load_state_dict(checkpoint) else: start_epoch = 0 if use_gpu: model = nn.DataParallel(model).cuda() if args.evaluate: print("Evaluate only") evaluate(model, dataset, test_keys, use_gpu) return if args.dataset is None: print("==> Start training") start_time = time.time() model.train() baselines = {key: 0. for key in train_keys} # baseline rewards for videos reward_writers = {key: [] for key in train_keys } # record reward changes for each video for epoch in range(start_epoch, args.max_epoch): idxs = np.arange(len(train_keys)) np.random.shuffle(idxs) # shuffle indices for idx in idxs: key_parts = train_keys[idx].split('/') name, key = key_parts seq = dataset[name][key]['features'][ ...] # sequence of features, (seq_len, dim) seq = torch.from_numpy(seq).unsqueeze( 0) # input shape (1, seq_len, dim) if use_gpu: seq = seq.cuda() probs, out_feats, att_score = model( seq) # output shape (1, seq_len, 1) cost = args.beta * ( probs.mean() - 0.5)**2 # minimize summary length penalty term m = Bernoulli(probs) epis_rewards = [] for _ in range(args.num_episode): actions = m.sample() log_probs = m.log_prob(actions) reward = compute_reward(seq, actions, use_gpu=use_gpu) expected_reward = log_probs.mean() * ( reward - baselines[train_keys[idx]]) cost -= expected_reward epis_rewards.append(reward.item()) recon_loss = reconstruction_loss(seq, out_feats) spar_loss = sparsity_loss(att_score) total_loss = cost + recon_loss + spar_loss optimizer.zero_grad() total_loss.backward() optimizer.step() torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0) baselines[train_keys[ idx]] = 0.9 * baselines[train_keys[idx]] + 0.1 * np.mean( epis_rewards ) # update baseline reward via moving average reward_writers[train_keys[idx]].append(np.mean(epis_rewards)) epoch_reward = np.mean( [reward_writers[key][epoch] for key in train_keys]) #print("epoch {}/{}\t reward {}\t loss {}".format(epoch+1, args.max_epoch, epoch_reward, total_loss)) else: print("==> Start training") start_time = time.time() model.train() baselines = {key: 0. for key in train_keys} # baseline rewards for videos reward_writers = {key: [] for key in train_keys } # record reward changes for each video for epoch in range(start_epoch, args.max_epoch): idxs = np.arange(len(train_keys)) np.random.shuffle(idxs) # shuffle indices for idx in idxs: key = train_keys[idx] seq = dataset[key]['features'][ ...] # sequence of features, (seq_len, dim) seq = torch.from_numpy(seq).unsqueeze( 0) # input shape (1, seq_len, dim) if use_gpu: seq = seq.cuda() probs, out_feats, att_score = model( seq) # output shape (1, seq_len, 1) cost = args.beta * ( probs.mean() - 0.5)**2 # minimize summary length penalty term m = Bernoulli(probs) epis_rewards = [] for _ in range(args.num_episode): actions = m.sample() log_probs = m.log_prob(actions) reward = compute_reward(seq, actions, use_gpu=use_gpu) expected_reward = log_probs.mean() * (reward - baselines[key]) cost -= expected_reward epis_rewards.append(reward.item()) recon_loss = reconstruction_loss(seq, out_feats) spar_loss = sparsity_loss(att_score) total_loss = cost + recon_loss + spar_loss #print(cost.item(), recon_loss.item(), spar_loss.item()) optimizer.zero_grad() total_loss.backward() optimizer.step() torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0) baselines[key] = 0.9 * baselines[key] + 0.1 * np.mean( epis_rewards) # update baseline reward via moving average reward_writers[key].append(np.mean(epis_rewards)) epoch_reward = np.mean( [reward_writers[key][epoch] for key in train_keys]) #print("epoch {}/{}\t reward {}\t loss {}".format(epoch+1, args.max_epoch, epoch_reward, total_loss)) write_json(reward_writers, osp.join(args.save_dir, 'rewards.json')) evaluate(model, dataset, userscoreset, test_keys, use_gpu) elapsed = round(time.time() - start_time) elapsed = str(datetime.timedelta(seconds=elapsed)) print("Finished. Total elapsed time (h:m:s): {}".format(elapsed)) model_state_dict = model.module.state_dict( ) if use_gpu else model.state_dict() model_save_path = osp.join( args.save_dir, args.metric + '_model_epoch_' + str(args.max_epoch) + '_split_id_' + str(args.split_id) + '-' + str(args.rnn_cell) + '.pth.tar') save_checkpoint(model_state_dict, model_save_path) print("Model saved to {}".format(model_save_path))