def main():
args = parser.parse_args()
val_transform = ResNetUtils.M_Res50_val_transform()
useCuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
model = ResNetUtils.getM_Res50Model(eval=True, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
image_root_directory = os.path.join(path_vars.dataset_dir, 'frames')
save_root_directory = dir_utils.get_dir(os.path.join(path_vars.dataset_dir, 'features/Moments/ResNet50'))
image_directories = glob.glob(os.path.join(image_root_directory, '*/'))
for idx_dir, s_image_direcotry in enumerate(image_directories):
stem_name = s_image_direcotry.split(os.sep)[-2]
print '[{:02d} | {:02d}] {:s}'.format(idx_dir, len(image_directories), stem_name)
stem_name = stem_name.replace(' ', '_')
s_save_file = os.path.join(save_root_directory, '{:s}.npy'.format(stem_name))
s_dataset = SingleFrameDataset.SingleFrameDataset(s_image_direcotry, transform=val_transform)
s_dataset_loader = torch.utils.data.DataLoader(s_dataset, batch_size=1, shuffle=False, drop_last=False, num_workers=args.workers, pin_memory=True)
s_scores = []
pbar = progressbar.ProgressBar(max_value=len(s_dataset))
for i, s_image in enumerate(s_dataset_loader):
pbar.update(i)
if useCuda:
s_image = s_image.cuda()
input_image = Variable(s_image, volatile=True)
preds = model(input_image)
s_score = preds.data.cpu().numpy().squeeze(0)
s_scores.append(s_score)
s_scores = np.asarray(s_scores)
np.save(s_save_file, s_scores)
def create():
print("==========\nArgs:{}\n==========".format(args))
print("Goal: randomly split data for {} times, {:.1%} for training and the rest for testing".format(args.num_splits, args.train_percent))
print("Loading dataset from {}".format(args.dataset))
dataset = h5py.File(args.dataset, 'r')
keys = dataset.keys()
num_videos = len(keys)
num_train = int(math.ceil(num_videos * args.train_percent))
num_test = num_videos - num_train
print("Split breakdown: # total videos {}. # train videos {}. # test videos {}".format(num_videos, num_train, num_test))
splits = []
for split_idx in range(args.num_splits):
train_keys, test_keys = split_random(keys, num_videos, num_train)
splits.append({
'train_keys': train_keys,
'test_keys': test_keys,
})
if args.save_name is None:
save_name = dir_utils.get_stem(args.dataset)
else:
save_name = args.save_name
save_dir = dir_utils.get_dir(args.save_dir)
saveto = osp.join(save_dir, save_name + '.json')
write_json(splits, saveto)
print("Splits saved to {}".format(saveto))
dataset.close()
def vis_summaries(image_list, user_scores, save_target=None):
n_frames, n_users = user_scores.shape
min_frames = min(len(image_list), n_frames)
image_list = image_list[:n_frames]
user_scores = user_scores[:n_frames,:]
average_user_scores = np.mean(user_scores, axis=1)
n_plots = 2
pbar = progressbar.ProgressBar(max_value=min_frames)
for i, s_image_path in enumerate(image_list):
pbar.update(i)
s_image = default_loader(s_image_path)
ax = plt.subplot(n_plots,1,1)
plt.imshow(np.array(s_image))
ax.axis('off')
plt.title('F:{:08d} S:{:.04f}'.format(i, average_user_scores[i]))
plt.subplot(n_plots,1,2)
plt.plot(range(min_frames), average_user_scores, '-')
plt.annotate('', xy=(i, average_user_scores[i]), xytext=(i, average_user_scores[i]+0.1),
arrowprops=dict(facecolor='black', shrink=0.05),
)
if save_target is None:
plt.show()
else:
save_target = dir_utils.get_dir(save_target)
plt.savefig(os.path.join(save_target, os.path.basename(s_image_path)))
plt.close()
def main():
args = parser.parse_args()
val_transform = I3DUtils.simple_I3D_transform(224)
useCuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
model = I3DUtils.getK_I3D_RGBModel(eval=True,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
# Buffer = {}
# def _fc7_hook(self, input, output):
# Buffer['fc7'] = output.data.clone()
#
# model.full7.register_forward_hook(_fc7_hook)
image_root_directory = os.path.join(path_vars.dataset_dir, 'frames')
save_root_directory = dir_utils.get_dir(
os.path.join(path_vars.dataset_dir, 'features/Kinetics/I3D-test'))
chunk_size = 64
image_directories = glob.glob(os.path.join(image_root_directory, '*/'))
for idx_dir, s_image_direcotry in enumerate(image_directories):
stem_name = s_image_direcotry.split(os.sep)[-2]
print '[{:02d} | {:02d}] {:s}'.format(idx_dir, len(image_directories),
stem_name)
stem_name = stem_name.replace(' ', '_')
s_save_file = os.path.join(save_root_directory,
'{:s}.npy'.format(stem_name))
s_dataset = SingleVideoFrameDataset.VideoChunkDenseDataset(
s_image_direcotry, chunk_size=chunk_size, transform=val_transform)
s_dataset_loader = torch.utils.data.DataLoader(
s_dataset,
batch_size=1,
shuffle=False,
drop_last=False,
num_workers=args.workers,
pin_memory=True)
s_scores = []
pbar = progressbar.ProgressBar(max_value=len(s_dataset))
for i, s_image in enumerate(s_dataset_loader):
pbar.update(i)
s_image = s_image.permute(0, 2, 1, 3, 4)
if useCuda:
s_image = s_image.cuda()
input_image = Variable(s_image)
_, preds = model(input_image)
s_score = preds.data.cpu().numpy().squeeze(0)
for cnt in range(chunk_size):
s_scores.append(s_score)
# Padding
s_scores = np.asarray(s_scores)
# if s_scores.shape[0] < len(s_dataset.image_path_list):
# padding = np.asarray([s_scores[-1, :]] * (- s_scores.shape[0] + len(s_dataset.image_path_list)))
# s_scores = np.vstack((s_scores, padding))
np.save(s_save_file, s_scores)
def main():
args = parser.parse_args()
val_transform = BNInceptionUtils.get_val_transform()
Dataset = PathVars()
useCuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
model = BNInception.bninception(pretrained=True)
model = cuda_model.convertModel2Cuda(model,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
model = model.eval()
model_name = 'BNInception'
image_root_directory = Dataset.flow_directory
save_root_directory = dir_utils.get_dir(
os.path.join(Dataset.feature_directory, '{:s}'.format(model_name)))
image_directories = glob.glob(os.path.join(image_root_directory, '*/'))
for idx_dir, s_image_direcotry in enumerate(image_directories):
stem_name = s_image_direcotry.split(os.sep)[-2]
print '[{:02d} | {:02d}] {:s}'.format(idx_dir, len(image_directories),
stem_name)
# if stem_name != 'video_test_0001292':
# continue
stem_name = stem_name.replace(' ', '_')
s_image_list = glob.glob(os.path.join(s_image_direcotry, 'i_*.jpg'))
s_save_file = os.path.join(save_root_directory,
'{:s}.npy'.format(stem_name))
s_dataset = ImageDirectoryDataset.ImageListDataset(
s_image_list, transform=val_transform)
s_dataset_loader = torch.utils.data.DataLoader(s_dataset,
batch_size=1,
shuffle=False,
drop_last=False)
s_scores = []
pbar = progressbar.ProgressBar(max_value=len(s_dataset))
for i, s_image in enumerate(s_dataset_loader):
pbar.update(i)
if useCuda:
s_image = s_image.cuda()
s_image = s_image[:, [2, 1, 0], :, :]
s_image = s_image * 256
input_image = Variable(s_image)
preds, feature = model(input_image)
feature = F.avg_pool2d(feature, kernel_size=7)
s_score = feature.data.cpu().numpy().squeeze()
s_scores.append(s_score)
s_scores = np.asarray(s_scores)
np.save(s_save_file, s_scores)
def save_checkpoint(state, is_best_status, file_direcotry):
filename = 'checkpoint_{:s}.pth.tar'
file_direcotry = dir_utils.get_dir(file_direcotry)
if is_best_status['val_accuracy']:
file_path = os.path.join(file_direcotry, filename.format('val_accuracy'))
torch.save(state, file_path)
if is_best_status['val_loss']:
file_path = os.path.join(file_direcotry, filename.format('val_loss'))
torch.save(state, file_path)
if is_best_status['train_accuracy']:
file_path = os.path.join(file_direcotry, filename.format('val_loss'))
torch.save(state, file_path)
if is_best_status['train_loss']:
file_path = os.path.join(file_direcotry, filename.format('val_loss'))
torch.save(state, file_path)
def main():
PCA_info = pkl.load(open('pca_c3dd_fc7_val_annot_thumos14.pkl', 'rb'))
x_mean = PCA_info['x_mean']
U = PCA_info['U']
num_red_dim = 500
feature_directory = '/home/zwei/datasets/THUMOS14/features/c3dd-fc7'
feature_files = glob.glob(os.path.join(feature_directory, '*.mat'))
target_directory = dir_utils.get_dir(
'/home/zwei/datasets/THUMOS14/features/c3dd-fc7-red500')
print time.ctime(), 'start'
pbar = progressbar.ProgressBar(max_value=len(feature_files))
for feat_idx, s_feature_file in enumerate(feature_files):
s_file_stem = dir_utils.get_stem(s_feature_file)
output_file = os.path.join(target_directory,
'{:s}.npy'.format(s_file_stem))
pbar.update(feat_idx)
s_feature = scipy.io.loadmat(s_feature_file)['fc7']
s_feature_red = np.dot(s_feature - x_mean, U[:, :num_red_dim])
np.save(output_file, s_feature_red)
#
#
# if i == len(scores)-1:
# xtext, ytext = x[i, :]
# txt = ax.text(xtext, ytext, 'endF', fontsize=12, color='r')
# txts.append(txt)
return f, ax, sc
sample_rate = 5
pdefined_segs = getSegs.getSumMeShotBoundaris()
video_file_stem_names = dataset_pathvars.file_names
video_file_stem_names.sort()
totalF1 = 0
save_dir = dir_utils.get_dir('t-SNEVisualization')
full_video_features = []
full_video_labels = []
for video_idx, s_filename in enumerate(video_file_stem_names):
# s_filename = video_file_stem_names[0]
s_segments = pdefined_segs[s_filename].tolist()
video_features, user_labels, feature_sizes = SumMeMultiViewFeatureLoader.load_by_name(s_filename, doSoftmax=False)
avg_labels = np.mean(user_labels, axis=1)
video_features = video_features[::sample_rate, :]
avg_labels = avg_labels[::sample_rate]
# print "DB"
full_video_features.append(video_features)
full_video_labels.append(avg_labels)
if i == len(scores)-1:
xtext, ytext = x[i, :]
txt = ax.text(xtext, ytext, 'endF', fontsize=12, color='r')
txts.append(txt)
# return f, ax, sc, txts
sample_rate = 5
pdefined_segs = getSegs.getSumMeShotBoundaris()
video_file_stem_names = dataset_pathvars.file_names
video_file_stem_names.sort()
totalF1 = 0
save_dir = dir_utils.get_dir('t-SNEVisualization')
for video_idx, s_filename in enumerate(video_file_stem_names):
# if video_idx ==0:
# continue
print '{:d} | {:d} \t {:s}'.format(video_idx, len(video_file_stem_names), s_filename)
# s_filename = video_file_stem_names[0]
s_segments = pdefined_segs[s_filename].tolist()
video_features, user_labels, feature_sizes = SumMeMultiViewFeatureLoader.load_by_name(s_filename, doSoftmax=False)
avg_labels = np.mean(user_labels, axis=1)
video_features = video_features[::sample_rate, :]
avg_labels = avg_labels[::sample_rate]
# print "DB"
digits_proj = TSNE(random_state=0).fit_transform(video_features)
s_save_dir = dir_utils.get_dir(os.path.join(save_dir, s_filename+'-NonNorm-Image'))
image_dir = os.path.join('/home/zwei/datasets/SumMe/frames', s_filename)
pbar= progressbar.ProgressBar(max_value=video_features.shape[0])
def run(mode='rgb', batch_size=1,
load_model='/home/zwei/Dev/NetModules/ObjBank/pytorch-i3d/models/rgb_imagenet.pt'):
# setup dataset
# test_transforms = transforms.Compose([videotransforms.CenterCrop(224)])
val_transform = I3DUtils.simple_I3D_transform(224)
# dataset = Dataset(split, 'training', root, mode, test_transforms, num=-1, save_dir=save_dir)
# dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)
#
# val_dataset = Dataset(split, 'testing', root, mode, test_transforms, num=-1, save_dir=save_dir)
# val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)
#
# dataloaders = {'train': dataloader, 'val': val_dataloader}
# datasets = {'train': dataset, 'val': val_dataset}
# setup the model
if mode == 'flow':
i3d = InceptionI3d(400, in_channels=2)
else:
i3d = InceptionI3d(400, in_channels=3)
i3d.replace_logits(400)
i3d.load_state_dict(torch.load(load_model))
i3d.cuda()
# for phase in ['train', 'val']:
i3d.train(False) # Set model to evaluate mode
image_root_directory = os.path.join(path_vars.dataset_dir, 'frames')
save_root_directory = dir_utils.get_dir(os.path.join(path_vars.dataset_dir, 'features/I3D/I3D-feature'))
useCuda=True
chunk_size=64
image_directories = glob.glob(os.path.join(image_root_directory, '*/'))
for idx_dir, s_image_direcotry in enumerate(image_directories):
stem_name = s_image_direcotry.split(os.sep)[-2]
print '[{:02d} | {:02d}] {:s}'.format(idx_dir, len(image_directories), stem_name)
stem_name = stem_name.replace(' ', '_')
s_save_file = os.path.join(save_root_directory, '{:s}.npy'.format(stem_name))
s_dataset = SingleVideoFrameDataset.VideoChunkDenseDataset(s_image_direcotry, chunk_size=chunk_size, transform=val_transform)
s_dataset_loader = torch.utils.data.DataLoader(s_dataset, batch_size=1, shuffle=False, drop_last=False, num_workers=4, pin_memory=True)
s_scores = []
pbar = progressbar.ProgressBar(max_value=len(s_dataset))
for i, s_image in enumerate(s_dataset_loader):
pbar.update(i)
s_image = s_image.permute(0,2,1,3,4)
if useCuda:
s_image = s_image.cuda()
input_image = Variable(s_image)
print("InpuImageShape:")
print(input_image.shape)
features = i3d.extract_features(input_image)
s_score = features.data.cpu().numpy().squeeze(0)
print(s_score.shape)
for cnt in range(chunk_size):
s_scores.append(s_score)
# Padding
s_scores = np.asarray(s_scores)
# if s_scores.shape[0] < len(s_dataset.image_path_list):
# padding = np.asarray([s_scores[-1, :]] * (- s_scores.shape[0] + len(s_dataset.image_path_list)))
# s_scores = np.vstack((s_scores, padding))
np.save(s_save_file, s_scores)
import os
import sys
project_root = os.path.join(os.path.expanduser('~'), 'Dev/VideoSum')
sys.path.append(project_root)
import PyUtils.dir_utils as dir_utils
import glob
import subprocess
import path_vars
dataset_dir = path_vars.dataset_dir
src_video_dir = os.path.join(dataset_dir, 'videos')
dst_frame_dir = dir_utils.get_dir(os.path.join(dataset_dir, 'optflow'))
src_videopath_list = glob.glob(os.path.join(src_video_dir, '*.mp4'))
for i, s_src_videopath in enumerate(src_videopath_list):
if not os.path.isfile(s_src_videopath):
print "Not Exist: {:s}".format(s_src_videopath)
continue
video_stem = os.path.splitext(os.path.basename(s_src_videopath))[0]
print "[{:02d} | {:02d}], {:s}".format(i, len(src_videopath_list),
video_stem)
dst_framepath = dir_utils.get_dir(os.path.join(dst_frame_dir, video_stem))
exe = './denseFlow_gpu'
command = [
exe,
'-f="%s"' % s_src_videopath, '-b=20', '-t=1', '-d=0', '-s=1', '-h=0',
'-w=0',
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
script_name_stem = dir_utils.get_stem(__file__)
if args.resume is None:
save_directory = dir_utils.get_dir(os.path.join(project_root, 'ckpts', '{:s}'.format(args.dataset), '{:s}-{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-seqlen{:d}-{:s}-{:s}'.
format(script_name_stem, args.sufix, args.hassign_thres, args.alpha, args.hidden_dim, args.dropout, args.seq_len, 'L2', match_type[args.hmatch])))
else:
save_directory = args.resume
log_file = os.path.join(save_directory, 'log-{:s}.txt'.format(dir_utils.get_date_str()))
logger = log_utils.get_logger(log_file)
log_utils.print_config(vars(args), logger)
model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=args.dropout, n_enc_layers=2, output_classes=2)
logger.info("Number of Params\t{:d}".format(sum([p.data.nelement() for p in model.parameters()])))
logger.info('Saving logs to {:s}'.format(log_file))
if args.resume is not None:
ckpt_idx = args.fileid
ckpt_filename = os.path.join(args.resume, 'checkpoint_{:04d}.pth.tar'.format(ckpt_idx))
assert os.path.isfile(ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
logger.info("=> loading checkpoint '{}', current iou: {:.04f}".format(ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
train_dataset = cDataset(dataset_split='train', seq_length=args.seq_len, sample_rate=[4], rdOffset=True, rdDrop=True)
val_dataset =cDataset(dataset_split='val', seq_length=args.seq_len, sample_rate=[4], rdDrop=False, rdOffset=False)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model_optim = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()), lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min', patience=10)
cls_weights = torch.FloatTensor([0.05, 1.0]).cuda()
# cls_weights = None
widgets = ['Test: ', ' -- [ ', progressbar.Counter(), '|', str(len(train_dataloader)), ' ] ',
progressbar.Bar(), ' cls loss: ', progressbar.FormatLabel(''),
' loc loss: ', progressbar.FormatLabel(''),
' IoU : ', progressbar.FormatLabel(''),
' (', progressbar.ETA(), ' ) ']
# bar = progressbar.ProgressBar(max_value=step_per_epoch, widgets=widgets)
# bar.start()
for epoch in range(args.start_epoch, args.nof_epoch+args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
matched_IOU = AverageMeter()
true_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader), widgets=widgets)
pbar.start()
for i_batch, sample_batch in enumerate(train_dataloader):
# pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# gt_overlaps = Variable(sample_batch[4])
# seq_labels = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(feature_batch)
pred_positions = torch.stack([head_positions, tail_positions], dim=-1)
# pred_scores = F.sigmoid(cls_scores)
if args.hmatch:
assigned_scores, assigned_locations, total_valid, total_iou = h_match.Assign_Batch_v2(gt_positions, pred_positions, gt_valids, thres=args.hassign_thres)
else:
#FIXME: do it later!
assigned_scores, assigned_locations, total_valid, total_iou = f_match.Assign_Batch_v2(gt_positions, pred_positions, gt_valids, thres=args.hassign_thres)
# _, _, total_valid, total_iou = h_match.Assign_Batch_v2(gt_positions, pred_positions, gt_valids, thres=args.hassign_thres)
true_valid, true_iou = h_match.totalMatch_Batch(gt_positions, pred_positions, gt_valids)
assert true_valid == total_valid, 'WRONG'
if total_valid>0:
matched_IOU.update(total_iou / total_valid, total_valid)
true_IOU.update(true_iou/total_valid, total_valid)
assigned_scores = Variable(torch.LongTensor(assigned_scores),requires_grad=False)
# assigned_overlaps = Variable(torch.FloatTensor(assigned_overlaps), requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
# assigned_overlaps = assigned_overlaps.cuda()
# pred_scores = pred_scores.contiguous().view(-1)
# assigned_scores = assigned_scores.contiguous().view(-1)
# assigned_overlaps = assigned_overlaps.contiguous().view(-1)
# cls_loss = ClsLocLoss2_OneClsRegression(pred_scores, assigned_scores, assigned_overlaps)
cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores, weight=cls_weights)
if total_valid>0:
assigned_head_positions = assigned_locations[:,:,0]
assigned_head_positions = assigned_head_positions.contiguous().view(-1)
#
assigned_tail_positions = assigned_locations[:,:,1]
assigned_tail_positions = assigned_tail_positions.contiguous().view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1])
assigned_head_positions = torch.masked_select(assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1))
# if args.EMD:
assigned_head_positions = to_one_hot(assigned_head_positions, args.seq_len)
assigned_tail_positions = to_one_hot(assigned_tail_positions, args.seq_len)
prediction_head_loss = Simple_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True)
prediction_tail_loss = Simple_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True)
# else:
# prediction_head_loss = F.cross_entropy(head_pointer_probs, assigned_head_positions)
# prediction_tail_loss = F.cross_entropy(tail_pointer_probs, assigned_tail_positions)
loc_losses.update(prediction_head_loss.data.item() + prediction_tail_loss.data.item(),
total_valid)#FIXME
total_loss = args.alpha*(prediction_head_loss + prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
widgets[-8] = progressbar.FormatLabel('{:04.4f}'.format(cls_losses.avg))
widgets[-6] = progressbar.FormatLabel('{:04.4f}'.format(loc_losses.avg))
widgets[-4] = progressbar.FormatLabel('{:01.4f}'.format(matched_IOU.avg))
pbar.update(i_batch)
logger.info(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tAvg-matched_IOU:{:.4f}\t Avg-true-IOU:{:.4f}".format(
epoch,
model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, matched_IOU.avg, true_IOU.avg))
train_iou = matched_IOU.avg
optim_scheduler.step(total_losses.avg)
model.eval()
matched_IOU = AverageMeter()
pbar = progressbar.ProgressBar(max_value=len(val_dataloader))
for i_batch, sample_batch in enumerate(val_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# valid_indices = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch)
pred_positions = torch.stack([head_positions, tail_positions], dim=-1)
# assigned_scores, assigned_locations, total_valid, total_iou = h_match.Assign_Batch_eval(gt_positions, pred_positions, gt_valids, thres=args.hassign_thres) #FIXME
matched_valid, matched_iou = h_match.totalMatch_Batch(gt_positions, pred_positions, gt_valids)
if matched_valid>0:
matched_IOU.update(matched_iou / matched_valid, matched_valid)
logger.info(
"Val -- Epoch :{:06d}, LR: {:.6f},\tloc-Avg-matched_IOU:{:.4f}".format(
epoch,model_optim.param_groups[0]['lr'], matched_IOU.avg, ))
if epoch % 1 == 0 :
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss':total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'train-IOU':train_iou,
'IoU': matched_IOU.avg}, (epoch+1), file_direcotry=save_directory)
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
script_name_stem = dir_utils.get_stem(__file__)
save_directory = dir_utils.get_dir(os.path.join(project_root, 'ckpts', '{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-seqlen{:d}-{:s}-{:s}'.
format(script_name_stem, args.hassign_thres, args.alpha, args.hidden_dim, args.dropout, args.seq_len, loss_type[args.EMD], match_type[args.hmatch])))
log_file = os.path.join(save_directory, 'log-{:s}.txt'.format(dir_utils.get_date_str()))
logger = chinese_utils.get_logger(log_file)
chinese_utils.print_config(vars(args), logger)
model = BaseLSTMNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=args.dropout, n_enc_layers=2)
hassign_thres = args.hassign_thres
logger.info("Number of Params\t{:d}".format(sum([p.data.nelement() for p in model.parameters()])))
logger.info('Saving logs to {:s}'.format(log_file))
if args.resume is not None:
ckpt_idx = 48
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
logger.info("=> loading checkpoint '{}', current iou: {:.04f}".format(ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
train_dataset = MNIST(dataset_split='train')
val_dataset =MNIST(dataset_split='val')
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model_optim = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()), lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min', patience=10)
alpha=args.alpha
# cls_weights = torch.FloatTensor([0.05, 1.0]).cuda()
for epoch in range(args.start_epoch, args.nof_epoch+args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
labels = Variable(sample_batch[1])
if use_cuda:
feature_batch = feature_batch.cuda()
labels = labels.cuda()
# end_indices = end_indices.cuda()
pred_labels = model(feature_batch)
labels = labels.contiguous().view(-1)
pred_labels = pred_labels.contiguous().view(-1, pred_labels.size()[-1])
pred_probs = F.softmax(pred_labels, dim=1)[:, 1]
pred_probs[pred_probs>0.5] = 1
pred_probs[pred_probs<=0.5] = -1
n_positives = torch.sum(labels).item()
iou = torch.sum(pred_probs==labels.float()).item()*1. / n_positives
IOU.update(iou, 1.)
total_loss = F.cross_entropy(pred_labels, labels)
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
# cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
logger.info(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format(
epoch,
model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg))
optim_scheduler.step(total_losses.avg)
model.eval()
IOU = AverageMeter()
pbar = progressbar.ProgressBar(max_value=len(val_dataloader))
for i_batch, sample_batch in enumerate(val_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
labels = Variable(sample_batch[1])
if use_cuda:
feature_batch = feature_batch.cuda()
labels = labels.cuda()
labels = labels.contiguous().view(-1)
pred_labels = model(feature_batch)
pred_labels = pred_labels.contiguous().view(-1, pred_labels.size()[-1])
pred_probs = F.softmax(pred_labels, dim=1)[:, 1]
n_positives = torch.sum(labels).item()
pred_probs[pred_probs > 0.5] = 1
pred_probs[pred_probs <= 0.5] = -1
iou = torch.sum(pred_probs == labels.float()).item() * 1. / n_positives
IOU.update(iou, 1.)
logger.info(
"Val -- Epoch :{:06d}, LR: {:.6f},\tloc-Avg-IOU:{:.4f}".format(
epoch,model_optim.param_groups[0]['lr'], IOU.avg, ))
if epoch % 1 == 0:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss':total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'IoU': IOU.avg}, (epoch+1), file_direcotry=save_directory)
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
script_name_stem = dir_utils.get_stem(__file__)
save_directory = dir_utils.get_dir(
os.path.join(
project_root, 'ckpts',
'{:s}-{:s}-{:s}-split-{:d}-decoderatio-{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}'
.format(script_name_stem, args.dataset, args.eval_metrics,
args.split, args.decode_ratio, args.alpha, args.hidden_dim,
args.dropout)))
log_file = os.path.join(save_directory,
'log-{:s}.txt'.format(dir_utils.get_date_str()))
logger = log_utils.get_logger(log_file)
log_utils.print_config(vars(args), logger)
# get train/val split
if args.dataset == 'SumMe':
train_val_perms = np.arange(25)
elif args.dataset == 'TVSum':
train_val_perms = np.arange(50)
# fixed permutation
random.Random(0).shuffle(train_val_perms)
train_val_perms = train_val_perms.reshape([5, -1])
train_perms = np.delete(train_val_perms, args.split, 0).reshape([-1])
val_perms = train_val_perms[args.split]
logger.info(" training split: " + str(train_perms))
logger.info(" val split: " + str(val_perms))
if args.location == 'home':
data_path = os.path.join(os.path.expanduser('~'), 'datasets')
else:
data_path = os.path.join('/nfs/%s/boyu/SDN' % (args.location),
'datasets')
train_dataset = vsTVSum_Loader3_c3dd_segment.cDataset(
dataset_name=args.dataset,
split='train',
decode_ratio=args.decode_ratio,
train_val_perms=train_perms,
data_path=data_path)
max_input_len = train_dataset.max_input_len
maximum_outputs = int(args.decode_ratio * max_input_len)
val_dataset = vsTVSum_Loader3_c3dd_segment.cDataset(
dataset_name=args.dataset,
split='val',
decode_ratio=args.decode_ratio,
train_val_perms=val_perms,
data_path=data_path)
train_evaluator = Evaluator.Evaluator(dataset_name=args.dataset,
split='tr',
max_input_len=max_input_len,
maximum_outputs=maximum_outputs,
sum_budget=0.15,
train_val_perms=train_perms,
eval_metrics=args.eval_metrics,
data_path=data_path)
val_evaluator = Evaluator.Evaluator(dataset_name=args.dataset,
split='val',
max_input_len=max_input_len,
maximum_outputs=maximum_outputs,
sum_budget=0.15,
train_val_perms=val_perms,
eval_metrics=args.eval_metrics,
data_path=data_path)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model = PointerNetwork(input_dim=args.input_dim,
embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim,
max_decoding_len=maximum_outputs,
dropout=args.dropout,
n_enc_layers=2,
output_classes=1)
# hassign_thres = args.hassign_thres
logger.info("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
logger.info('Saving logs to {:s}'.format(log_file))
if args.resume is not None:
ckpt_idx = 48
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(
ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename,
map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
logger.info("=> loading checkpoint '{}', current iou: {:.04f}".format(
ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
model_optim = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim,
'min',
patience=10)
alpha = args.alpha
# cls_weights = torch.FloatTensor([0.2, 1.0]).cuda()
# if args.set_cls_weight:
# cls_weights = torch.FloatTensor([1.*train_dataset.n_positive_train_samples/train_dataset.n_total_train_samples, args.cls_pos_weight]).cuda()
# else:
# cls_weights = torch.FloatTensor([0.5, 0.5]).cuda()
# logger.info(" total: {:d}, total pos: {:d}".format(train_dataset.n_total_train_samples, train_dataset.n_positive_train_samples))
# logger.info(" classify weight: " + str(cls_weights[0]) + str(cls_weights[1]))
for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch):
total_losses = AverageMeter()
# loc_losses = AverageMeter()
pointer_losses = AverageMeter()
rgs_losses = AverageMeter()
Accuracy = AverageMeter()
# IOU = AverageMeter()
# ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
pointer_indices = Variable(sample_batch[1])
pointer_scores = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# seq_labels = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
pointer_indices = pointer_indices.cuda()
pointer_scores = pointer_scores.cuda()
gt_positions = pointer_indices
gt_scores = pointer_scores
pointer_probs, pointer_positions, cls_scores, _ = model(
feature_batch)
pred_positions = pointer_positions
cls_scores = cls_scores.contiguous().squeeze(2)
# print(pointer_probs.size())
# print(gt_positions.size())
pointer_loss = F.cross_entropy(pointer_probs.permute(0, 2, 1),
gt_positions)
rgs_loss = F.mse_loss(cls_scores, gt_scores)
total_loss = alpha * pointer_loss + rgs_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
pointer_losses.update(pointer_loss.data.item(),
feature_batch.size(0))
rgs_losses.update(rgs_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
logger.info(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t pointer-loss:{:.4f}, \tregress-loss:{:.4f}\tcls-Accuracy:{:.4f}"
.format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg,
pointer_losses.avg, rgs_losses.avg, Accuracy.avg))
optim_scheduler.step(total_losses.avg)
model.eval()
pointer_losses = AverageMeter()
rgs_losses = AverageMeter()
pbar = progressbar.ProgressBar(max_value=len(val_dataloader))
for i_batch, sample_batch in enumerate(val_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
pointer_indices = Variable(sample_batch[1])
pointer_scores = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# valid_indices = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
pointer_indices = pointer_indices.cuda()
pointer_scores = pointer_scores.cuda()
gt_positions = pointer_indices
gt_scores = pointer_scores
pointer_probs, pointer_positions, cls_scores, _ = model(
feature_batch)
pred_positions = pointer_positions
cls_scores = cls_scores.contiguous().squeeze(2)
pointer_loss = F.cross_entropy(pointer_probs.permute(0, 2, 1),
gt_positions)
rgs_loss = F.mse_loss(cls_scores, gt_scores)
pointer_losses.update(pointer_loss.data.item(),
feature_batch.size(0))
rgs_losses.update(rgs_loss.data.item(), feature_batch.size(0))
train_F1s = train_evaluator.Evaluate(model)
val_F1s = val_evaluator.Evaluate(model)
logger.info("Train -- Epoch :{:06d},\tF1s:{:.4f}".format(
epoch, train_F1s))
logger.info(
"Val -- Epoch :{:06d},\t pointer-loss:{:.4f}, \tregress-loss:{:.4f}, \tF1s{:.4f}"
.format(epoch, pointer_losses.avg, rgs_losses.avg, val_F1s))
if epoch % 1 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss': total_losses.avg,
'pointer_loss': pointer_losses.avg,
'rgs_loss': rgs_losses.avg,
'val_F1s': val_F1s
}, (epoch + 1),
file_direcotry=save_directory)
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
script_name_stem = dir_utils.get_stem(__file__)
save_directory = dir_utils.get_dir(
os.path.join(
project_root, 'ckpts',
'Delete-{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-seqlen{:d}-{:s}-{:s}'
.format(script_name_stem, args.hassign_thres, args.alpha,
args.hidden_dim, args.dropout, args.seq_len,
loss_type[args.EMD], match_type[args.hmatch])))
log_file = os.path.join(save_directory,
'log-{:s}.txt'.format(dir_utils.get_date_str()))
logger = chinese_utils.get_logger(log_file)
chinese_utils.print_config(vars(args), logger)
model = PointerNetwork(input_dim=args.input_dim,
embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim,
max_decoding_len=args.net_outputs,
dropout=args.dropout,
n_enc_layers=2)
hassign_thres = args.hassign_thres
logger.info("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
logger.info('Saving logs to {:s}'.format(log_file))
if args.resume is not None:
ckpt_idx = 48
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(
ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename,
map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
logger.info("=> loading checkpoint '{}', current iou: {:.04f}".format(
ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
train_dataset = cDataset(dataset_split='train')
val_dataset = cDataset(dataset_split='val')
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model_optim = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim,
'min',
patience=10)
alpha = args.alpha
# cls_weights = torch.FloatTensor([0.05, 1.0]).cuda()
for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# seq_labels = Variable(sample_batch[4])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch)
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
if args.hmatch:
assigned_scores, assigned_locations, total_valid, total_iou = h_match.Assign_Batch_v2(
gt_positions,
pred_positions,
gt_valids,
thres=hassign_thres)
IOU.update(total_iou / total_valid, total_valid)
else:
assigned_scores, assigned_locations = f_match.Assign_Batch(
gt_positions,
pred_positions,
gt_valids,
thres=hassign_thres)
_, _, total_valid, total_iou = h_match.Assign_Batch_v2(
gt_positions,
pred_positions,
gt_valids,
thres=hassign_thres)
IOU.update(total_iou / total_valid, total_valid)
assigned_scores = Variable(torch.LongTensor(assigned_scores),
requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations),
requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1,
cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores)
if total_valid > 0:
assigned_head_positions = assigned_locations[:, :, 0]
assigned_head_positions = assigned_head_positions.contiguous(
).view(-1)
#
assigned_tail_positions = assigned_locations[:, :, 1]
assigned_tail_positions = assigned_tail_positions.contiguous(
).view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(
-1,
head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(
-1,
tail_pointer_probs.size()[-1])
assigned_head_positions = torch.masked_select(
assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(
assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(
head_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(
tail_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
if args.EMD:
assigned_head_positions = to_one_hot(
assigned_head_positions, args.seq_len)
assigned_tail_positions = to_one_hot(
assigned_tail_positions, args.seq_len)
prediction_head_loss = EMD_L2(head_pointer_probs,
assigned_head_positions,
needSoftMax=True)
prediction_tail_loss = EMD_L2(tail_pointer_probs,
assigned_tail_positions,
needSoftMax=True)
else:
prediction_head_loss = F.cross_entropy(
head_pointer_probs, assigned_head_positions)
prediction_tail_loss = F.cross_entropy(
tail_pointer_probs, assigned_tail_positions)
loc_losses.update(
prediction_head_loss.data.item() +
prediction_tail_loss.data.item(), feature_batch.size(0))
total_loss = alpha * (prediction_head_loss +
prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
logger.info(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}"
.format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg,
cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg,
ordered_IOU.avg))
optim_scheduler.step(total_losses.avg)
model.eval()
IOU = AverageMeter()
pbar = progressbar.ProgressBar(max_value=len(val_dataloader))
for i_batch, sample_batch in enumerate(val_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# valid_indices = Variable(sample_batch[4])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch
) #Update: compared to the previous version, we now update the matching rules
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
#TODO: should NOT change here for evaluation!
assigned_scores, assigned_locations, total_valid, total_iou = h_match.Assign_Batch_eval(
gt_positions, pred_positions, gt_valids, thres=hassign_thres)
IOU.update(total_iou / total_valid, total_valid)
logger.info(
"Val -- Epoch :{:06d}, LR: {:.6f},\tloc-Avg-IOU:{:.4f}".format(
epoch,
model_optim.param_groups[0]['lr'],
IOU.avg,
))
if epoch % 1 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss': total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'IoU': IOU.avg
}, (epoch + 1),
file_direcotry=save_directory)
c=palette[(scores * (n_colors - 1)).astype(np.int)])
ax.axis('equal')
ax.set_xlim([-80, 80])
ax.set_ylim([-80, 80])
# ax.xlim(-80, 80)
# ax.ylim(-80, 80)
return
sample_rate = 5
pdefined_segs = getSegs.getSumMeShotBoundaris()
video_file_stem_names = dataset_pathvars.file_names
video_file_stem_names.sort()
totalF1 = 0
save_dir = dir_utils.get_dir('t-SNEVisualization')
for video_idx, s_filename in enumerate(video_file_stem_names):
print '[{:d} | {:d}], {:s}'.format(video_idx, len(video_file_stem_names),
s_filename)
video_features, user_labels, feature_sizes = SumMeMultiViewFeatureLoader.load_by_name(
s_filename, doSoftmax=False)
avg_labels = np.mean(user_labels, axis=1)
video_features = video_features[::sample_rate, :]
avg_labels = avg_labels[::sample_rate]
feature_types = ['ImageNet', 'Kinectics', 'Places', 'Moments']
feature_sizes = np.cumsum(np.asarray(feature_sizes))
s_save_dir = dir_utils.get_dir(os.path.join(save_dir,
s_filename + '-Sep4'))
pbar = progressbar.ProgressBar(max_value=video_features.shape[0])
print("Using ALL {:d} GPUs".format(device_count))
model = nn.DataParallel(
Inception, device_ids=[i for i in range(device_count)]).cuda()
else:
print("Using GPUs: {:s}".format(args.gpu_id))
device_ids = [int(x) for x in args.gpu_id]
model = nn.DataParallel(Inception, device_ids=device_ids).cuda()
else:
torch.cuda.set_device(int(args.gpu_id))
Inception.cuda()
cudnn.benchmark = True
if args.pick is not None:
save_dir = dir_utils.get_dir(
os.path.join(SumMe_pathvars.dataset_dir,
'Annotation_{:s}_{:.2d}'.format(args.network, args.pick)))
else:
save_dir = dir_utils.get_dir(
os.path.join(SumMe_pathvars.dataset_dir,
'Annotation_{:s}'.format(args.network)))
for video_idx, s_frame_folder in enumerate(frame_folder_list):
frame_feature = []
s_video_stem = (s_frame_folder.split(os.sep))[-2]
print "[{:02d} | {:02d}]Processing {:s}".format(video_idx,
len(frame_folder_list),
s_video_stem)
label_file = os.path.join(SumMe_pathvars.ground_truth_dir,
'{:s}.mat'.format(s_video_stem))
gt_mat = LoadLabels.getRawData(label_file)
import scipy.io as sio
import sys, os
from PyUtils import dir_utils
import numpy as np
import glob
import progressbar
feature_directory = '/home/zwei/datasets/THUMOS14/features/c3d_feat_dense'
target_directory = dir_utils.get_dir(
'/home/zwei/datasets/THUMOS14/features/c3d-dense')
feature_lists = glob.glob(os.path.join(feature_directory, '*.mat'))
# video_name = 'video_validation_0000940.mat'
pbar = progressbar.ProgressBar(max_value=len(feature_lists))
for file_id, s_feature_path in enumerate(feature_lists):
pbar.update(file_id)
# s_feature_name = os.path.basename(s_feature_path)
s_feature_stem = dir_utils.get_stem(s_feature_path)
# feature_path = os.path.join(, video_name)
s_full_feature = sio.loadmat(s_feature_path)['relu6']
np.save(os.path.join(target_directory, '{:s}.npy'.format(s_feature_stem)),
s_full_feature)
print("DEBUG")
NUM_TRAIN = 50000
NUM_VAL = 5000
NOISE_DIM = 100
batch_size = 1
# mnist_train = dset.MNIST('./data/MNIST_data', train=True, download=True,
# transform=T.ToTensor())
# loader_train = DataLoader(mnist_train, batch_size=batch_size,
# sampler=ChunkSampler(NUM_TRAIN, 0))
mnist_val = dset.MNIST('./data/MNIST_data',
train=False,
download=True,
transform=T.ToTensor())
loader_val = DataLoader(mnist_val, batch_size=batch_size)
image_save_dir = dir_utils.get_dir('./data/MNIST_data/images')
pbar = progressbar.ProgressBar(max_value=len(loader_val))
for s_idx, s_data in enumerate(loader_val):
pbar.update(s_idx)
s_digit = s_data[0]
s_label = s_data[1]
s_digit = s_digit.view(batch_size, 784).numpy()
save_name = '{:d}-{:06d}.png'.format(s_label.item(), s_idx)
save_path = os.path.join(image_save_dir, save_name)
show_images(s_digit, save_path)
# print("DB")
# imgs = loader_train.__iter__().next()[0].view(batch_size, 784).numpy().squeeze()
# show_images(imgs)
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
script_name_stem = dir_utils.get_stem(__file__)
save_directory = dir_utils.get_dir(
os.path.join(
project_root, 'ckpts',
'{:s}-{:s}-{:s}-split-{:d}-claweight-{:s}-{:.1f}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-seqlen{:d}-samplerate-{:d}-{:s}-{:s}'
.format(script_name_stem, args.dataset, args.eval_metrics,
args.split, str(args.set_cls_weight), args.cls_pos_weight,
args.hassign_thres, args.alpha, args.hidden_dim,
args.dropout, args.seq_len, args.sample_rate,
loss_type[args.EMD], match_type[args.hmatch])))
log_file = os.path.join(save_directory,
'log-{:s}.txt'.format(dir_utils.get_date_str()))
logger = log_utils.get_logger(log_file)
log_utils.print_config(vars(args), logger)
model = PointerNetwork(input_dim=args.input_dim,
embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim,
max_decoding_len=args.net_outputs,
dropout=args.dropout,
n_enc_layers=2,
output_classes=2)
hassign_thres = args.hassign_thres
logger.info("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
logger.info('Saving logs to {:s}'.format(log_file))
if args.resume is not None:
ckpt_idx = 48
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(
ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename,
map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
logger.info("=> loading checkpoint '{}', current iou: {:.04f}".format(
ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
# get train/val split
if args.dataset == 'SumMe':
train_val_test_perms = np.arange(25)
elif args.dataset == 'TVSum':
train_val_test_perms = np.arange(50)
# fixed permutation
random.Random(0).shuffle(train_val_test_perms)
train_val_test_perms = train_val_test_perms.reshape([5, -1])
train_val_perms = np.delete(train_val_test_perms, args.split,
0).reshape([-1])
train_perms = train_val_perms[:17]
val_perms = train_val_perms[17:]
test_perms = train_val_test_perms[args.split]
logger.info(" training split: " + str(train_perms))
logger.info(" val split: " + str(val_perms))
logger.info(" test split: " + str(test_perms))
if args.location == 'home':
data_path = os.path.join(os.path.expanduser('~'), 'datasets')
else:
data_path = os.path.join('/nfs/%s/boyu/SDN' % (args.location),
'datasets')
train_dataset = vsSumLoader3_c3dd.cDataset(dataset_name=args.dataset,
split='train',
seq_length=args.seq_len,
overlap=0.9,
sample_rate=[args.sample_rate],
train_val_perms=train_perms,
data_path=data_path)
val_evaluator = Evaluator.Evaluator(dataset_name=args.dataset,
split='val',
seq_length=args.seq_len,
overlap=0.9,
sample_rate=[args.sample_rate],
sum_budget=0.15,
train_val_perms=val_perms,
eval_metrics=args.eval_metrics,
data_path=data_path)
test_evaluator = Evaluator.Evaluator(dataset_name=args.dataset,
split='test',
seq_length=args.seq_len,
overlap=0.9,
sample_rate=[args.sample_rate],
sum_budget=0.15,
train_val_perms=test_perms,
eval_metrics=args.eval_metrics,
data_path=data_path)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
# val_dataloader = DataLoader(val_dataset,
# batch_size=args.batch_size,
# shuffle=False,
# num_workers=4)
model_optim = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim,
'min',
patience=10)
alpha = args.alpha
# cls_weights = torch.FloatTensor([0.2, 1.0]).cuda()
if args.set_cls_weight:
cls_weights = torch.FloatTensor([
1. * train_dataset.n_positive_train_samples /
train_dataset.n_total_train_samples, args.cls_pos_weight
]).cuda()
else:
cls_weights = torch.FloatTensor([0.5, 0.5]).cuda()
logger.info(" total: {:d}, total pos: {:d}".format(
train_dataset.n_total_train_samples,
train_dataset.n_positive_train_samples))
logger.info(" classify weight: " + str(cls_weights[0]) +
str(cls_weights[1]))
for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# seq_labels = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch)
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
if args.hmatch:
assigned_scores, assigned_locations, total_valid, total_iou = h_match.Assign_Batch_v2(
gt_positions,
pred_positions,
gt_valids,
thres=hassign_thres)
else:
assigned_scores, assigned_locations = f_match.Assign_Batch(
gt_positions,
pred_positions,
gt_valids,
thres=hassign_thres)
_, _, total_valid, total_iou = h_match.Assign_Batch_v2(
gt_positions,
pred_positions,
gt_valids,
thres=hassign_thres)
if total_valid > 0:
IOU.update(total_iou / total_valid, total_valid)
assigned_scores = Variable(torch.LongTensor(assigned_scores),
requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations),
requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1,
cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores,
assigned_scores,
weight=cls_weights)
if total_valid > 0:
assigned_head_positions = assigned_locations[:, :, 0]
assigned_head_positions = assigned_head_positions.contiguous(
).view(-1)
#
assigned_tail_positions = assigned_locations[:, :, 1]
assigned_tail_positions = assigned_tail_positions.contiguous(
).view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(
-1,
head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(
-1,
tail_pointer_probs.size()[-1])
assigned_head_positions = torch.masked_select(
assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(
assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(
head_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(
tail_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
if args.EMD:
assigned_head_positions = to_one_hot(
assigned_head_positions, args.seq_len)
assigned_tail_positions = to_one_hot(
assigned_tail_positions, args.seq_len)
prediction_head_loss = EMD_L2(head_pointer_probs,
assigned_head_positions,
needSoftMax=True)
prediction_tail_loss = EMD_L2(tail_pointer_probs,
assigned_tail_positions,
needSoftMax=True)
else:
prediction_head_loss = F.cross_entropy(
head_pointer_probs, assigned_head_positions)
prediction_tail_loss = F.cross_entropy(
tail_pointer_probs, assigned_tail_positions)
loc_losses.update(
prediction_head_loss.data.item() +
prediction_tail_loss.data.item(), feature_batch.size(0))
total_loss = alpha * (prediction_head_loss +
prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
logger.info(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}"
.format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg,
cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg,
ordered_IOU.avg))
optim_scheduler.step(total_losses.avg)
model.eval()
# IOU = AverageMeter()
# pbar = progressbar.ProgressBar(max_value=len(val_evaluator))
# for i_batch, sample_batch in enumerate(val_dataloader):
# pbar.update(i_batch)
# feature_batch = Variable(sample_batch[0])
# start_indices = Variable(sample_batch[1])
# end_indices = Variable(sample_batch[2])
# gt_valids = Variable(sample_batch[3])
# # valid_indices = Variable(sample_batch[3])
# if use_cuda:
# feature_batch = feature_batch.cuda()
# start_indices = start_indices.cuda()
# end_indices = end_indices.cuda()
# gt_positions = torch.stack([start_indices, end_indices], dim=-1)
# head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
# feature_batch)#Update: compared to the previous version, we now update the matching rules
# pred_positions = torch.stack([head_positions, tail_positions], dim=-1)
# pred_scores = cls_scores[:, :, -1]
# #TODO: should NOT change here for evaluation!
# assigned_scores, assigned_locations, total_valid, total_iou = h_match.Assign_Batch_v2(gt_positions, pred_positions, gt_valids, thres=hassign_thres)
# if total_valid>0:
# IOU.update(total_iou / total_valid, total_valid)
val_F1s = val_evaluator.Evaluate(model)
test_F1s = test_evaluator.Evaluate(model)
logger.info("Val -- Epoch :{:06d}, LR: {:.6f},\tF1s:{:.4f}".format(
epoch, model_optim.param_groups[0]['lr'], val_F1s))
logger.info("Test -- Epoch :{:06d},\tF1s:{:.4f}".format(
epoch, test_F1s))
if epoch % 1 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss': total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'IoU': IOU.avg,
'val_F1s': val_F1s,
'test_F1s': test_F1s
}, (epoch + 1),
file_direcotry=save_directory)
)
if save_target is None:
plt.show()
else:
save_target = dir_utils.get_dir(save_target)
plt.savefig(
os.path.join(save_target, '{:08d}.jpg'.format(image_idx)))
plt.close()
image_idx += 1
if __name__ == '__main__':
frame_dir = os.path.join(path_vars.dataset_dir, 'frames')
tvsum_gt = LoadLabels.load_annotations()
save_directory = dir_utils.get_dir(
os.path.join(path_vars.dataset_dir, 'visualizations'))
image_foramt = 'jpg'
for video_idx, s_video_name in enumerate(tvsum_gt):
# s_name = os.path.splitext(os.path.basename(s_annotation_file))[0]
print "[{:d} | {:d}]".format(video_idx, len(tvsum_gt))
s_image_directory = os.path.join(frame_dir, s_video_name)
image_pathlist = glob.glob(
os.path.join(s_image_directory, '*.{:s}'.format(image_foramt)))
image_pathlist.sort()
s_video_info = tvsum_gt[s_video_name]
s_FPS = s_video_info['video_nframes'] / s_video_info[
'video_seg_seconds']
user_scores = s_video_info['video_user_scores']
s_save_directory = os.path.join(save_directory, s_video_name)
vis_summaries(image_pathlist,
def save_checkpoint(state, epoch, file_direcotry):
filename = 'checkpoint_{:04d}.pth.tar'
file_direcotry = dir_utils.get_dir(file_direcotry)
file_path = os.path.join(file_direcotry, filename.format(epoch))
torch.save(state, file_path)
def infer_segmentation_from_scores(user_score):
segment_list = []
for i in range(len(user_score) - 1):
if user_score[i] != user_score[i + 1]:
segment_list.append(i)
return segment_list
feature_directory = '/home/zwei/datasets/SumMe/Annotation_InceptionV3'
feature_filelist = glob.glob(
os.path.join(feature_directory, '*.{:s}'.format('mat')))
feature_filelist.sort()
target_directory = dir_utils.get_dir(
'/home/zwei/datasets/SumMe/Annotation_InceptionV3_SegV1')
pbar = progressbar.ProgressBar(max_value=len(feature_filelist))
for file_idx, s_feaure_file in enumerate(feature_filelist):
pbar.update(file_idx)
mat_content = scipy.io.loadmat(s_feaure_file)
features = mat_content['features']
# features = features[::5, :]
K = np.dot(features, features.T)
# This 1.85 magic number is from the Summe dataset
m = K.shape[0] / (1.85 * 30)
cps, scores = cpd_auto.cpd_auto(K, int(m), 1, verbose=False)
mat_content['segs'] = cps
scipy.io.savemat(
os.path.join(target_directory, os.path.basename(s_feaure_file)),
mat_content)