def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
ckpt_savedir = 'lstm2heads_proposal_c3d_s4_ckpt'
# Pretty print the run args
pp.pprint(vars(args))
model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=0.5)
print("Number of Params\t{:d}".format(sum([p.data.nelement() for p in model.parameters()])))
if args.resume is not None:
ckpt_idx = args.fileid
ckpt_filename = os.path.join(args.resume, 'checkpoint_{:04d}.pth.tar'.format(ckpt_idx))
assert os.path.isfile(ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
print("=> loading checkpoint '{}', current iou: {:.04f}".format(ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
train_dataset = THUMOST14(seq_length=args.seq_len, overlap=0.9, sample_rate=4)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
# val_dataloader = DataLoader(val_dataset,
# batch_size=args.batch_size,
# shuffle=True,
# num_workers=4)
model_optim = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()), lr=float(args.lr))
alpha=0.1
for epoch in range(args.start_epoch, args.nof_epoch+args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
valid_indices = Variable(sample_batch[3])
# gt_index_batch = sample_batch[1].numpy()
# score_batch = Variable(sample_batch[2])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores = model(feature_batch)
pred_positions = torch.stack([head_positions, tail_positions], dim=-1)
assigned_scores, assigned_locations = h_assign.Assign_Batch(gt_positions, pred_positions, valid_indices, thres=0.5)
# if np.sum(assigned_scores) > 1:
# print("DEBUG")
# correct_predictions = np.sum(assigned_scores[:,:args.n_outputs])
# cls_rate = correct_predictions*1./np.sum(assigned_scores)
if np.sum(assigned_scores)>=1:
iou_rate, effective_positives = Metrics.get_avg_iou2(np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)),
np.reshape(assigned_locations, (-1, 2)), np.reshape(assigned_scores,
assigned_scores.shape[
0] *
assigned_scores.shape[
1]))
# _, top_assigned_locations = h_assign_proposal.Assign_Batch(gt_positions, pred_positions[:, : args.n_outputs, :], valid_indices, thres=0.5)
#
# ordered_iou_rate = Metrics.get_avg_iou(np.reshape(pred_positions[:,:args.n_outputs,:].data.cpu().numpy(), (-1, 2)),
# np.reshape(top_assigned_locations, (-1, 2)))
# Accuracy.update(cls_rate, np.sum(assigned_scores))
# iou_rate = Metrics.get_avg_iou(np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(gt_positions.data.cpu().numpy(), (-1, 2)))
IOU.update(iou_rate/(effective_positives), effective_positives)
# ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs)
# n_effective_batches += 1
assigned_scores = Variable(torch.LongTensor(assigned_scores),requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores)
assigned_head_positions = assigned_locations[:,:,0]
assigned_head_positions = assigned_head_positions.contiguous().view(-1)
#
assigned_tail_positions = assigned_locations[:,:,1]
assigned_tail_positions = assigned_tail_positions.contiguous().view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1])
# start_indices = start_indices.contiguous().view(-1)
# end_indices = end_indices.contiguous().view(-1)
# with case instances....
prediction_head_loss = F.cross_entropy((head_pointer_probs), assigned_head_positions, reduce=False)
prediction_head_loss = torch.mean(prediction_head_loss * assigned_scores.float())
prediction_tail_loss = F.cross_entropy((tail_pointer_probs), assigned_tail_positions, reduce=False)
prediction_tail_loss = torch.mean(prediction_tail_loss * assigned_scores.float())
total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), 1.)
model_optim.step()
cls_losses.update(cls_loss.data[0], feature_batch.size(0))
loc_losses.update(prediction_head_loss.data[0] + prediction_tail_loss.data[0], feature_batch.size(0))
total_losses.update(total_loss.data[0], feature_batch.size(0))
print(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format(
epoch,
model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg))
if epoch % 1 == 0:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss':total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'IoU': IOU.avg}, (epoch+1), file_direcotry=ckpt_savedir)
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
# Pretty print the run args
pp.pprint(vars(args))
model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=args.dropout, n_enc_layers=2)
hassign_thres = args.hassign_thres
print("Number of Params\t{:d}".format(sum([p.data.nelement() for p in model.parameters()])))
script_name_stem = dir_utils.get_stem(__file__)
save_directory = os.path.join(project_root, 'ckpts', '{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-seqlen{:d}-ckpt'.
format(script_name_stem, hassign_thres, args.alpha, args.hidden_dim, args.dropout, args.seq_len))
print("Save ckpt to {:s}".format(save_directory))
if args.resume is not None:
ckpt_idx = 3
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
print("=> loading checkpoint '{}', current iou: {:.04f}".format(ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
train_dataset = cDataset(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='train', rdDrop=True, rdOffset=True)
val_dataset = cDataset(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='val', rdDrop=False, rdOffset=False)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model_optim = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()), lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min', patience=20)
alpha=args.alpha
cls_weights = torch.FloatTensor([0.05, 1.0]).cuda()
for epoch in range(args.start_epoch, args.nof_epoch+args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(feature_batch)
pred_positions = torch.stack([head_positions, tail_positions], dim=-1)
assigned_scores, assigned_locations = h_assign.Assign_Batch(gt_positions, pred_positions, gt_valids, thres=hassign_thres)
# if np.sum(assigned_scores) > 1:
# print("DEBUG")
# correct_predictions = np.sum(assigned_scores[:,:args.n_outputs])
# cls_rate = correct_predictions*1./np.sum(assigned_scores)
if np.sum(assigned_scores)>=1:
iou_rate, effective_positives = Metrics.get_avg_iou2(np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)),
np.reshape(assigned_locations, (-1, 2)), np.reshape(assigned_scores,
assigned_scores.shape[
0] *
assigned_scores.shape[
1]))
IOU.update(iou_rate/(effective_positives), effective_positives)
# ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs)
# n_effective_batches += 1
assigned_scores = Variable(torch.LongTensor(assigned_scores),requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores, weight=cls_weights)
if torch.sum(assigned_scores)>0:
# print("HAHA")
assigned_head_positions = assigned_locations[:,:,0]
assigned_head_positions = assigned_head_positions.contiguous().view(-1)
#
assigned_tail_positions = assigned_locations[:,:,1]
assigned_tail_positions = assigned_tail_positions.contiguous().view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1])
# mask here: if there is non in assigned scores, no need to compute ...
assigned_head_positions = torch.masked_select(assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1))
assigned_head_positions = to_one_hot(assigned_head_positions, args.seq_len)
assigned_tail_positions = to_one_hot(assigned_tail_positions, args.seq_len)
prediction_head_loss = EMD_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True)
prediction_tail_loss = EMD_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True)
loc_losses.update(prediction_head_loss.data.item() + prediction_tail_loss.data.item(),
feature_batch.size(0))
total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
print(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format(
epoch,
model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg))
optim_scheduler.step(total_losses.avg)
model.eval()
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
pbar = progressbar.ProgressBar(max_value=len(val_dataloader))
for i_batch, sample_batch in enumerate(val_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
# valid_indices = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch)
pred_positions = torch.stack([head_positions, tail_positions], dim=-1)
assigned_scores, assigned_locations = h_assign.Assign_Batch(gt_positions, pred_positions, gt_valids, thres=hassign_thres)
# if np.sum(assigned_scores) > 1:
# print("DEBUG")
# correct_predictions = np.sum(assigned_scores[:,:args.n_outputs])
# cls_rate = correct_predictions*1./np.sum(assigned_scores)
if np.sum(assigned_scores) >= 1:
iou_rate, effective_positives = Metrics.get_avg_iou2(
np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)),
np.reshape(assigned_locations, (-1, 2)), np.reshape(assigned_scores,
assigned_scores.shape[
0] *
assigned_scores.shape[
1]))
IOU.update(iou_rate / (effective_positives), effective_positives)
assigned_scores = Variable(torch.LongTensor(assigned_scores), requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores, weight=cls_weights)
if torch.sum(assigned_scores)>0:
# print("HAHA")
assigned_head_positions = assigned_locations[:,:,0]
assigned_head_positions = assigned_head_positions.contiguous().view(-1)
#
assigned_tail_positions = assigned_locations[:,:,1]
assigned_tail_positions = assigned_tail_positions.contiguous().view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1])
# mask here: if there is non in assigned scores, no need to compute ...
assigned_head_positions = torch.masked_select(assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1))
assigned_head_positions = to_one_hot(assigned_head_positions, args.seq_len)
assigned_tail_positions = to_one_hot(assigned_tail_positions, args.seq_len)
prediction_head_loss = EMD_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True)
prediction_tail_loss = EMD_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True)
loc_losses.update(prediction_head_loss.data.item() + prediction_tail_loss.data.item(),
feature_batch.size(0))
total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.item(), feature_batch.size(0))
print(
"Val -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format(
epoch,
model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg,
IOU.avg, ordered_IOU.avg))
if epoch % 1 == 0:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss':total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'IoU': IOU.avg}, (epoch+1), file_direcotry=save_directory)
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
pp.pprint(vars(args))
model = PointerNetwork(input_dim=args.input_dim,
embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim,
max_decoding_len=args.net_outputs,
dropout=0.5)
print("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
# save_directory = 'gru2heads_proposal_s4-2_ckpt'
if args.resume is not None:
ckpt_idx = 2
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(
ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename,
map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=True)
args.start_epoch = checkpoint['epoch']
train_iou = checkpoint['IoU']
train_tloss = checkpoint['loss']
train_cls_loss = checkpoint['cls_loss']
train_loc_loss = checkpoint['loc_loss']
print(
"=> loading checkpoint '{}', total loss: {:.04f},\t cls_loss: {:.04f},\t loc_loss: {:.04f},"
" \tcurrent iou: {:.04f}".format(ckpt_filename, train_tloss,
train_cls_loss, train_loc_loss,
train_iou))
model = cuda_model.convertModel2Cuda(model,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
train_dataset = THUMOST14(seq_length=args.seq_len,
overlap=0.9,
sample_rate=[1, 2, 4],
dataset_split='val')
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model_optim = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min')
alpha = 0.1
for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
#Update here!
model.eval()
pbar = progressbar.ProgressBar(max_value=len(train_dataloader))
for i_batch, sample_batch in enumerate(train_dataloader):
# pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
valid_indices = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
#
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch)
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
assigned_scores, assigned_locations = h_match.Assign_Batch(
gt_positions, pred_positions, valid_indices, thres=0.5)
if valid_indices.byte().any() > 0:
print "Output at {:d}".format(i_batch)
# n_valid = valid_indices.data[0, 0]
view_idx = valid_indices.nonzero()[0][0].item()
n_valid = valid_indices[view_idx, 0].item()
print "GT:"
print(gt_positions[view_idx, :n_valid, :])
print("Pred")
print(pred_positions[view_idx])
_, head_sort = head_pointer_probs[view_idx, 0, :].sort()
_, tail_sort = tail_pointer_probs[view_idx, 0, :].sort()
print("END of {:d}".format(i_batch))
#
iou_rate, effective_positives = Losses.Metrics.get_avg_iou2(
np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)),
np.reshape(assigned_locations, (-1, 2)),
np.reshape(
assigned_scores,
assigned_scores.shape[0] * assigned_scores.shape[1]))
IOU.update(iou_rate / (effective_positives),
effective_positives)
assigned_scores = Variable(torch.LongTensor(assigned_scores),
requires_grad=False)
assigned_locations = Variable(
torch.LongTensor(assigned_locations), requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(
-1,
cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores)
assigned_head_positions = assigned_locations[:, :, 0]
assigned_head_positions = assigned_head_positions.contiguous(
).view(-1)
#
assigned_tail_positions = assigned_locations[:, :, 1]
assigned_tail_positions = assigned_tail_positions.contiguous(
).view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(
-1,
head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(
-1,
tail_pointer_probs.size()[-1])
# start_indices = start_indices.contiguous().view(-1)
# end_indices = end_indices.contiguous().view(-1)
# with case instances....
prediction_head_loss = F.cross_entropy((head_pointer_probs),
assigned_head_positions,
reduce=False)
prediction_head_loss = torch.mean(prediction_head_loss *
assigned_scores.float())
prediction_tail_loss = F.cross_entropy((tail_pointer_probs),
assigned_tail_positions,
reduce=False)
prediction_tail_loss = torch.mean(prediction_tail_loss *
assigned_scores.float())
total_loss = alpha * (prediction_head_loss +
prediction_tail_loss) + cls_loss
#
# # model_optim.zero_grad()
# # total_loss.backward()
# # torch.nn.utils.clip_grad_norm(model.parameters(), 1.)
# # model_optim.step()
cls_losses.update(cls_loss.data.item(), feature_batch.size(0))
loc_losses.update(
prediction_head_loss.data.item() +
prediction_tail_loss.data.item(), feature_batch.size(0))
total_losses.update(total_loss.data.item(),
feature_batch.size(0))
#
#
print(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}"
.format(epoch, model_optim.param_groups[0]['lr'],
total_losses.avg, cls_losses.avg, loc_losses.avg,
Accuracy.avg, IOU.avg, ordered_IOU.avg))
break
def main():
global args
args = (parser.parse_args())
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
# Pretty print the run args
pp.pprint(vars(args))
model = PointerNetwork(input_dim=args.input_dim,
embedding_dim=args.embedding_dim,
hidden_dim=args.hidden_dim,
max_decoding_len=args.net_outputs,
dropout=args.dropout,
n_enc_layers=2)
hassign_thres = args.hassign_thres
print("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
script_name_stem = dir_utils.get_stem(__file__)
save_directory = '{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-ckpt'.format(
script_name_stem, hassign_thres, args.alpha, args.hidden_dim,
args.dropout)
print("Save ckpt to {:s}".format(save_directory))
if args.resume is not None:
ckpt_idx = 7
ckpt_filename = args.resume.format(ckpt_idx)
assert os.path.isfile(
ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename,
map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=False)
train_iou = checkpoint['IoU']
args.start_epoch = checkpoint['epoch']
print("=> loading checkpoint '{}', current iou: {:.04f}".format(
ckpt_filename, train_iou))
model = cuda_model.convertModel2Cuda(model,
gpu_id=args.gpu_id,
multiGpu=args.multiGpu)
# train_dataset = THUMOST14(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='train',rdDrop=True,rdOffset=True)
val_dataset = THUMOST14(seq_length=args.seq_len,
overlap=0.9,
sample_rate=[4],
dataset_split='val',
rdDrop=False,
rdOffset=False)
# train_dataloader = DataLoader(train_dataset,
# batch_size=args.batch_size,
# shuffle=True,
# num_workers=4)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model_optim = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim,
'min',
patience=20)
alpha = args.alpha
cls_weights = torch.FloatTensor([0.05, 1.0]).cuda()
for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=len(val_dataloader))
for i_batch, sample_batch in enumerate(val_dataloader):
pbar.update(i_batch)
feature_batch = Variable(sample_batch[0])
start_indices = Variable(sample_batch[1])
end_indices = Variable(sample_batch[2])
gt_valids = Variable(sample_batch[3])
if use_cuda:
feature_batch = feature_batch.cuda()
start_indices = start_indices.cuda()
end_indices = end_indices.cuda()
gt_positions = torch.stack([start_indices, end_indices], dim=-1)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
feature_batch)
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
assigned_scores, assigned_locations = h_assign.Assign_Batch(
gt_positions, pred_positions, gt_valids, thres=hassign_thres)
if np.sum(assigned_scores) > 0:
print "Output at {:d}".format(i_batch)
# n_valid = valid_indices.data[0, 0]
# view_idx = valid_indices.nonzero()[0][0].item()
# n_valid = valid_indices[view_idx, 0].item()
print "GT:"
print(assigned_locations[0])
print("Pred")
print(pred_positions[0])
_, head_sort = head_pointer_probs[0, 0, :].sort()
_, tail_sort = tail_pointer_probs[0, 0, :].sort()
print("END of {:d}".format(i_batch))
def main():
# load data sets
global args
args = parser.parse_args()
pp.pprint(vars(args))
running_name = 'X'
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
# use_cuda = False
train_file = 'data/example.train'
dev_file = 'data/example.dev'
test_file = 'data/example.test'
embedding_file = 'data/vec.txt'
map_file = 'map.pkl'
config_file = 'config_file_pytorch'
tag_file = 'tag.pkl'
embedding_easy_file = 'data/easy_embedding.npy'
train_sentences = load_sentences(train_file)
dev_sentences = load_sentences(dev_file)
test_sentences = load_sentences(test_file)
# train_sentences = dev_sentences
update_tag_scheme(train_sentences, args.tag_schema)
update_tag_scheme(test_sentences, args.tag_schema)
update_tag_scheme(dev_sentences, args.tag_schema)
if not os.path.isfile(tag_file):
_, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(tag_file, "wb") as f:
pickle.dump([tag_to_id, id_to_tag], f)
else:
with open(tag_file, 'rb') as t:
tag_to_id, id_to_tag = pickle.load(t)
if not os.path.isfile(map_file):
# create dictionary for word
dico_chars_train = char_mapping(train_sentences)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), embedding_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
# _, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char], f)
else:
with open(map_file, "rb") as f:
char_to_id, id_to_char = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
train_manager = BatchManager(train_data, args.batch_size)
dev_manager = BatchManager(dev_data, 50)
test_manager = BatchManager(test_data, 50)
# make path for store log and model if not exist
# make_path(FLAGS)
if os.path.isfile(config_file):
config = load_config(config_file)
else:
config = config_model(char_to_id, tag_to_id, args)
save_config(config, config_file)
# make_path(running_name)
save_places = dir_utils.save_places(running_name)
# log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(
os.path.join(save_places.log_save_dir,
'{:s}.txt'.format(dir_utils.get_date_str())))
print_config(config, logger)
logger.info("start training")
# loss = []
#Update: create model and embedding!
model = NERModel.CNERPointer(char_dim=args.char_dim,
seg_dim=args.seg_dim,
hidden_dim=args.hidden_dim,
max_length=15,
embedding_path=embedding_file,
id_to_word=id_to_char,
easy_load=embedding_easy_file)
print("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
#Update: this won't work!
# model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
if use_cuda:
model = model.cuda()
model_optim = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=float(args.lr))
optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim,
'min',
patience=10)
for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch):
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.train()
pbar = progressbar.ProgressBar(max_value=train_manager.len_data)
for batch_idx, batch in enumerate(
train_manager.iter_batch(shuffle=True)):
pbar.update(batch_idx)
word_vectors = torch.LongTensor(batch[1])
seg_vectors = torch.LongTensor(batch[2])
batch_size = word_vectors.shape[0]
input_length = word_vectors.shape[1]
word_input = Variable(word_vectors)
seg_input = Variable(seg_vectors)
if use_cuda:
word_input = word_input.cuda()
seg_input = seg_input.cuda()
tagging_BIOUS = batch[3]
segments, max_len = convertBIOU2SegmentsBatch(
tagging_BIOUS, id_to_tag)
gt_positions, gt_valids = createPytorchLabels(segments, max_len)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
word_input, seg_input, max_len)
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
assigned_scores, assigned_locations = h_assign.Assign_Batch(
gt_positions,
pred_positions,
gt_valids,
thres=args.hassign_thres)
if np.sum(assigned_scores) >= 1:
iou_rate, effective_positives = Metrics.get_avg_iou2(
np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)),
np.reshape(assigned_locations, (-1, 2)),
np.reshape(
assigned_scores,
assigned_scores.shape[0] * assigned_scores.shape[1]))
IOU.update(iou_rate / (effective_positives),
effective_positives)
# ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs)
# n_effective_batches += 1
assigned_scores = Variable(torch.LongTensor(assigned_scores),
requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations),
requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1,
cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores)
if torch.sum(assigned_scores) > 0:
# print("HAHA")
assigned_head_positions = assigned_locations[:, :, 0]
assigned_head_positions = assigned_head_positions.contiguous(
).view(-1)
#
assigned_tail_positions = assigned_locations[:, :, 1]
assigned_tail_positions = assigned_tail_positions.contiguous(
).view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(
-1,
head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(
-1,
tail_pointer_probs.size()[-1])
# mask here: if there is non in assigned scores, no need to compute ...
assigned_head_positions = torch.masked_select(
assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(
assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(
head_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(
tail_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
assigned_head_positions = to_one_hot(assigned_head_positions,
input_length)
assigned_tail_positions = to_one_hot(assigned_tail_positions,
input_length)
prediction_head_loss = EMD_L2(head_pointer_probs,
assigned_head_positions,
needSoftMax=True)
prediction_tail_loss = EMD_L2(tail_pointer_probs,
assigned_tail_positions,
needSoftMax=True)
loc_losses.update(
prediction_head_loss.data.item() +
prediction_tail_loss.data.item(), batch_size)
total_loss = args.alpha * (prediction_head_loss +
prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
model_optim.zero_grad()
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
model_optim.step()
cls_losses.update(cls_loss.data.item(), batch_size)
total_losses.update(total_loss.item(), batch_size)
logger.info(
"Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}"
.format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg,
cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg,
ordered_IOU.avg))
optim_scheduler.step(total_losses.avg)
total_losses = AverageMeter()
loc_losses = AverageMeter()
cls_losses = AverageMeter()
Accuracy = AverageMeter()
IOU = AverageMeter()
ordered_IOU = AverageMeter()
model.eval()
pbar = progressbar.ProgressBar(max_value=dev_manager.len_data)
for batch_idx, batch in enumerate(
dev_manager.iter_batch(shuffle=True)):
pbar.update(batch_idx)
word_vectors = torch.LongTensor(batch[1])
seg_vectors = torch.LongTensor(batch[2])
batch_size = word_vectors.shape[0]
input_length = word_vectors.shape[1]
word_input = Variable(word_vectors)
seg_input = Variable(seg_vectors)
if use_cuda:
word_input = word_input.cuda()
seg_input = seg_input.cuda()
tagging_BIOUS = batch[3]
segments, max_len = convertBIOU2SegmentsBatch(
tagging_BIOUS, id_to_tag)
head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(
word_input, seg_input, max_len)
pred_positions = torch.stack([head_positions, tail_positions],
dim=-1)
gt_positions, gt_valids = createPytorchLabels(segments, max_len)
assigned_scores, assigned_locations = h_assign.Assign_Batch(
gt_positions,
pred_positions,
gt_valids,
thres=args.hassign_thres)
if np.sum(assigned_scores) >= 1:
iou_rate, effective_positives = Metrics.get_avg_iou2(
np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)),
np.reshape(assigned_locations, (-1, 2)),
np.reshape(
assigned_scores,
assigned_scores.shape[0] * assigned_scores.shape[1]))
IOU.update(iou_rate / (effective_positives),
effective_positives)
# ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs)
# n_effective_batches += 1
assigned_scores = Variable(torch.LongTensor(assigned_scores),
requires_grad=False)
assigned_locations = Variable(torch.LongTensor(assigned_locations),
requires_grad=False)
if use_cuda:
assigned_scores = assigned_scores.cuda()
assigned_locations = assigned_locations.cuda()
cls_scores = cls_scores.contiguous().view(-1,
cls_scores.size()[-1])
assigned_scores = assigned_scores.contiguous().view(-1)
cls_loss = F.cross_entropy(cls_scores, assigned_scores)
if torch.sum(assigned_scores) > 0:
# print("HAHA")
assigned_head_positions = assigned_locations[:, :, 0]
assigned_head_positions = assigned_head_positions.contiguous(
).view(-1)
#
assigned_tail_positions = assigned_locations[:, :, 1]
assigned_tail_positions = assigned_tail_positions.contiguous(
).view(-1)
head_pointer_probs = head_pointer_probs.contiguous().view(
-1,
head_pointer_probs.size()[-1])
tail_pointer_probs = tail_pointer_probs.contiguous().view(
-1,
tail_pointer_probs.size()[-1])
# mask here: if there is non in assigned scores, no need to compute ...
assigned_head_positions = torch.masked_select(
assigned_head_positions, assigned_scores.byte())
assigned_tail_positions = torch.masked_select(
assigned_tail_positions, assigned_scores.byte())
head_pointer_probs = torch.index_select(
head_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
tail_pointer_probs = torch.index_select(
tail_pointer_probs,
dim=0,
index=assigned_scores.nonzero().squeeze(1))
assigned_head_positions = to_one_hot(assigned_head_positions,
input_length)
assigned_tail_positions = to_one_hot(assigned_tail_positions,
input_length)
prediction_head_loss = EMD_L2(head_pointer_probs,
assigned_head_positions,
needSoftMax=True)
prediction_tail_loss = EMD_L2(tail_pointer_probs,
assigned_tail_positions,
needSoftMax=True)
loc_losses.update(
prediction_head_loss.data.item() +
prediction_tail_loss.data.item(), batch_size)
total_loss = args.alpha * (prediction_head_loss +
prediction_tail_loss) + cls_loss
else:
total_loss = cls_loss
# model_optim.zero_grad()
# total_loss.backward()
# torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
# model_optim.step()
cls_losses.update(cls_loss.data.item(), batch_size)
total_losses.update(total_loss.item(), batch_size)
logger.info(
"Val -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}"
.format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg,
cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg,
ordered_IOU.avg))
if epoch % 1 == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss': total_losses.avg,
'cls_loss': cls_losses.avg,
'loc_loss': loc_losses.avg,
'IoU': IOU.avg
}, (epoch + 1),
file_direcotry=save_places.model_save_dir)
