def main(): global args args = (parser.parse_args()) use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu) ckpt_savedir = 'lstm2heads_proposal_c3d_s4_ckpt' # Pretty print the run args pp.pprint(vars(args)) model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim, hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=0.5) print("Number of Params\t{:d}".format(sum([p.data.nelement() for p in model.parameters()]))) if args.resume is not None: ckpt_idx = args.fileid ckpt_filename = os.path.join(args.resume, 'checkpoint_{:04d}.pth.tar'.format(ckpt_idx)) assert os.path.isfile(ckpt_filename), 'Error: no checkpoint directory found!' checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage) model.load_state_dict(checkpoint['state_dict'], strict=False) train_iou = checkpoint['IoU'] args.start_epoch = checkpoint['epoch'] print("=> loading checkpoint '{}', current iou: {:.04f}".format(ckpt_filename, train_iou)) model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu) train_dataset = THUMOST14(seq_length=args.seq_len, overlap=0.9, sample_rate=4) train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4) # val_dataloader = DataLoader(val_dataset, # batch_size=args.batch_size, # shuffle=True, # num_workers=4) model_optim = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()), lr=float(args.lr)) alpha=0.1 for epoch in range(args.start_epoch, args.nof_epoch+args.start_epoch): total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() model.train() pbar = progressbar.ProgressBar(max_value=len(train_dataloader)) for i_batch, sample_batch in enumerate(train_dataloader): pbar.update(i_batch) feature_batch = Variable(sample_batch[0]) start_indices = Variable(sample_batch[1]) end_indices = Variable(sample_batch[2]) valid_indices = Variable(sample_batch[3]) # gt_index_batch = sample_batch[1].numpy() # score_batch = Variable(sample_batch[2]) if use_cuda: feature_batch = feature_batch.cuda() start_indices = start_indices.cuda() end_indices = end_indices.cuda() gt_positions = torch.stack([start_indices, end_indices], dim=-1) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores = model(feature_batch) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) assigned_scores, assigned_locations = h_assign.Assign_Batch(gt_positions, pred_positions, valid_indices, thres=0.5) # if np.sum(assigned_scores) > 1: # print("DEBUG") # correct_predictions = np.sum(assigned_scores[:,:args.n_outputs]) # cls_rate = correct_predictions*1./np.sum(assigned_scores) if np.sum(assigned_scores)>=1: iou_rate, effective_positives = Metrics.get_avg_iou2(np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(assigned_locations, (-1, 2)), np.reshape(assigned_scores, assigned_scores.shape[ 0] * assigned_scores.shape[ 1])) # _, top_assigned_locations = h_assign_proposal.Assign_Batch(gt_positions, pred_positions[:, : args.n_outputs, :], valid_indices, thres=0.5) # # ordered_iou_rate = Metrics.get_avg_iou(np.reshape(pred_positions[:,:args.n_outputs,:].data.cpu().numpy(), (-1, 2)), # np.reshape(top_assigned_locations, (-1, 2))) # Accuracy.update(cls_rate, np.sum(assigned_scores)) # iou_rate = Metrics.get_avg_iou(np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(gt_positions.data.cpu().numpy(), (-1, 2))) IOU.update(iou_rate/(effective_positives), effective_positives) # ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs) # n_effective_batches += 1 assigned_scores = Variable(torch.LongTensor(assigned_scores),requires_grad=False) assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False) if use_cuda: assigned_scores = assigned_scores.cuda() assigned_locations = assigned_locations.cuda() cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1]) assigned_scores = assigned_scores.contiguous().view(-1) cls_loss = F.cross_entropy(cls_scores, assigned_scores) assigned_head_positions = assigned_locations[:,:,0] assigned_head_positions = assigned_head_positions.contiguous().view(-1) # assigned_tail_positions = assigned_locations[:,:,1] assigned_tail_positions = assigned_tail_positions.contiguous().view(-1) head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1]) tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1]) # start_indices = start_indices.contiguous().view(-1) # end_indices = end_indices.contiguous().view(-1) # with case instances.... prediction_head_loss = F.cross_entropy((head_pointer_probs), assigned_head_positions, reduce=False) prediction_head_loss = torch.mean(prediction_head_loss * assigned_scores.float()) prediction_tail_loss = F.cross_entropy((tail_pointer_probs), assigned_tail_positions, reduce=False) prediction_tail_loss = torch.mean(prediction_tail_loss * assigned_scores.float()) total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss model_optim.zero_grad() total_loss.backward() torch.nn.utils.clip_grad_norm(model.parameters(), 1.) model_optim.step() cls_losses.update(cls_loss.data[0], feature_batch.size(0)) loc_losses.update(prediction_head_loss.data[0] + prediction_tail_loss.data[0], feature_batch.size(0)) total_losses.update(total_loss.data[0], feature_batch.size(0)) print( "Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format( epoch, model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg)) if epoch % 1 == 0: save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'loss':total_losses.avg, 'cls_loss': cls_losses.avg, 'loc_loss': loc_losses.avg, 'IoU': IOU.avg}, (epoch+1), file_direcotry=ckpt_savedir)
def main(): global args args = (parser.parse_args()) use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu) # Pretty print the run args pp.pprint(vars(args)) model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim, hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=args.dropout, n_enc_layers=2) hassign_thres = args.hassign_thres print("Number of Params\t{:d}".format(sum([p.data.nelement() for p in model.parameters()]))) script_name_stem = dir_utils.get_stem(__file__) save_directory = os.path.join(project_root, 'ckpts', '{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-seqlen{:d}-ckpt'. format(script_name_stem, hassign_thres, args.alpha, args.hidden_dim, args.dropout, args.seq_len)) print("Save ckpt to {:s}".format(save_directory)) if args.resume is not None: ckpt_idx = 3 ckpt_filename = args.resume.format(ckpt_idx) assert os.path.isfile(ckpt_filename), 'Error: no checkpoint directory found!' checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage) model.load_state_dict(checkpoint['state_dict'], strict=False) train_iou = checkpoint['IoU'] args.start_epoch = checkpoint['epoch'] print("=> loading checkpoint '{}', current iou: {:.04f}".format(ckpt_filename, train_iou)) model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu) train_dataset = cDataset(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='train', rdDrop=True, rdOffset=True) val_dataset = cDataset(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='val', rdDrop=False, rdOffset=False) train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4) val_dataloader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) model_optim = optim.Adam(filter(lambda p:p.requires_grad, model.parameters()), lr=float(args.lr)) optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min', patience=20) alpha=args.alpha cls_weights = torch.FloatTensor([0.05, 1.0]).cuda() for epoch in range(args.start_epoch, args.nof_epoch+args.start_epoch): total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() model.train() pbar = progressbar.ProgressBar(max_value=len(train_dataloader)) for i_batch, sample_batch in enumerate(train_dataloader): pbar.update(i_batch) feature_batch = Variable(sample_batch[0]) start_indices = Variable(sample_batch[1]) end_indices = Variable(sample_batch[2]) gt_valids = Variable(sample_batch[3]) if use_cuda: feature_batch = feature_batch.cuda() start_indices = start_indices.cuda() end_indices = end_indices.cuda() gt_positions = torch.stack([start_indices, end_indices], dim=-1) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model(feature_batch) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) assigned_scores, assigned_locations = h_assign.Assign_Batch(gt_positions, pred_positions, gt_valids, thres=hassign_thres) # if np.sum(assigned_scores) > 1: # print("DEBUG") # correct_predictions = np.sum(assigned_scores[:,:args.n_outputs]) # cls_rate = correct_predictions*1./np.sum(assigned_scores) if np.sum(assigned_scores)>=1: iou_rate, effective_positives = Metrics.get_avg_iou2(np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(assigned_locations, (-1, 2)), np.reshape(assigned_scores, assigned_scores.shape[ 0] * assigned_scores.shape[ 1])) IOU.update(iou_rate/(effective_positives), effective_positives) # ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs) # n_effective_batches += 1 assigned_scores = Variable(torch.LongTensor(assigned_scores),requires_grad=False) assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False) if use_cuda: assigned_scores = assigned_scores.cuda() assigned_locations = assigned_locations.cuda() cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1]) assigned_scores = assigned_scores.contiguous().view(-1) cls_loss = F.cross_entropy(cls_scores, assigned_scores, weight=cls_weights) if torch.sum(assigned_scores)>0: # print("HAHA") assigned_head_positions = assigned_locations[:,:,0] assigned_head_positions = assigned_head_positions.contiguous().view(-1) # assigned_tail_positions = assigned_locations[:,:,1] assigned_tail_positions = assigned_tail_positions.contiguous().view(-1) head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1]) tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1]) # mask here: if there is non in assigned scores, no need to compute ... assigned_head_positions = torch.masked_select(assigned_head_positions, assigned_scores.byte()) assigned_tail_positions = torch.masked_select(assigned_tail_positions, assigned_scores.byte()) head_pointer_probs = torch.index_select(head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) tail_pointer_probs = torch.index_select(tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) assigned_head_positions = to_one_hot(assigned_head_positions, args.seq_len) assigned_tail_positions = to_one_hot(assigned_tail_positions, args.seq_len) prediction_head_loss = EMD_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True) prediction_tail_loss = EMD_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True) loc_losses.update(prediction_head_loss.data.item() + prediction_tail_loss.data.item(), feature_batch.size(0)) total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss else: total_loss = cls_loss model_optim.zero_grad() total_loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) model_optim.step() cls_losses.update(cls_loss.data.item(), feature_batch.size(0)) total_losses.update(total_loss.item(), feature_batch.size(0)) print( "Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format( epoch, model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg)) optim_scheduler.step(total_losses.avg) model.eval() total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() pbar = progressbar.ProgressBar(max_value=len(val_dataloader)) for i_batch, sample_batch in enumerate(val_dataloader): pbar.update(i_batch) feature_batch = Variable(sample_batch[0]) start_indices = Variable(sample_batch[1]) end_indices = Variable(sample_batch[2]) gt_valids = Variable(sample_batch[3]) # valid_indices = Variable(sample_batch[3]) if use_cuda: feature_batch = feature_batch.cuda() start_indices = start_indices.cuda() end_indices = end_indices.cuda() gt_positions = torch.stack([start_indices, end_indices], dim=-1) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model( feature_batch) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) assigned_scores, assigned_locations = h_assign.Assign_Batch(gt_positions, pred_positions, gt_valids, thres=hassign_thres) # if np.sum(assigned_scores) > 1: # print("DEBUG") # correct_predictions = np.sum(assigned_scores[:,:args.n_outputs]) # cls_rate = correct_predictions*1./np.sum(assigned_scores) if np.sum(assigned_scores) >= 1: iou_rate, effective_positives = Metrics.get_avg_iou2( np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(assigned_locations, (-1, 2)), np.reshape(assigned_scores, assigned_scores.shape[ 0] * assigned_scores.shape[ 1])) IOU.update(iou_rate / (effective_positives), effective_positives) assigned_scores = Variable(torch.LongTensor(assigned_scores), requires_grad=False) assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False) if use_cuda: assigned_scores = assigned_scores.cuda() assigned_locations = assigned_locations.cuda() cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1]) assigned_scores = assigned_scores.contiguous().view(-1) cls_loss = F.cross_entropy(cls_scores, assigned_scores, weight=cls_weights) if torch.sum(assigned_scores)>0: # print("HAHA") assigned_head_positions = assigned_locations[:,:,0] assigned_head_positions = assigned_head_positions.contiguous().view(-1) # assigned_tail_positions = assigned_locations[:,:,1] assigned_tail_positions = assigned_tail_positions.contiguous().view(-1) head_pointer_probs = head_pointer_probs.contiguous().view(-1, head_pointer_probs.size()[-1]) tail_pointer_probs = tail_pointer_probs.contiguous().view(-1, tail_pointer_probs.size()[-1]) # mask here: if there is non in assigned scores, no need to compute ... assigned_head_positions = torch.masked_select(assigned_head_positions, assigned_scores.byte()) assigned_tail_positions = torch.masked_select(assigned_tail_positions, assigned_scores.byte()) head_pointer_probs = torch.index_select(head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) tail_pointer_probs = torch.index_select(tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) assigned_head_positions = to_one_hot(assigned_head_positions, args.seq_len) assigned_tail_positions = to_one_hot(assigned_tail_positions, args.seq_len) prediction_head_loss = EMD_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True) prediction_tail_loss = EMD_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True) loc_losses.update(prediction_head_loss.data.item() + prediction_tail_loss.data.item(), feature_batch.size(0)) total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss else: total_loss = cls_loss cls_losses.update(cls_loss.data.item(), feature_batch.size(0)) total_losses.update(total_loss.item(), feature_batch.size(0)) print( "Val -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}".format( epoch, model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg)) if epoch % 1 == 0: save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'loss':total_losses.avg, 'cls_loss': cls_losses.avg, 'loc_loss': loc_losses.avg, 'IoU': IOU.avg}, (epoch+1), file_direcotry=save_directory)
def main(): global args args = (parser.parse_args()) use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu) pp.pprint(vars(args)) model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim, hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=0.5) print("Number of Params\t{:d}".format( sum([p.data.nelement() for p in model.parameters()]))) # save_directory = 'gru2heads_proposal_s4-2_ckpt' if args.resume is not None: ckpt_idx = 2 ckpt_filename = args.resume.format(ckpt_idx) assert os.path.isfile( ckpt_filename), 'Error: no checkpoint directory found!' checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage) model.load_state_dict(checkpoint['state_dict'], strict=True) args.start_epoch = checkpoint['epoch'] train_iou = checkpoint['IoU'] train_tloss = checkpoint['loss'] train_cls_loss = checkpoint['cls_loss'] train_loc_loss = checkpoint['loc_loss'] print( "=> loading checkpoint '{}', total loss: {:.04f},\t cls_loss: {:.04f},\t loc_loss: {:.04f}," " \tcurrent iou: {:.04f}".format(ckpt_filename, train_tloss, train_cls_loss, train_loc_loss, train_iou)) model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu) train_dataset = THUMOST14(seq_length=args.seq_len, overlap=0.9, sample_rate=[1, 2, 4], dataset_split='val') train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) model_optim = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=float(args.lr)) optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min') alpha = 0.1 for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch): total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() #Update here! model.eval() pbar = progressbar.ProgressBar(max_value=len(train_dataloader)) for i_batch, sample_batch in enumerate(train_dataloader): # pbar.update(i_batch) feature_batch = Variable(sample_batch[0]) start_indices = Variable(sample_batch[1]) end_indices = Variable(sample_batch[2]) valid_indices = Variable(sample_batch[3]) if use_cuda: feature_batch = feature_batch.cuda() start_indices = start_indices.cuda() end_indices = end_indices.cuda() # gt_positions = torch.stack([start_indices, end_indices], dim=-1) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model( feature_batch) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) assigned_scores, assigned_locations = h_match.Assign_Batch( gt_positions, pred_positions, valid_indices, thres=0.5) if valid_indices.byte().any() > 0: print "Output at {:d}".format(i_batch) # n_valid = valid_indices.data[0, 0] view_idx = valid_indices.nonzero()[0][0].item() n_valid = valid_indices[view_idx, 0].item() print "GT:" print(gt_positions[view_idx, :n_valid, :]) print("Pred") print(pred_positions[view_idx]) _, head_sort = head_pointer_probs[view_idx, 0, :].sort() _, tail_sort = tail_pointer_probs[view_idx, 0, :].sort() print("END of {:d}".format(i_batch)) # iou_rate, effective_positives = Losses.Metrics.get_avg_iou2( np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(assigned_locations, (-1, 2)), np.reshape( assigned_scores, assigned_scores.shape[0] * assigned_scores.shape[1])) IOU.update(iou_rate / (effective_positives), effective_positives) assigned_scores = Variable(torch.LongTensor(assigned_scores), requires_grad=False) assigned_locations = Variable( torch.LongTensor(assigned_locations), requires_grad=False) if use_cuda: assigned_scores = assigned_scores.cuda() assigned_locations = assigned_locations.cuda() cls_scores = cls_scores.contiguous().view( -1, cls_scores.size()[-1]) assigned_scores = assigned_scores.contiguous().view(-1) cls_loss = F.cross_entropy(cls_scores, assigned_scores) assigned_head_positions = assigned_locations[:, :, 0] assigned_head_positions = assigned_head_positions.contiguous( ).view(-1) # assigned_tail_positions = assigned_locations[:, :, 1] assigned_tail_positions = assigned_tail_positions.contiguous( ).view(-1) head_pointer_probs = head_pointer_probs.contiguous().view( -1, head_pointer_probs.size()[-1]) tail_pointer_probs = tail_pointer_probs.contiguous().view( -1, tail_pointer_probs.size()[-1]) # start_indices = start_indices.contiguous().view(-1) # end_indices = end_indices.contiguous().view(-1) # with case instances.... prediction_head_loss = F.cross_entropy((head_pointer_probs), assigned_head_positions, reduce=False) prediction_head_loss = torch.mean(prediction_head_loss * assigned_scores.float()) prediction_tail_loss = F.cross_entropy((tail_pointer_probs), assigned_tail_positions, reduce=False) prediction_tail_loss = torch.mean(prediction_tail_loss * assigned_scores.float()) total_loss = alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss # # # model_optim.zero_grad() # # total_loss.backward() # # torch.nn.utils.clip_grad_norm(model.parameters(), 1.) # # model_optim.step() cls_losses.update(cls_loss.data.item(), feature_batch.size(0)) loc_losses.update( prediction_head_loss.data.item() + prediction_tail_loss.data.item(), feature_batch.size(0)) total_losses.update(total_loss.data.item(), feature_batch.size(0)) # # print( "Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}" .format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg)) break
def main(): global args args = (parser.parse_args()) use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu) # Pretty print the run args pp.pprint(vars(args)) model = PointerNetwork(input_dim=args.input_dim, embedding_dim=args.embedding_dim, hidden_dim=args.hidden_dim, max_decoding_len=args.net_outputs, dropout=args.dropout, n_enc_layers=2) hassign_thres = args.hassign_thres print("Number of Params\t{:d}".format( sum([p.data.nelement() for p in model.parameters()]))) script_name_stem = dir_utils.get_stem(__file__) save_directory = '{:s}-assgin{:.2f}-alpha{:.4f}-dim{:d}-dropout{:.4f}-ckpt'.format( script_name_stem, hassign_thres, args.alpha, args.hidden_dim, args.dropout) print("Save ckpt to {:s}".format(save_directory)) if args.resume is not None: ckpt_idx = 7 ckpt_filename = args.resume.format(ckpt_idx) assert os.path.isfile( ckpt_filename), 'Error: no checkpoint directory found!' checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage) model.load_state_dict(checkpoint['state_dict'], strict=False) train_iou = checkpoint['IoU'] args.start_epoch = checkpoint['epoch'] print("=> loading checkpoint '{}', current iou: {:.04f}".format( ckpt_filename, train_iou)) model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu) # train_dataset = THUMOST14(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='train',rdDrop=True,rdOffset=True) val_dataset = THUMOST14(seq_length=args.seq_len, overlap=0.9, sample_rate=[4], dataset_split='val', rdDrop=False, rdOffset=False) # train_dataloader = DataLoader(train_dataset, # batch_size=args.batch_size, # shuffle=True, # num_workers=4) val_dataloader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) model_optim = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=float(args.lr)) optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min', patience=20) alpha = args.alpha cls_weights = torch.FloatTensor([0.05, 1.0]).cuda() for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch): total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() model.train() pbar = progressbar.ProgressBar(max_value=len(val_dataloader)) for i_batch, sample_batch in enumerate(val_dataloader): pbar.update(i_batch) feature_batch = Variable(sample_batch[0]) start_indices = Variable(sample_batch[1]) end_indices = Variable(sample_batch[2]) gt_valids = Variable(sample_batch[3]) if use_cuda: feature_batch = feature_batch.cuda() start_indices = start_indices.cuda() end_indices = end_indices.cuda() gt_positions = torch.stack([start_indices, end_indices], dim=-1) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model( feature_batch) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) assigned_scores, assigned_locations = h_assign.Assign_Batch( gt_positions, pred_positions, gt_valids, thres=hassign_thres) if np.sum(assigned_scores) > 0: print "Output at {:d}".format(i_batch) # n_valid = valid_indices.data[0, 0] # view_idx = valid_indices.nonzero()[0][0].item() # n_valid = valid_indices[view_idx, 0].item() print "GT:" print(assigned_locations[0]) print("Pred") print(pred_positions[0]) _, head_sort = head_pointer_probs[0, 0, :].sort() _, tail_sort = tail_pointer_probs[0, 0, :].sort() print("END of {:d}".format(i_batch))
def main(): # load data sets global args args = parser.parse_args() pp.pprint(vars(args)) running_name = 'X' use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu) # use_cuda = False train_file = 'data/example.train' dev_file = 'data/example.dev' test_file = 'data/example.test' embedding_file = 'data/vec.txt' map_file = 'map.pkl' config_file = 'config_file_pytorch' tag_file = 'tag.pkl' embedding_easy_file = 'data/easy_embedding.npy' train_sentences = load_sentences(train_file) dev_sentences = load_sentences(dev_file) test_sentences = load_sentences(test_file) # train_sentences = dev_sentences update_tag_scheme(train_sentences, args.tag_schema) update_tag_scheme(test_sentences, args.tag_schema) update_tag_scheme(dev_sentences, args.tag_schema) if not os.path.isfile(tag_file): _, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(tag_file, "wb") as f: pickle.dump([tag_to_id, id_to_tag], f) else: with open(tag_file, 'rb') as t: tag_to_id, id_to_tag = pickle.load(t) if not os.path.isfile(map_file): # create dictionary for word dico_chars_train = char_mapping(train_sentences)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), embedding_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) # _, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(map_file, "wb") as f: pickle.dump([char_to_id, id_to_char], f) else: with open(map_file, "rb") as f: char_to_id, id_to_char = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), len(dev_data), len(test_data))) train_manager = BatchManager(train_data, args.batch_size) dev_manager = BatchManager(dev_data, 50) test_manager = BatchManager(test_data, 50) # make path for store log and model if not exist # make_path(FLAGS) if os.path.isfile(config_file): config = load_config(config_file) else: config = config_model(char_to_id, tag_to_id, args) save_config(config, config_file) # make_path(running_name) save_places = dir_utils.save_places(running_name) # log_path = os.path.join("log", FLAGS.log_file) logger = get_logger( os.path.join(save_places.log_save_dir, '{:s}.txt'.format(dir_utils.get_date_str()))) print_config(config, logger) logger.info("start training") # loss = [] #Update: create model and embedding! model = NERModel.CNERPointer(char_dim=args.char_dim, seg_dim=args.seg_dim, hidden_dim=args.hidden_dim, max_length=15, embedding_path=embedding_file, id_to_word=id_to_char, easy_load=embedding_easy_file) print("Number of Params\t{:d}".format( sum([p.data.nelement() for p in model.parameters()]))) #Update: this won't work! # model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu) if use_cuda: model = model.cuda() model_optim = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=float(args.lr)) optim_scheduler = optim.lr_scheduler.ReduceLROnPlateau(model_optim, 'min', patience=10) for epoch in range(args.start_epoch, args.nof_epoch + args.start_epoch): total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() model.train() pbar = progressbar.ProgressBar(max_value=train_manager.len_data) for batch_idx, batch in enumerate( train_manager.iter_batch(shuffle=True)): pbar.update(batch_idx) word_vectors = torch.LongTensor(batch[1]) seg_vectors = torch.LongTensor(batch[2]) batch_size = word_vectors.shape[0] input_length = word_vectors.shape[1] word_input = Variable(word_vectors) seg_input = Variable(seg_vectors) if use_cuda: word_input = word_input.cuda() seg_input = seg_input.cuda() tagging_BIOUS = batch[3] segments, max_len = convertBIOU2SegmentsBatch( tagging_BIOUS, id_to_tag) gt_positions, gt_valids = createPytorchLabels(segments, max_len) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model( word_input, seg_input, max_len) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) assigned_scores, assigned_locations = h_assign.Assign_Batch( gt_positions, pred_positions, gt_valids, thres=args.hassign_thres) if np.sum(assigned_scores) >= 1: iou_rate, effective_positives = Metrics.get_avg_iou2( np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(assigned_locations, (-1, 2)), np.reshape( assigned_scores, assigned_scores.shape[0] * assigned_scores.shape[1])) IOU.update(iou_rate / (effective_positives), effective_positives) # ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs) # n_effective_batches += 1 assigned_scores = Variable(torch.LongTensor(assigned_scores), requires_grad=False) assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False) if use_cuda: assigned_scores = assigned_scores.cuda() assigned_locations = assigned_locations.cuda() cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1]) assigned_scores = assigned_scores.contiguous().view(-1) cls_loss = F.cross_entropy(cls_scores, assigned_scores) if torch.sum(assigned_scores) > 0: # print("HAHA") assigned_head_positions = assigned_locations[:, :, 0] assigned_head_positions = assigned_head_positions.contiguous( ).view(-1) # assigned_tail_positions = assigned_locations[:, :, 1] assigned_tail_positions = assigned_tail_positions.contiguous( ).view(-1) head_pointer_probs = head_pointer_probs.contiguous().view( -1, head_pointer_probs.size()[-1]) tail_pointer_probs = tail_pointer_probs.contiguous().view( -1, tail_pointer_probs.size()[-1]) # mask here: if there is non in assigned scores, no need to compute ... assigned_head_positions = torch.masked_select( assigned_head_positions, assigned_scores.byte()) assigned_tail_positions = torch.masked_select( assigned_tail_positions, assigned_scores.byte()) head_pointer_probs = torch.index_select( head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) tail_pointer_probs = torch.index_select( tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) assigned_head_positions = to_one_hot(assigned_head_positions, input_length) assigned_tail_positions = to_one_hot(assigned_tail_positions, input_length) prediction_head_loss = EMD_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True) prediction_tail_loss = EMD_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True) loc_losses.update( prediction_head_loss.data.item() + prediction_tail_loss.data.item(), batch_size) total_loss = args.alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss else: total_loss = cls_loss model_optim.zero_grad() total_loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) model_optim.step() cls_losses.update(cls_loss.data.item(), batch_size) total_losses.update(total_loss.item(), batch_size) logger.info( "Train -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}" .format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg)) optim_scheduler.step(total_losses.avg) total_losses = AverageMeter() loc_losses = AverageMeter() cls_losses = AverageMeter() Accuracy = AverageMeter() IOU = AverageMeter() ordered_IOU = AverageMeter() model.eval() pbar = progressbar.ProgressBar(max_value=dev_manager.len_data) for batch_idx, batch in enumerate( dev_manager.iter_batch(shuffle=True)): pbar.update(batch_idx) word_vectors = torch.LongTensor(batch[1]) seg_vectors = torch.LongTensor(batch[2]) batch_size = word_vectors.shape[0] input_length = word_vectors.shape[1] word_input = Variable(word_vectors) seg_input = Variable(seg_vectors) if use_cuda: word_input = word_input.cuda() seg_input = seg_input.cuda() tagging_BIOUS = batch[3] segments, max_len = convertBIOU2SegmentsBatch( tagging_BIOUS, id_to_tag) head_pointer_probs, head_positions, tail_pointer_probs, tail_positions, cls_scores, _ = model( word_input, seg_input, max_len) pred_positions = torch.stack([head_positions, tail_positions], dim=-1) gt_positions, gt_valids = createPytorchLabels(segments, max_len) assigned_scores, assigned_locations = h_assign.Assign_Batch( gt_positions, pred_positions, gt_valids, thres=args.hassign_thres) if np.sum(assigned_scores) >= 1: iou_rate, effective_positives = Metrics.get_avg_iou2( np.reshape(pred_positions.data.cpu().numpy(), (-1, 2)), np.reshape(assigned_locations, (-1, 2)), np.reshape( assigned_scores, assigned_scores.shape[0] * assigned_scores.shape[1])) IOU.update(iou_rate / (effective_positives), effective_positives) # ordered_IOU.update(ordered_iou_rate/(args.batch_size*args.n_outputs),args.batch_size*args.n_outputs) # n_effective_batches += 1 assigned_scores = Variable(torch.LongTensor(assigned_scores), requires_grad=False) assigned_locations = Variable(torch.LongTensor(assigned_locations), requires_grad=False) if use_cuda: assigned_scores = assigned_scores.cuda() assigned_locations = assigned_locations.cuda() cls_scores = cls_scores.contiguous().view(-1, cls_scores.size()[-1]) assigned_scores = assigned_scores.contiguous().view(-1) cls_loss = F.cross_entropy(cls_scores, assigned_scores) if torch.sum(assigned_scores) > 0: # print("HAHA") assigned_head_positions = assigned_locations[:, :, 0] assigned_head_positions = assigned_head_positions.contiguous( ).view(-1) # assigned_tail_positions = assigned_locations[:, :, 1] assigned_tail_positions = assigned_tail_positions.contiguous( ).view(-1) head_pointer_probs = head_pointer_probs.contiguous().view( -1, head_pointer_probs.size()[-1]) tail_pointer_probs = tail_pointer_probs.contiguous().view( -1, tail_pointer_probs.size()[-1]) # mask here: if there is non in assigned scores, no need to compute ... assigned_head_positions = torch.masked_select( assigned_head_positions, assigned_scores.byte()) assigned_tail_positions = torch.masked_select( assigned_tail_positions, assigned_scores.byte()) head_pointer_probs = torch.index_select( head_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) tail_pointer_probs = torch.index_select( tail_pointer_probs, dim=0, index=assigned_scores.nonzero().squeeze(1)) assigned_head_positions = to_one_hot(assigned_head_positions, input_length) assigned_tail_positions = to_one_hot(assigned_tail_positions, input_length) prediction_head_loss = EMD_L2(head_pointer_probs, assigned_head_positions, needSoftMax=True) prediction_tail_loss = EMD_L2(tail_pointer_probs, assigned_tail_positions, needSoftMax=True) loc_losses.update( prediction_head_loss.data.item() + prediction_tail_loss.data.item(), batch_size) total_loss = args.alpha * (prediction_head_loss + prediction_tail_loss) + cls_loss else: total_loss = cls_loss # model_optim.zero_grad() # total_loss.backward() # torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) # model_optim.step() cls_losses.update(cls_loss.data.item(), batch_size) total_losses.update(total_loss.item(), batch_size) logger.info( "Val -- Epoch :{:06d}, LR: {:.6f},\tloss={:.4f}, \t c-loss:{:.4f}, \tloc-loss:{:.4f}\tcls-Accuracy:{:.4f}\tloc-Avg-IOU:{:.4f}\t topIOU:{:.4f}" .format(epoch, model_optim.param_groups[0]['lr'], total_losses.avg, cls_losses.avg, loc_losses.avg, Accuracy.avg, IOU.avg, ordered_IOU.avg)) if epoch % 1 == 0: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'loss': total_losses.avg, 'cls_loss': cls_losses.avg, 'loc_loss': loc_losses.avg, 'IoU': IOU.avg }, (epoch + 1), file_direcotry=save_places.model_save_dir)