def test(model, data_loader, criterion, posp, max_fpr): model.eval() targets, predicts = list(), list() loss = Averager() posp = torch.FloatTensor([posp]).cuda() one = torch.FloatTensor([1]).cuda() with torch.no_grad(): for j, (ids, values, seqlength, label, seq_mask) in enumerate(data_loader): ids, values = ids.cuda(), values.cuda() label = label.cuda().float() seq_mask = seq_mask.cuda() y, _ = model(ids, values, seqlength, seq_mask, 'tgt') p = posp * label + (one - posp) * (one - label) loss.add(torch.mean(p * criterion(y, label)).item()) targets.extend(label.tolist()) predicts.extend(y.tolist()) model.train() return roc_auc_score(targets, predicts, max_fpr=max_fpr), loss.item(), roc_auc_score( targets, predicts)
def validation(model, ctc_criterion, attn_criterion, evaluation_loader, ctc_converter, attn_converter, opt): """ validation or evaluation """ for p in model.parameters(): p.requires_grad = False n_correct = 0 norm_ED = 0 length_of_data = 0 infer_time = 0 valid_loss_avg = Averager() ctc_correct = 0 for i, (image_tensors, labels) in enumerate(evaluation_loader): batch_size = image_tensors.size(0) length_of_data = length_of_data + batch_size #image = image_tensors.cuda() image = image_tensors.to(device) length_for_pred = torch.IntTensor([opt.batch_max_length] * batch_size).to(device) text_for_pred = torch.LongTensor(batch_size, opt.batch_max_length + 1).fill_(0).to(device) ctc_text_for_loss, ctc_length_for_loss = ctc_converter.encode(labels) attn_text_for_loss, attn_length_for_loss = attn_converter.encode(labels) start_time = time.time() ctc_preds, attn_preds = model(image, text_for_pred) forward_time = time.time() - start_time # ctc ctc_preds = ctc_preds.log_softmax(2) # Calculate evaluation loss for CTC deocder. preds_size = torch.IntTensor([ctc_preds.size(1)] * batch_size) ctc_preds = ctc_preds.permute(1, 0, 2) # to use CTCloss format ctc_cost = ctc_criterion(ctc_preds, ctc_text_for_loss, preds_size, ctc_length_for_loss) # Select max probabilty (greedy decoding) then decode index to character _, preds_index = ctc_preds.max(2) preds_index = preds_index.transpose(1, 0).contiguous().view(-1) ctc_preds_str = ctc_converter.decode(preds_index.data, preds_size.data) # attention attn_preds = attn_preds[:, :attn_text_for_loss.shape[1] - 1, :] target = attn_text_for_loss[:, 1:] # without [GO] Symbol attn_cost = attn_criterion(attn_preds.contiguous().view(-1, attn_preds.shape[-1]), target.contiguous().view(-1)) # select max probabilty (greedy decoding) then decode index to character _, attn_preds_index = attn_preds.max(2) attn_preds_str = attn_converter.decode(attn_preds_index, length_for_pred) attn_labels = attn_converter.decode(attn_text_for_loss[:, 1:], attn_length_for_loss) cost = opt.ctc_weight * ctc_cost + (1.0 - opt.ctc_weight) * attn_cost infer_time += forward_time valid_loss_avg.add(cost) # calculate accuracy. #for attn_pred, attn_gt in zip(attn_preds_str, attn_labels): for pred, gt, attn_pred, attn_gt in zip(ctc_preds_str, labels, attn_preds_str, attn_labels): attn_pred = attn_pred[:attn_pred.find('[s]')] # prune after "end of sentence" token ([s]) attn_gt = attn_gt[:attn_gt.find('[s]')] if pred == gt: ctc_correct += 1 if attn_pred == attn_gt: n_correct += 1 norm_ED += edit_distance(attn_pred, attn_gt) / len(attn_gt) accuracy = n_correct / float(length_of_data) * 100 ctc_accuracy = ctc_correct / float(length_of_data) * 100 return valid_loss_avg.val(), accuracy, ctc_accuracy, norm_ED, attn_preds_str, attn_labels, infer_time, length_of_data
dim=1, largest=True, sorted=True) p = cfg.progalambda * (allExtraproto[topkindex] * topsim.unsqueeze(2).expand(cfg.way, cfg.topk, 1600)).sum(dim=1)/cfg.topk \ + (1 - cfg.progalambda) * proto else: p = proto logits = euclidean_metric(model(data_query), p) label = torch.arange(cfg.way).repeat(cfg.query) label = label.type(torch.cuda.LongTensor) acc = count_acc(logits, label) ave_acc.add(acc) print('batch {}: {:.2f}({:.2f})'.format(i, ave_acc.item() * 100, acc * 100)) allacc.append(acc) x = None p = None logits = None allacc = np.array(allacc) torch.save(allacc, cfg.result + '/allacc') mean, std, conf_intveral = CI(allacc)
def train(opt): """ dataset preparation """ opt.select_data = opt.select_data.split('-') opt.batch_ratio = opt.batch_ratio.split('-') train_dataset = Batch_Balanced_Dataset(opt) AlignCollate_valid = AlignCollate(imgH=opt.imgH, imgW=opt.imgW, keep_ratio_with_pad=opt.PAD) valid_dataset = hierarchical_dataset(root=opt.valid_data, opt=opt) valid_loader = torch.utils.data.DataLoader( valid_dataset, batch_size=opt.batch_size, shuffle=True, # 'True' to check training progress with validation function. num_workers=int(opt.workers), collate_fn=AlignCollate_valid, pin_memory=True) print('-' * 80) """ model configuration """ if 'CTC' in opt.Prediction: converter = CTCLabelConverter(opt.character) else: converter = AttnLabelConverter(opt.character) opt.num_class = len(converter.character) if opt.rgb: opt.input_channel = 3 model = Model(opt) print('model input parameters', opt.imgH, opt.imgW, opt.num_fiducial, opt.input_channel, opt.output_channel, opt.hidden_size, opt.num_class, opt.batch_max_length, opt.Transformation, opt.FeatureExtraction, opt.SequenceModeling, opt.Prediction) # weight initialization for name, param in model.named_parameters(): if 'localization_fc2' in name: print(f'Skip {name} as it is already initialized') continue try: if 'bias' in name: init.constant_(param, 0.0) elif 'weight' in name: init.kaiming_normal_(param) except Exception as e: # for batchnorm. if 'weight' in name: param.data.fill_(1) continue # data parallel for multi-GPU model = torch.nn.DataParallel(model).cuda() model.train() if opt.continue_model != '': if opt.without_prediction: load_model_without_prediction(opt.continue_model, model) print(f'loading pretrained model from {opt.continue_model}, without prediction layer') else: print(f'loading pretrained model from {opt.continue_model}') model.load_state_dict(torch.load(opt.continue_model)) print("Model:") print(model) """ setup loss """ if 'CTC' in opt.Prediction: criterion = torch.nn.CTCLoss(zero_infinity=True).cuda() else: criterion = torch.nn.CrossEntropyLoss(ignore_index=0).cuda() # ignore [GO] token = ignore index 0 # loss averager loss_avg = Averager() # filter that only require gradient decent filtered_parameters = [] params_num = [] for p in filter(lambda p: p.requires_grad, model.parameters()): filtered_parameters.append(p) params_num.append(np.prod(p.size())) print('Trainable params num : ', sum(params_num)) # [print(name, p.numel()) for name, p in filter(lambda p: p[1].requires_grad, model.named_parameters())] # setup optimizer if opt.adam: optimizer = optim.Adam(filtered_parameters, lr=opt.lr, betas=(opt.beta1, 0.999)) else: optimizer = optim.Adadelta(filtered_parameters, lr=opt.lr, rho=opt.rho, eps=opt.eps) print("Optimizer:") print(optimizer) """ final options """ # print(opt) with open(f'./saved_models/{opt.experiment_name}/opt.txt', 'a') as opt_file: opt_log = '------------ Options -------------\n' args = vars(opt) for k, v in args.items(): opt_log += f'{str(k)}: {str(v)}\n' opt_log += '---------------------------------------\n' print(opt_log) opt_file.write(opt_log) """ start training """ start_iter = 0 if opt.continue_model != '': print(f'continue to train, start_iter: {start_iter}') start_time = time.time() best_accuracy = -1 best_norm_ED = 1e+6 i = start_iter while True: # train part for p in model.parameters(): p.requires_grad = True image_tensors, labels = train_dataset.get_batch() image = image_tensors.cuda() text, length = converter.encode(labels) batch_size = image.size(0) if 'CTC' in opt.Prediction: preds = model(image, text).log_softmax(2) preds_size = torch.IntTensor([preds.size(1)] * batch_size) preds = preds.permute(1, 0, 2) # to use CTCLoss format cost = criterion(preds, text, preds_size, length) else: preds = model(image, text) target = text[:, 1:] # without [GO] Symbol cost = criterion(preds.view(-1, preds.shape[-1]), target.contiguous().view(-1)) model.zero_grad() cost.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), opt.grad_clip) # gradient clipping with 5 (Default) optimizer.step() loss_avg.add(cost) # validation part if i % opt.valInterval == 0: elapsed_time = time.time() - start_time logging.info(f'[{i}/{opt.num_iter}] Loss: {loss_avg.val():0.5f} elapsed_time: {elapsed_time:0.5f}') # for log with open(f'./saved_models/{opt.experiment_name}/log_train.txt', 'a') as log: log.write(f'[{i}/{opt.num_iter}] Loss: {loss_avg.val():0.5f} elapsed_time: {elapsed_time:0.5f}\n') loss_avg.reset() model.eval() valid_loss, current_accuracy, current_norm_ED, preds, labels, infer_time, length_of_data = validation( model, criterion, valid_loader, converter, opt) model.train() for pred, gt in zip(preds[:5], labels[:5]): if 'Attn' in opt.Prediction: pred = pred[:pred.find('[s]')] gt = gt[:gt.find('[s]')] print(f'{pred:20s}, gt: {gt:20s}, {str(pred == gt)}') log.write(f'{pred:20s}, gt: {gt:20s}, {str(pred == gt)}\n') valid_log = f'[{i}/{opt.num_iter}] valid loss: {valid_loss:0.5f}' valid_log += f' accuracy: {current_accuracy:0.3f}, norm_ED: {current_norm_ED:0.2f}' log.write(valid_log + '\n') # keep best accuracy model if current_accuracy > best_accuracy: best_accuracy = current_accuracy torch.save(model.state_dict(), f'./saved_models/{opt.experiment_name}/mtl_best_accuracy.pth') if current_norm_ED < best_norm_ED: best_norm_ED = current_norm_ED torch.save(model.state_dict(), f'./saved_models/{opt.experiment_name}/best_norm_ED.pth') best_model_log = f'best_accuracy: {best_accuracy:0.3f}, best_norm_ED: {best_norm_ED:0.2f}' logging.info(best_model_log) log.write(best_model_log + '\n') # save model per 1e+5 iter. if (i + 1) % 50000 == 0: torch.save( model.state_dict(), f'./saved_models/{opt.experiment_name}/iter_{i+1}.pth') if i == opt.num_iter: logging.info('end the training') sys.exit() i += 1
def train(opt): #logging.info(opt) train_dataset = Batch_Dataset(opt) AlignCollate_valid = AlignCollate(imgH=opt.imgH, imgW=opt.imgW, keep_ratio_with_pad=opt.PAD) valid_dataset = LmdbDataset(root=opt.valid_data, opt=opt) valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=int(opt.workers), collate_fn=AlignCollate_valid, pin_memory=True) print('-' * 80) """ model configuration """ ctc_converter = CTCLabelConverter(opt.character, opt.subword) attn_converter = AttnLabelConverter(opt.character, opt.subword, opt.batch_max_length) opt.num_class = len(attn_converter.character) opt.ctc_num_class = len(ctc_converter.character) print("ctc num class {}".format(len(ctc_converter.character))) print("attention num class {}".format(len(attn_converter.character))) if opt.rgb: opt.input_channel = 3 model = MyModel(opt) # weight initialization for name, param in model.named_parameters(): if 'localization_fc2' in name: print('Skip {name} as it is already initialized'.format(name)) continue try: if 'bias' in name: init.constant_(param, 0.0) elif 'weight' in name: init.kaiming_normal_(param) except Exception as e: if 'weight' in name: param.data.fill_(1) continue model = torch.nn.DataParallel(model).to(device) model.train() if opt.continue_model != '': print('loading pretrained model from {}'.format(opt.continue_model)) model.load_state_dict(torch.load(opt.continue_model)) """ setup loss """ ctc_criterion = torch.nn.CTCLoss(zero_infinity=True).to(device) attn_criterion = torch.nn.CrossEntropyLoss(ignore_index=0).to(device) loss_avg = Averager() filtered_parameters = [] params_num = [] for p in filter(lambda p: p.requires_grad, model.parameters()): filtered_parameters.append(p) params_num.append(np.prod(p.size())) print('Trainable params num : ', sum(params_num)) if opt.adam: optimizer = optim.Adam(filtered_parameters, lr=opt.lr, betas=(opt.beta1, 0.999)) else: optimizer = optim.Adadelta(filtered_parameters, lr=opt.lr, rho=opt.rho, eps=opt.eps) print("Optimizer:") print(optimizer) """ final options """ with open(osj(opt.outPath, '{}/opt.txt'.format(opt.experiment_name)), 'a') as opt_file: opt_log = '------------ Options -------------\n' args = vars(opt) for k, v in args.items(): opt_log += '{}: {}\n'.format(str(k), str(v)) opt_log += '---------------------------------------\n' print(opt_log) opt_file.write(opt_log) """ start training """ start_iter = 0 if opt.continue_model != '': print('continue to train, start_iter: {}'.format(start_iter)) start_time = time.time() best_accuracy = -1 i = start_iter while True: # train part for p in model.parameters(): p.requires_grad = True image_tensors, labels = train_dataset.get_batch() image = image_tensors.to(device) ctc_text, ctc_length = ctc_converter.encode(labels) attn_text, attn_length = attn_converter.encode(labels) batch_size = image.size(0) # ctc loss ctc_preds, attn_preds = model(image, attn_text) ctc_preds = ctc_preds.log_softmax(2) preds_size = torch.IntTensor([ctc_preds.size(1)] * batch_size) ctc_preds = ctc_preds.permute(1, 0, 2) ctc_cost = ctc_criterion(ctc_preds, ctc_text, preds_size, ctc_length) # attn loss target = attn_text[:, 1:] attn_cost = attn_criterion(attn_preds.view(-1, attn_preds.shape[-1]), target.contiguous().view(-1)) cost = opt.ctc_weight * ctc_cost + (1.0 - opt.ctc_weight) * attn_cost model.zero_grad() cost.backward() torch.nn.utils.clip_grad_norm_( model.parameters(), opt.grad_clip) # gradient clipping with 5 (Default) optimizer.step() loss_avg.add(cost) # validation part if i % opt.valInterval == 0: elapsed_time = time.time() - start_time logging.info('[{}/{}] Loss: {:0.5f} elapsed_time: {:0.5f}'.format( i, opt.num_iter, loss_avg.val(), elapsed_time)) # for log with open( osj(opt.outPath, '{}/log_train.txt'.format(opt.experiment_name)), 'a') as log: log.write( '[{}/{}] Loss: {:0.5f} elapsed_time: {:0.5f}\n'.format( i, opt.num_iter, loss_avg.val(), elapsed_time)) loss_avg.reset() model.eval() with torch.no_grad(): valid_loss, current_accuracy, ctc_accuracy, current_norm_ED, preds, labels, infer_time, length_of_data \ = mtl_validation(model, ctc_criterion, attn_criterion, valid_loader, ctc_converter, attn_converter, opt) model.train() for pred, gt in zip(preds[:5], labels[:5]): pred = pred[:pred.find('[s]')] gt = gt[:gt.find('[s]')] print('{:20s}, gt: {:20s}, {}'.format( pred, gt, str(pred == gt))) log.write('{:20s}, gt: {:20s}, {}\n'.format( pred, gt, str(pred == gt))) valid_log = '[{}/{}] valid loss: {:0.5f}'.format( i, opt.num_iter, valid_loss) valid_log += ' accuracy: {:0.3f}'.format(current_accuracy) log.write(valid_log + '\n') # save best accuracy model if current_accuracy > best_accuracy: best_accuracy = current_accuracy torch.save( model.state_dict(), osj(opt.outPath, '{}/best_accuracy.pth'.format( opt.experiment_name))) best_model_log = 'best_accuracy: {:0.3f}'.format(best_accuracy) logging.info(best_model_log) log.write(best_model_log + '\n') if (i + 1) % 50000 == 0: torch.save( model.state_dict(), osj(opt.outPath, '{}/iter_{}.pth'.format(opt.experiment_name, i + 1))) if i == opt.num_iter: logging.info('end the training') sys.exit() i += 1
def validation(model, criterion, evaluation_loader, converter, opt): """ validation or evaluation """ for p in model.parameters(): p.requires_grad = False n_correct = 0 norm_ED = 0 length_of_data = 0 infer_time = 0 valid_loss_avg = Averager() for i, (image_tensors, labels) in enumerate(evaluation_loader): batch_size = image_tensors.size(0) length_of_data = length_of_data + batch_size with torch.no_grad(): image = image_tensors.cuda() # For max length prediction length_for_pred = torch.cuda.IntTensor([opt.batch_max_length] * batch_size) text_for_pred = torch.cuda.LongTensor( batch_size, opt.batch_max_length + 1).fill_(0) text_for_loss, length_for_loss = converter.encode(labels) start_time = time.time() if 'CTC' in opt.Prediction: preds = model(image, text_for_pred).log_softmax(2) forward_time = time.time() - start_time # Calculate evaluation loss for CTC deocder. preds_size = torch.IntTensor([preds.size(1)] * batch_size) preds = preds.permute(1, 0, 2) # to use CTCloss format cost = criterion(preds, text_for_loss, preds_size, length_for_loss) # Select max probabilty (greedy decoding) then decode index to character _, preds_index = preds.max(2) preds_index = preds_index.transpose(1, 0).contiguous().view(-1) preds_str = converter.decode(preds_index.data, preds_size.data) else: preds = model(image, text_for_pred, is_train=False) forward_time = time.time() - start_time preds = preds[:, :text_for_loss.shape[1] - 1, :] target = text_for_loss[:, 1:] # without [GO] Symbol cost = criterion(preds.contiguous().view(-1, preds.shape[-1]), target.contiguous().view(-1)) # select max probabilty (greedy decoding) then decode index to character _, preds_index = preds.max(2) preds_str = converter.decode(preds_index, length_for_pred) labels = converter.decode(text_for_loss[:, 1:], length_for_loss) infer_time += forward_time valid_loss_avg.add(cost) # calculate accuracy. for pred, gt in zip(preds_str, labels): if 'Attn' in opt.Prediction: pred = pred[:pred.find( '[s]')] # prune after "end of sentence" token ([s]) gt = gt[:gt.find('[s]')] if pred == gt: n_correct += 1 norm_ED += edit_distance(pred, gt) / len(gt) accuracy = n_correct / float(length_of_data) * 100 return valid_loss_avg.val( ), accuracy, norm_ED, preds_str, labels, infer_time, length_of_data
sorted=True) p = p + cfg.progalambda * ( allExtraproto[topkindex] * topsim.unsqueeze(2).expand( cfg.train_way, cfg.topk, 1600)).sum(dim=1) / cfg.topk label = torch.arange(cfg.train_way).repeat(cfg.query) label = label.type(torch.cuda.LongTensor) logits = euclidean_metric(model(data_query), proto) loss = F.cross_entropy(logits, label) acc = count_acc(logits, label) print('epoch {}, train {}/{}, loss={:.4f} acc={:.4f}'.format( epoch, i, len(train_loader), loss.item(), acc)) tl.add(loss.item()) ta.add(acc) optimizer.zero_grad() loss.backward() optimizer.step() p = None proto = None logits = None loss = None tl = tl.item() ta = ta.item() torch.cuda.empty_cache()
def train(model, optimizer, src_loader, tgt_loader, valid_loader, criterion, log_interval=1000, val_interval=50, posp=1, nagp=0.5, params_cls=0.5, params_da=0.5, da_type='cmmd', max_fpr=0.01): global max_auc global max_auchead global min_loss posp = torch.FloatTensor([posp]).cuda() nagp = torch.FloatTensor([nagp]).cuda() one = torch.FloatTensor([1]).cuda() iter_src = iter(src_loader) iter_tgt = iter(tgt_loader) num_iter = len(src_loader) stoper = Stoper() avg_all_loss = Averager() avg_src_loss = Averager() avg_tgt_loss = Averager() avg_da_loss = Averager() start_time = time.time() for i in range(1, num_iter * 20): model.train() src_ids, src_values, src_seqlength, src_label, src_seq_mask = iter_src.next( ) src_ids, src_values, src_label = src_ids.cuda(), src_values.cuda( ), src_label.cuda().float() src_seq_mask = src_seq_mask.cuda() if i % len(src_loader) == 0: iter_src = iter(src_loader) if i % len(tgt_loader) == 0: iter_tgt = iter(tgt_loader) src_p = posp * src_label + nagp * (one - src_label) src_y, src_fea_LSTM = model(src_ids, src_values, src_seqlength, src_seq_mask, 'src') src_loss = torch.mean( src_p * criterion(src_y, src_label) ) # + torch.mean(src_p * criterion(src_spey, src_label)) tgt_ids, tgt_values, tgt_seqlength, tgt_label, tgt_seq_mask = iter_tgt.next( ) tgt_ids, tgt_values, tgt_label = tgt_ids.cuda(), tgt_values.cuda( ), tgt_label.cuda().float() tgt_seq_mask = tgt_seq_mask.cuda() # print(tgt_seqlength, tgt_label) tgt_p = posp * tgt_label + nagp * (one - tgt_label) tgt_y, tgt_fea_LSTM, tgt_spey = model(tgt_ids, tgt_values, tgt_seqlength, tgt_seq_mask, 'tgt') tgt_loss = torch.mean( tgt_p * criterion(tgt_y, tgt_label) ) # + 0.5 * torch.mean(tgt_p * criterion(tgt_spey, tgt_label)) if da_type == 'cmmd': da_loss = cmmd(src_fea_LSTM, tgt_fea_LSTM, src_label.long(), tgt_label.long()) elif da_type == 'mmd': da_loss = mmd_rbf_noaccelerate(src_fea_LSTM, tgt_fea_LSTM) elif da_type == 'coral': da_loss = coral(src_fea_LSTM, tgt_fea_LSTM) elif da_type == 'euclidian': da_loss = euclidian(src_fea_LSTM, tgt_fea_LSTM) elif da_type == 'c_euclidian': da_loss = c_euclidian(src_fea_LSTM, tgt_fea_LSTM, src_label.long(), tgt_label.long()) elif da_type == 'nometric': da_loss = nometric(src_fea_LSTM, tgt_fea_LSTM) elif da_type == 'ced': da_loss = ced(src_fea_LSTM, tgt_fea_LSTM, src_label.long(), tgt_label.long()) lambd = 2 / (1 + math.exp((-5 * i) / (len(src_loader)))) - 1 loss = params_cls * src_loss + tgt_loss + params_da * lambd * da_loss model.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5) optimizer.step() avg_all_loss.add(loss.item()) avg_src_loss.add(src_loss.item()) avg_tgt_loss.add(tgt_loss.item()) avg_da_loss.add(da_loss.item()) if (i + 1) % log_interval == 0: print( 'step: {}, loss: {:.4f}, src_loss: {:.4f}, tgt_loss: {:.4f}, {}_loss:, {:.4f}, lambda: {}' .format(i + 1, avg_all_loss.item(), avg_src_loss.item(), avg_tgt_loss.item(), da_type, avg_da_loss.item(), lambd)) avg_all_loss = Averager() avg_src_loss = Averager() avg_tgt_loss = Averager() avg_da_loss = Averager() if (i + 1) % val_interval == 0: end_time = time.time() print('train time (s):', end_time - start_time) start_time = time.time() auc_head, loss, auc = test(model, valid_loader, criterion, posp, max_fpr) if loss < min_loss: min_loss = loss if auc > max_auc: max_auc = auc if auc_head > max_auchead: torch.save(model, f'{save_dir}/tmp.pt') max_auchead = auc_head print( 'dev --- auchead: {:.4f}, max_auchead: {:.4f}, auc: {:.4f}, max_auc: {:.4f}, loss: {:.4f}, minloss: {:.4f}' .format(auc_head, max_auchead, auc, max_auc, loss, min_loss)) end_time = time.time() print('dev time (s):', end_time - start_time) start_time = time.time() if stoper.add(auc_head): print('training end') break