def main(test_file, vocab_file, embeddings_file, pretrained_file, max_length=50, gpu_index=0, batch_size=128): """ Test the ESIM model with pretrained weights on some dataset. Args: test_file: The path to a file containing preprocessed NLI data. pretrained_file: The path to a checkpoint produced by the 'train_model' script. vocab_size: The number of words in the vocabulary of the model being tested. embedding_dim: The size of the embeddings in the model. hidden_size: The size of the hidden layers in the model. Must match the size used during training. Defaults to 300. num_classes: The number of classes in the output of the model. Must match the value used during training. Defaults to 3. batch_size: The size of the batches used for testing. Defaults to 32. """ device = torch.device("cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for testing ", 20 * "=") if platform == "linux" or platform == "linux2": checkpoint = torch.load(pretrained_file) else: checkpoint = torch.load(pretrained_file, map_location=device) # Retrieving model parameters from checkpoint. hidden_size = checkpoint["model"]["projection.0.weight"].size(0) num_classes = checkpoint["model"]["classification.6.weight"].size(0) embeddings = load_embeddings(embeddings_file) print("\t* Loading test data...") test_data = LCQMC_Dataset(test_file, vocab_file, max_length) test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size) print("\t* Building model...") model = ESIM(hidden_size, embeddings=embeddings, num_classes=num_classes, device=device).to(device) model.load_state_dict(checkpoint["model"]) print(20 * "=", " Testing ESIM model on device: {} ".format(device), 20 * "=") batch_time, total_time, accuracy, auc = test(model, test_loader) print("\n-> Average batch processing time: {:.4f}s, total test time: {:.4f}s, accuracy: {:.4f}%, auc: {:.4f}\n".format(batch_time, total_time, (accuracy*100), auc))
def model_load_test(test_df, vocab_file, embeddings_file, pretrained_file, test_prediction_dir, test_prediction_name, mode, num_labels, max_length=50, gpu_index=0, batch_size=128): device = torch.device( "cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for testing ", 20 * "=") if platform == "linux" or platform == "linux2": checkpoint = torch.load(pretrained_file, map_location=device) else: checkpoint = torch.load(pretrained_file, map_location=device) # Retrieving model parameters from checkpoint. hidden_size = checkpoint["model"]["projection.0.weight"].size(0) num_classes = checkpoint["model"]["classification.6.weight"].size(0) embeddings = load_embeddings(embeddings_file) print("\t* Loading test data...") test_data = My_Dataset(test_df, vocab_file, max_length, mode) test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size) print("\t* Building model...") model = ESIM(hidden_size, embeddings=embeddings, num_labels=num_labels, device=device).to(device) model.load_state_dict(checkpoint["model"]) print(20 * "=", " Testing ESIM model on device: {} ".format(device), 20 * "=") batch_time, total_time, accuracy, predictions = test(model, test_loader) print( "\n-> Average batch processing time: {:.4f}s, total test time: {:.4f}s, accuracy: {:.4f}%\n" .format(batch_time, total_time, (accuracy * 100))) test_prediction = pd.DataFrame({'prediction': predictions}) if not os.path.exists(test_prediction_dir): os.makedirs(test_prediction_dir) test_prediction.to_csv(os.path.join(test_prediction_dir, test_prediction_name), index=False)
def main(args): print(20 * "=", " Preparing for training ", 20 * "=") if not os.path.exists(args.result): os.makedirs(args.result) # -------------------- Loda pretraining model ------------------- # checkpoints = torch.load(args.pretrained_file) # 可以从模型中直接恢复,也可以直接在前面定义 Retrieving model parameters from checkpoint. # hidden_size = checkpoints["model"]["projection.0.weight"].size(0) # num_classes = checkpoints["model"]["classification.6.weight"].size(0) # -------------------- Data loading ------------------- # print("\t* Loading training data...") test_data = LCQMC_dataset(args.test_file, args.vocab_file, args.max_length, test_flag=True) test_loader = DataLoader(test_data, batch_size=args.batch_size) # -------------------- Model definition ------------------- # print("\t* Building model...") embeddings = load_embeddings(args.embed_file) model = ESIM(args, embeddings=embeddings).to(args.device) model.load_state_dict(checkpoints["model"]) print(20 * "=", " Testing ESIM model on device: {} ".format(args.device), 20 * "=") all_predict = predict(model, test_loader) index = np.array([], dtype=int) for i in range(len(all_predict)): index = np.append(index, i) # ---------------------生成文件-------------------------- df_test = pd.DataFrame(columns=['index', 'prediction']) df_test['index'] = index df_test['prediction'] = all_predict df_test.to_csv(args.submit_example_path, index=False, columns=['index', 'prediction'], sep='\t')
def main(train_file, dev_file, vocab_file, target_dir, max_length=50, hidden_size=300, dropout=0.2, num_classes=2, epochs=1, batch_size=256, lr=0.0005, patience=5, max_grad_norm=10.0, gpu_index=0, checkpoint=None): #device = torch.device("cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") device = torch.device("cpu") print(20 * "=", " Preparing for training ", 20 * "=") # 保存模型的路径 if not os.path.exists(target_dir): os.makedirs(target_dir) # -------------------- Data loading ------------------- # print("\t* Loading training data...") train_data = LCQMC_Dataset(train_file, vocab_file, max_length) train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size) print("\t* Loading validation data...") dev_data = LCQMC_Dataset(dev_file, vocab_file, max_length) dev_loader = DataLoader(dev_data, shuffle=True, batch_size=batch_size) # -------------------- Model definition ------------------- # print("\t* Building model...") # embeddings = load_embeddings(embeddings_file) model = ESIM(hidden_size, dropout=dropout, num_labels=num_classes, device=device).to(device) # -------------------- Preparation for training ------------------- # print('a') criterion = nn.CrossEntropyLoss() # 过滤出需要梯度更新的参数 parameters = filter(lambda p: p.requires_grad, model.parameters()) print('b') # optimizer = optim.Adadelta(parameters, params["LEARNING_RATE"]) optimizer = torch.optim.Adam(parameters, lr=lr) # optimizer = torch.optim.Adam(model.parameters(), lr=lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", factor=0.85, patience=0) best_score = 0.0 start_epoch = 1 # Data for loss curves plot epochs_count = [] train_losses = [] valid_losses = [] # Continuing training from a checkpoint if one was given as argument if checkpoint: checkpoint = torch.load(checkpoint) start_epoch = checkpoint["epoch"] + 1 best_score = checkpoint["best_score"] print("\t* Training will continue on existing model from epoch {}...". format(start_epoch)) model.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) epochs_count = checkpoint["epochs_count"] train_losses = checkpoint["train_losses"] valid_losses = checkpoint["valid_losses"] # Compute loss and accuracy before starting (or resuming) training. _, valid_loss, valid_accuracy, auc = validate(model, dev_loader, criterion) print( "\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}" .format(valid_loss, (valid_accuracy * 100), auc)) # -------------------- Training epochs ------------------- # print("\n", 20 * "=", "Training ESIM model on device: {}".format(device), 20 * "=") patience_counter = 0 for epoch in range(start_epoch, epochs + 1): epochs_count.append(epoch) print("* Training epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader, optimizer, criterion, epoch, max_grad_norm) train_losses.append(epoch_loss) print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) print("* Validation for epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy, epoch_auc = validate( model, dev_loader, criterion) valid_losses.append(epoch_loss) print( "-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}\n" .format(epoch_time, epoch_loss, (epoch_accuracy * 100), epoch_auc)) # Update the optimizer's learning rate with the scheduler. scheduler.step(epoch_accuracy) # Early stopping on validation accuracy. if epoch_accuracy < best_score: patience_counter += 1 else: best_score = epoch_accuracy patience_counter = 0 torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses }, os.path.join(target_dir, "best.pth.tar")) # Save the model at each epoch. torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "optimizer": optimizer.state_dict(), "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses }, os.path.join(target_dir, "esim_{}.pth.tar".format(epoch))) if patience_counter >= patience: print("-> Early stopping: patience limit reached, stopping...") break
def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument( "--data_dir", default=None, type=str, required=True, help= "The input data dir. Should contain the .tsv files (or other data files) for the task." ) parser.add_argument("--embeddings_file", default=None, type=str, required=True) parser.add_argument("--output_dir", default=None, type=str, required=True) parser.add_argument("--train_language", default=None, type=str, required=True) parser.add_argument("--train_steps", default=-1, type=int, required=True) parser.add_argument("--eval_steps", default=-1, type=int, required=True) parser.add_argument( "--load_word2vec", action='store_true', help= 'if true, load word2vec file for the first time; if false, load generated word-vector csv file' ) parser.add_argument("--generate_word2vec_csv", action='store_true', help='if true, generate word2vec csv file') ## normal parameters parser.add_argument("--embedding_size", default=300, type=int) parser.add_argument("--query_maxlen", default=30, type=int) parser.add_argument("--hidden_size", default=300, type=int) parser.add_argument("--learning_rate", default=5e-4, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_classes", default=2, type=int) parser.add_argument("--dropout", default=0.2, type=float) parser.add_argument("--do_test", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--do_eval_train", action='store_true', help="Whether to run eval on the train set.") parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--per_gpu_eval_batch_size", default=10, type=int) parser.add_argument("--per_gpu_train_batch_size", default=10, type=int) parser.add_argument("--seed", default=1, type=int) parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--gradient_accumulation_steps", default=1, type=int) args = parser.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") args.n_gpu = torch.cuda.device_count() # device = torch.device("cpu") args.device = device # Set seed set_seed(args) logger.info("Training/evaluation parameters %s", args) args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Training if args.do_train: # build model logger.info("*** building model ***") embeddings = load_embeddings(args) model = ESIM(args.hidden_size, embeddings=embeddings, dropout=args.dropout, num_classes=args.num_classes, device=args.device) model.to(args.device) if args.n_gpu > 1: model = torch.nn.DataParallel(model) args.train_batch_size = args.per_gpu_train_batch_size * max( 1, args.n_gpu) logger.info("*** Loading training data ***") train_data = ATEC_Dataset(os.path.join(args.data_dir, 'train.csv'), os.path.join(args.data_dir, 'vocab.csv'), args.query_maxlen) train_loader = DataLoader(train_data, shuffle=True, batch_size=args.train_batch_size) logger.info("*** Loading validation data ***") dev_data = ATEC_Dataset(os.path.join(args.data_dir, 'dev.csv'), os.path.join(args.data_dir, 'vocab.csv'), args.query_maxlen) dev_loader = DataLoader(dev_data, shuffle=False, batch_size=args.eval_batch_size) num_train_optimization_steps = args.train_steps # 过滤出需要梯度更新的参数 parameters = filter(lambda p: p.requires_grad, model.parameters()) # optimizer = optim.Adadelta(parameters, params["LEARNING_RATE"]) optimizer = torch.optim.Adam(parameters, lr=args.learning_rate) # optimizer = torch.optim.Adam(model.parameters(), lr=lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", factor=0.85, patience=0) criterion = nn.CrossEntropyLoss() global_step = 0 logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_data)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) logger.info(" Num steps = %d", num_train_optimization_steps) best_acc = 0 model.train() tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 bar = tqdm(range(num_train_optimization_steps), total=num_train_optimization_steps) train_loader = cycle(train_loader) output_dir = args.output_dir + "eval_results_{}_{}_{}_{}_{}_{}".format( 'ESIM', str(args.query_maxlen), str(args.learning_rate), str(args.train_batch_size), str(args.train_language), str(args.train_steps)) try: os.makedirs(output_dir) except: pass output_eval_file = os.path.join(output_dir, 'eval_result.txt') with open(output_eval_file, "w") as writer: writer.write('*' * 80 + '\n') for step in bar: batch = next(train_loader) batch = tuple(t.to(device) for t in batch) q1, q1_lens, q2, q2_lens, labels = batch # 正常训练 optimizer.zero_grad() logits, probs = model(q1, q1_lens, q2, q2_lens) loss = criterion(logits, labels) if args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps tr_loss += loss.item() train_loss = round( tr_loss * args.gradient_accumulation_steps / (nb_tr_steps + 1), 4) bar.set_description("loss {}".format(train_loss)) nb_tr_examples += q1.size(0) nb_tr_steps += 1 loss.backward() # 对抗训练 # fgm.attack() # 在embedding上添加对抗扰动 # loss_adv = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids) # if args.n_gpu > 1: # loss_adv = loss_adv.mean() # mean() to average on multi-gpu. # if args.gradient_accumulation_steps > 1: # loss_adv = loss_adv / args.gradient_accumulation_steps # loss_adv.backward() # 反向传播,并在正常的grad基础上,累加对抗训练的梯度 # fgm.restore() # 恢复embedding参数 if (nb_tr_steps + 1) % args.gradient_accumulation_steps == 0: # scheduler.step() optimizer.step() global_step += 1 if (step + 1) % (args.eval_steps * args.gradient_accumulation_steps) == 0: tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 logger.info("***** Report result *****") logger.info(" %s = %s", 'global_step', str(global_step)) logger.info(" %s = %s", 'train loss', str(train_loss)) if args.do_eval and (step + 1) % ( args.eval_steps * args.gradient_accumulation_steps) == 0: if args.do_eval_train: file_list = ['train.csv', 'dev.csv'] else: file_list = ['dev.csv'] for file in file_list: inference_labels = [] gold_labels = [] inference_logits = [] logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(dev_data)) logger.info(" Batch size = %d", args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for q1, q1_lens, q2, q2_lens, labels in tqdm(dev_loader): with torch.no_grad(): logits, probs = model(q1, q1_lens, q2, q2_lens) probs = probs.detach().cpu().numpy() # print(logits.shape, probs.shape) # label_ids = labels.to('cpu').numpy() inference_labels.append(np.argmax(probs, 1)) gold_labels.append(labels) # eval_loss += tmp_eval_loss.mean().item() nb_eval_examples += logits.size(0) nb_eval_steps += 1 gold_labels = np.concatenate(gold_labels, 0) inference_labels = np.concatenate(inference_labels, 0) model.train() eval_loss = eval_loss / nb_eval_steps eval_accuracy = get_f1(inference_labels, gold_labels) result = { # 'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, 'train_loss': train_loss } if 'dev' in file: with open(output_eval_file, "a") as writer: writer.write(file + '\n') for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) writer.write('*' * 80) writer.write('\n') if eval_accuracy > best_acc and 'dev' in file: print("=" * 80) print("Best ACC", eval_accuracy) print("Saving Model......") best_acc = eval_accuracy # Save a trained model model_to_save = model.module if hasattr( model, 'module') else model # Only save the model it-self output_model_file = os.path.join( output_dir, "pytorch_model.bin") torch.save(model_to_save.state_dict(), output_model_file) print("=" * 80) else: print("=" * 80) with open(output_eval_file, "a") as writer: writer.write('bert_acc: %f' % best_acc) if args.do_test: if args.do_train == False: output_dir = args.output_dir # build model logger.info("*** building model ***") embeddings = load_embeddings(args) model = ESIM(args.hidden_size, embeddings=embeddings, dropout=args.dropout, num_classes=args.num_classes, device=args.device) model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'))) model.to(args.device) if args.n_gpu > 1: model = torch.nn.DataParallel(model) inference_labels = [] gold_labels = [] logger.info("*** Loading testing data ***") dev_data = ATEC_Dataset(os.path.join(args.data_dir, 'test.csv'), os.path.join(args.data_dir, 'vocab.csv'), args.query_maxlen) dev_loader = DataLoader(dev_data, shuffle=False, batch_size=args.eval_batch_size) logger.info(" *** Run Prediction ***") logger.info(" Num examples = %d", len(dev_data)) logger.info(" Batch size = %d", args.eval_batch_size) model.eval() for q1, q1_lens, q2, q2_lens, labels in tqdm(dev_loader): with torch.no_grad(): logits, probs = model(q1, q1_lens, q2, q2_lens) probs = probs.detach().cpu().numpy() inference_labels.append(np.argmax(probs, 1)) gold_labels.append(labels) gold_labels = np.concatenate(gold_labels, 0) logits = np.concatenate(inference_labels, 0) test_f1 = get_f1(logits, gold_labels) logger.info('predict f1:{}'.format(str(test_f1)))
trainer_config = { 'optimizer': optimizer, 'batch_size': args.batch_size, 'log_interval': args.log_interval, 'model_outfile': args.model_outfile, 'lr_reduce_factor': args.lr_reduce_factor, 'patience': args.patience, 'tensorboard': args.tensorboard, 'run_label': args.run_label, 'logger': logger } trainer = TrainerFactory.get_trainer(args.dataset, model, embedding, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) if not args.skip_training: total_params = 0 for param in model.parameters(): size = [s for s in param.size()] total_params += np.prod(size) logger.info('Total number of parameters: %s', total_params) trainer.train(args.epochs) _, _, state_dict, _, _ = load_checkpoint(args.model_outfile) for k, tensor in state_dict.items(): state_dict[k] = tensor.to(device) model.load_state_dict(state_dict) if dev_loader: evaluate_dataset('dev', dataset_cls, model, embedding, dev_loader, args.batch_size, args.device) evaluate_dataset('test', dataset_cls, model, embedding, test_loader, args.batch_size, args.device, args.keep_results)
def main(): device = args.device print(20 * "=", " Preparing for training ", 20 * "=") # 保存模型的路径 if not os.path.exists(args.target_dir): os.makedirs(args.target_dir) # -------------------- Data loading ------------------- # print("Loading data......") train_loader, dev_loader, test_loader, SEN1, SEN2 = load_data( args.batch_size, device) embedding = SEN1.vectors vocab_size = len(embedding) print("vocab_size:", vocab_size) # -------------------- Model definition ------------------- # print("\t* Building model...") model = ESIM(args.hidden_size, embedding=embedding, dropout=args.dropout, num_labels=args.num_classes, device=device).to(device) # -------------------- Preparation for training ------------------- # criterion = nn.CrossEntropyLoss() # 过滤出需要梯度更新的参数 parameters = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameters, lr=args.lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", factor=0.1, patience=10) best_score = 0.0 if args.ckp: checkpoint = torch.load(os.path.join(args.target_dir, args.ckp)) best_score = checkpoint["best_score"] model.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) _, valid_loss, valid_accuracy = validate(model, dev_loader, criterion) print("\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%". format(valid_loss, (valid_accuracy * 100))) # -------------------- Training epochs ------------------- # print("\n", 20 * "=", "Training ESIM model on device: {}".format(device), 20 * "=") patience_counter = 0 for epoch in range(args.num_epoch): print("* Training epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader, optimizer, criterion, args.max_grad_norm, device) print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) print("* Validation for epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = validate( model, dev_loader, criterion, device) print("-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%\n". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) # Update the optimizer's learning rate with the scheduler. scheduler.step(epoch_accuracy) # Early stopping on validation accuracy. if epoch_accuracy < best_score: patience_counter += 1 else: print("save model!!!!") best_score = epoch_accuracy patience_counter = 0 torch.save( { "model": model.state_dict(), "best_score": best_score, "optimizer": optimizer.state_dict(), }, os.path.join(args.target_dir, "best.pth.tar")) if patience_counter >= 5: print("-> Early stopping: patience limit reached, stopping...") break # ##-------------------- Testing epochs ------------------- # # print(20 * "=", " Testing ", 20 * "=") # if args.ckp: # checkpoint = torch.load(os.path.join(args.target_dir, args.ckp)) # best_score = checkpoint["best_score"] # model.load_state_dict(checkpoint["model"]) # optimizer.load_state_dict(checkpoint["optimizer"]) # # print("best_score:", best_score) # all_labels = test(model, test_loader, device) # print(all_labels[:10]) # target_label = [id2label[id] for id in all_labels] # print(target_label[:10]) # with open(os.path.join(args.target_dir, 'result.txt'), 'w+') as f: # for label in target_label: # f.write(label + '\n') del train_loader del dev_loader del test_loader del SEN1 del SEN2 del embedding
print('\tTrain Loss: %.3f | Train Acc: %.2f %%' % (train_loss, train_acc * 100)) print('\t Val. Loss: %.3f | Val. Acc: %.2f %%' % (valid_loss, valid_acc * 100)) if valid_loss < best_valid_loss: best_valid_loss = valid_loss best_valid_acc = valid_acc torch.save(model.state_dict(), './saved_model/esim.pt') print("New model saved!") f_log.write("New model saved!\n") f_log.flush() f_log.close() model.load_state_dict(torch.load('./saved_model/esim.pt')) model.eval() f_valid = open("data/test-set.data", "r", encoding='utf-8') f_res = open('prediction.txt', 'w') for i, rowlist in enumerate(f_valid): rowlist = rowlist[:-1].split('\t') input_sent = [] for sent in rowlist[:2]: tokenized = tokenizer(sent) indexed = [TEXT.vocab.stoi[t] for t in tokenized] tensor = torch.LongTensor(indexed).to(device) tensor = tensor.unsqueeze(1) input_sent.append(tensor) ans = F.softmax(model(input_sent[0], input_sent[1])[0])[1].item() f_res.write(str(ans) + '\n')
def model_train_validate_test(train_df, dev_df, test_df, embeddings_file, vocab_file, target_dir, mode, num_labels=2, max_length=50, hidden_size=200, dropout=0.2, epochs=50, batch_size=256, lr=0.0005, patience=5, max_grad_norm=10.0, gpu_index=0, if_save_model=False, checkpoint=None): device = torch.device( "cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for training ", 20 * "=") # 保存模型的路径 if not os.path.exists(target_dir): os.makedirs(target_dir) # -------------------- Data loading ------------------- # print("\t* Loading training data...") train_data = My_Dataset(train_df, vocab_file, max_length, mode) train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size) print("\t* Loading validation data...") dev_data = My_Dataset(dev_df, vocab_file, max_length, mode) dev_loader = DataLoader(dev_data, shuffle=True, batch_size=batch_size) print("\t* Loading test data...") test_data = My_Dataset(test_df, vocab_file, max_length, mode) test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size) # -------------------- Model definition ------------------- # print("\t* Building model...") if (embeddings_file is not None): embeddings = load_embeddings(embeddings_file) else: embeddings = None model = ESIM(hidden_size, embeddings=embeddings, dropout=dropout, num_labels=num_labels, device=device).to(device) total_params = sum(p.numel() for p in model.parameters()) print(f'{total_params:,} total parameters.') total_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print(f'{total_trainable_params:,} training parameters.') # -------------------- Preparation for training ------------------- # criterion = nn.CrossEntropyLoss() # 过滤出需要梯度更新的参数 parameters = filter(lambda p: p.requires_grad, model.parameters()) # optimizer = optim.Adadelta(parameters, params["LEARNING_RATE"]) optimizer = torch.optim.Adam(parameters, lr=lr) # optimizer = torch.optim.Adam(model.parameters(), lr=lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", factor=0.85, patience=0) best_score = 0.0 start_epoch = 1 # Data for loss curves plot epochs_count = [] train_losses = [] valid_losses = [] # Continuing training from a checkpoint if one was given as argument if checkpoint: checkpoint = torch.load(checkpoint) start_epoch = checkpoint["epoch"] + 1 best_score = checkpoint["best_score"] print("\t* Training will continue on existing model from epoch {}...". format(start_epoch)) model.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) epochs_count = checkpoint["epochs_count"] train_losses = checkpoint["train_losses"] valid_losses = checkpoint["valid_losses"] # Compute loss and accuracy before starting (or resuming) training. _, valid_loss, valid_accuracy, _, = validate(model, dev_loader, criterion) print("\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%". format(valid_loss, (valid_accuracy * 100))) # -------------------- Training epochs ------------------- # print("\n", 20 * "=", "Training ESIM model on device: {}".format(device), 20 * "=") patience_counter = 0 for epoch in range(start_epoch, epochs + 1): epochs_count.append(epoch) print("* Training epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader, optimizer, criterion, epoch, max_grad_norm) train_losses.append(epoch_loss) print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) print("* Validation for epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy, _, = validate( model, dev_loader, criterion) valid_losses.append(epoch_loss) print("-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%\n". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) # Update the optimizer's learning rate with the scheduler. scheduler.step(epoch_accuracy) # Early stopping on validation accuracy. if epoch_accuracy < best_score: patience_counter += 1 else: best_score = epoch_accuracy patience_counter = 0 if (if_save_model): torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses }, os.path.join(target_dir, "best.pth.tar")) print("save model succesfully!\n") print("* Test for epoch {}:".format(epoch)) _, _, test_accuracy, predictions = validate( model, test_loader, criterion) print("Test accuracy: {:.4f}%\n".format(test_accuracy)) test_prediction = pd.DataFrame({'prediction': predictions}) test_prediction.to_csv(os.path.join(target_dir, "test_prediction.csv"), index=False) if patience_counter >= patience: print("-> Early stopping: patience limit reached, stopping...") break
def main(test_q1_file, test_q2_file, test_labels_file, pretrained_file, gpu_index=0, batch_size=64): """ Test the ESIM model with pretrained weights on some dataset. Args: test_file: The path to a file containing preprocessed NLI data. pretrained_file: The path to a checkpoint produced by the 'train_model' script. vocab_size: The number of words in the vocabulary of the model being tested. embedding_dim: The size of the embeddings in the model. hidden_size: The size of the hidden layers in the model. Must match the size used during training. Defaults to 300. num_classes: The number of classes in the output of the model. Must match the value used during training. Defaults to 3. batch_size: The size of the batches used for testing. Defaults to 32. """ device = torch.device( "cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for testing ", 20 * "=") if platform == "linux" or platform == "linux2": checkpoint = torch.load(pretrained_file) else: checkpoint = torch.load(pretrained_file, map_location="cuda:0") # Retrieving model parameters from checkpoint. vocab_size = checkpoint["model"]["word_embedding.weight"].size(0) embedding_dim = checkpoint["model"]['word_embedding.weight'].size(1) hidden_size = checkpoint["model"]["projection.0.weight"].size(0) num_classes = checkpoint["model"]["classification.6.weight"].size(0) print("\t* Loading test data...") test_q1 = np.load(test_q1_file) test_q2 = np.load(test_q2_file) test_labels = np.load(test_labels_file) # test_labels = label_transformer(test_labels) test_data = {"q1": test_q1, "q2": test_q2, "labels": test_labels} test_data = QQPDataset(test_data) test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size) print("\t* Building model...") model = ESIM(vocab_size, embedding_dim, hidden_size, num_classes=num_classes, device=device).to(device) model.load_state_dict(checkpoint["model"]) print(20 * "=", " Testing ESIM model on device: {} ".format(device), 20 * "=") batch_time, total_time, accuracy = test(model, test_loader) print() print( "-> Average batch processing time: {:.4f}s, total test time: {:.4f}s, accuracy: {:.4f}%" .format(batch_time, total_time, (accuracy * 100))) print()
def main(args): print(20 * "=", " Preparing for training ", 20 * "=") # 保存模型的路径 if not os.path.exists(args.target_dir): os.makedirs(args.target_dir) # -------------------- Data loading ------------------- # print("\t* Loading training data...") # train_data = LCQMC_dataset(args.train_file, args.vocab_file, args.max_length, test_flag=False) train_data = LCQMC_dataset(args.train_file, args.vocab_file, args.max_length, test_flag=False) train_loader = DataLoader(train_data, batch_size=args.batch_size, shuffle=True) print("\t* Loading valid data...") dev_data = LCQMC_dataset(args.dev_file, args.vocab_file, args.max_length, test_flag=False) dev_loader = DataLoader(dev_data, batch_size=args.batch_size, shuffle=True) # -------------------- Model definition ------------------- # print("\t* Building model...") embeddings = load_embeddings(args.embed_file) model = ESIM(args, embeddings=embeddings).to(args.device) # -------------------- Preparation for training ------------------- # criterion = nn.CrossEntropyLoss() # 交叉熵损失函数 # 过滤出需要梯度更新的参数 parameters = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameters, lr=args.lr) # 优化器 # 学习计划 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.85, patience=0) best_score = 0.0 start_epoch = 1 epochs_count = [] train_losses = [] valid_losses = [] # Continuing training from a checkpoint if one was given as argument if args.checkpoint: # 从文件中加载checkpoint数据, 从而继续训练模型 checkpoints = torch.load(args.checkpoint) start_epoch = checkpoints["epoch"] + 1 best_score = checkpoints["best_score"] print("\t* Training will continue on existing model from epoch {}...".format(start_epoch)) model.load_state_dict(checkpoints["model"]) # 模型部分 optimizer.load_state_dict(checkpoints["optimizer"]) epochs_count = checkpoints["epochs_count"] train_losses = checkpoints["train_losses"] valid_losses = checkpoints["valid_losses"] # 这里改为只有从以前加载的checkpoint中才进行计算 valid, Compute loss and accuracy before starting (or resuming) training. _, valid_loss, valid_accuracy, auc = validate(model, dev_loader, criterion) print("\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}" .format(valid_loss, (valid_accuracy * 100), auc)) # -------------------- Training epochs ------------------- # print("\n", 20 * "=", "Training ESIM model on device: {}".format(args.device), 20 * "=") patience_counter = 0 for epoch in range(start_epoch, args.epochs + 1): epochs_count.append(epoch) print("* Training epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader, optimizer, criterion, epoch, args.max_grad_norm) train_losses.append(epoch_loss) print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%" .format(epoch_time, epoch_loss, (epoch_accuracy * 100))) print("* Validation for epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy, epoch_auc = validate(model, train_loader, criterion) valid_losses.append(epoch_loss) print("-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}\n" .format(epoch_time, epoch_loss, (epoch_accuracy * 100), epoch_auc)) # Update the optimizer's learning rate with the scheduler. scheduler.step(epoch_accuracy) # Early stopping on validation accuracy. if epoch_accuracy < best_score: patience_counter += 1 else: best_score = epoch_accuracy patience_counter = 0 # 保存最好的结果,需要保存的参数,这些参数在checkpoint中都能找到 torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses}, os.path.join(args.target_dir, "new_best.pth.tar")) # 保存每个epoch的结果 Save the model at each epoch.(这里可要可不要) torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "optimizer": optimizer.state_dict(), "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses}, os.path.join(args.target_dir, "new_esim_{}.pth.tar".format(epoch))) if patience_counter >= args.patience: print("-> Early stopping: patience limit reached, stopping...") break
def main(train_q1_file, train_q2_file, train_labels_file, dev_q1_file, dev_q2_file, dev_labels_file, embeddings_file, target_dir, hidden_size=128, dropout=0.5, num_classes=2, epochs=15, batch_size=64, lr=0.001, patience=5, max_grad_norm=10.0, gpu_index=0, checkpoint=None): device = torch.device( "cuda:{}".format(gpu_index) if torch.cuda.is_available() else "cpu") print(20 * "=", " Preparing for training ", 20 * "=") # 保存模型的路径 if not os.path.exists(target_dir): os.makedirs(target_dir) # -------------------- Data loading ------------------- # print("\t* Loading training data...") train_q1 = np.load(train_q1_file) train_q2 = np.load(train_q2_file) train_labels = np.load(train_labels_file) # train_labels = label_transformer(train_labels) train_data = {"q1": train_q1, "q2": train_q2, "labels": train_labels} train_data = QQPDataset(train_data) train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size) print("\t* Loading validation data...") dev_q1 = np.load(dev_q1_file) dev_q2 = np.load(dev_q2_file) dev_labels = np.load(dev_labels_file) # dev_labels = label_transformer(dev_labels) dev_data = {"q1": dev_q1, "q2": dev_q2, "labels": dev_labels} dev_data = QQPDataset(dev_data) dev_loader = DataLoader(dev_data, shuffle=True, batch_size=batch_size) # -------------------- Model definition ------------------- # print("\t* Building model...") embeddings = torch.tensor(np.load(embeddings_file), dtype=torch.float).to(device) model = ESIM(embeddings.shape[0], embeddings.shape[1], hidden_size, embeddings=embeddings, dropout=dropout, num_classes=num_classes, device=device).to(device) # -------------------- Preparation for training ------------------- # criterion = nn.CrossEntropyLoss() # 过滤出需要梯度更新的参数 parameters = filter(lambda p: p.requires_grad, model.parameters()) # optimizer = optim.Adadelta(parameters, params["LEARNING_RATE"]) optimizer = torch.optim.Adam(parameters, lr=lr) # optimizer = torch.optim.Adam(model.parameters(), lr=lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", factor=0.85, patience=0) best_score = 0.0 start_epoch = 1 # Data for loss curves plot epochs_count = [] train_losses = [] valid_losses = [] # Continuing training from a checkpoint if one was given as argument if checkpoint: checkpoint = torch.load(checkpoint) start_epoch = checkpoint["epoch"] + 1 best_score = checkpoint["best_score"] print("\t* Training will continue on existing model from epoch {}...". format(start_epoch)) model.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) epochs_count = checkpoint["epochs_count"] train_losses = checkpoint["train_losses"] valid_losses = checkpoint["valid_losses"] # Compute loss and accuracy before starting (or resuming) training. _, valid_loss, valid_accuracy = validate(model, dev_loader, criterion) print("\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%". format(valid_loss, (valid_accuracy * 100))) # -------------------- Training epochs ------------------- # print("\n", 20 * "=", "Training ESIM model on device: {}".format(device), 20 * "=") patience_counter = 0 for epoch in range(start_epoch, epochs + 1): epochs_count.append(epoch) print("* Training epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader, optimizer, criterion, epoch, max_grad_norm) train_losses.append(epoch_loss) print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) print("* Validation for epoch {}:".format(epoch)) epoch_time, epoch_loss, epoch_accuracy = validate( model, dev_loader, criterion) valid_losses.append(epoch_loss) print("-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%\n". format(epoch_time, epoch_loss, (epoch_accuracy * 100))) # Update the optimizer's learning rate with the scheduler. scheduler.step(epoch_accuracy) # Early stopping on validation accuracy. if epoch_accuracy < best_score: patience_counter += 1 else: best_score = epoch_accuracy patience_counter = 0 # Save the best model. The optimizer is not saved to avoid having # a checkpoint file that is too heavy to be shared. To resume # training from the best model, use the 'esim_*.pth.tar' # checkpoints instead. torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses }, os.path.join(target_dir, "best.pth.tar")) # Save the model at each epoch. torch.save( { "epoch": epoch, "model": model.state_dict(), "best_score": best_score, "optimizer": optimizer.state_dict(), "epochs_count": epochs_count, "train_losses": train_losses, "valid_losses": valid_losses }, os.path.join(target_dir, "esim_{}.pth.tar".format(epoch))) if patience_counter >= patience: print("-> Early stopping: patience limit reached, stopping...") break