def train(model, train_dataset, dev_dataset, config): start_time = time.time() train_dataloader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=2) n_sample = len(train_dataloader) model.train() optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate) classified_loss = torch.nn.CrossEntropyLoss().to(config.device) total_batch = 0 for epoch in range(config.num_epochs): total_loss = 0 model.train() print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs)) for sample in tqdm(train_dataloader): sample = (i.to(config.device) for i in sample) sentence1, sentence2, label, seq1, seq2 = sample # print(sentence1) model.zero_grad() out = model(sentence1, sentence2) loss = classified_loss(out, label) total_loss += loss.clone().detach() loss.backward() optimizer.step() total_batch += 1 time_dif = get_time_dif(start_time) print("Time usage:", time_dif) print(f'train loss: {total_loss / n_sample}') evaluate(config, model, dev_dataset) torch.save(model.state_dict(), config.save_path)
def train(): print('=========model architecture==========') model = GCA(args).to(device) print(model) print('=============== end =================') loss_fn = nn.CrossEntropyLoss(ignore_index=constants.PAD_IDX) parameters = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameters, args["lr"], weight_decay=args["wdecay"]) print('Training and evaluating...') logger.info( "Batch Size: %d, Dropout: %.2f" % ( args["batch_size"], args["dropout"] ) ) if args["ac_re"]: logger.info( "Alpha L2 Regularization: %d, Beta Slowness Regularization: %d" % ( args["alpha"], args["beta"], ) ) start_time = time.time() if not os.path.exists(args["exp_dir"]): os.makedirs(args["exp_dir"]) best_ppl = 9999999 last_improved = 0 require_improvement = 5 with open(args["exp_dir"] + "params.json", "w") as outfile: json.dump(args, outfile, indent=4) for epoch in range(args["epochs"]): print('=============== Epoch %d=================' % (epoch + 1)) logger.info("Optimizer: %s" % (optimizer)) train_loss, train_ppl = train_epoch( epoch, train_dataloader, model, loss_fn, optimizer, device, args["clip"], args["ac_re"], args["alpha"], args["beta"] ) valid_loss, valid_ppl = test( model, valid_dataloader, device ) if valid_ppl < best_ppl: best_ppl = valid_ppl last_improved = epoch torch.save(model.state_dict(), args["exp_dir"] + 'model_params_%s_min_ppl.pkl' % (epoch + 1) ) improved_str = '*' else: improved_str = '' time_dif = get_time_dif(start_time) msg = 'Epoch: {0:>6},Train Loss: {1:>6.6}, Train Ppl: {2:>6.6},' \ + ' Val loss: {3:>6.6}, Val Ppl: {4:>6.6},Time:{5} {6}' print(msg.format(epoch + 1, train_loss, train_ppl, valid_loss, valid_ppl, time_dif, improved_str)) if epoch - last_improved > require_improvement: print("No optimization for a long time, auto-stopping...") break return 1
def test(model, test_iter): # test model.load_state_dict(torch.load(Config.save_path)) model.eval() start_time = time.time() test_acc, test_loss, test_report, test_confusion = evaluate(model, test_iter, test=True) msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}' print(msg.format(test_loss, test_acc)) print("Precision, Recall and F1-Score...") print(test_report) print("Confusion Matrix...") print(test_confusion) time_dif = get_time_dif(start_time) print("Time usage:", time_dif) model.train()
voc.add_token(word) for filepath in args.defs: data = read_data(filepath) for elem in data: voc.add_token(elem[0]) char_voc.token_maxlen = max(len(elem[0]), char_voc.token_maxlen) for c in elem[0]: char_voc.add_token(c) definition = elem[1] for d in definition: voc.add_token(d) if args.usage: usage = elem[3] for u in usage: voc.add_token(u) context = elem[2] if args.same: for t in context: voc.add_token(t) else: context_voc.add_token(elem[0]) for t in context: context_voc.add_token(t) voc.save(args.save) char_voc.save(args.save_chars) if not args.same: context_voc.save(args.save_context) time_dif = get_time_dif(start_time) print("Finished! Build vocabulary time usage:", time_dif)
def train(): print('=========model architecture==========') if args["type"] == "same_level": model = SenseGenerator_Emb(args).to(device) elif args["type"] == "hir_level": model = SenseGenerator(args).to(device) print(model) print('=============== end =================') loss_fn = nn.CrossEntropyLoss(ignore_index=constants.PAD_IDX) # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( # torch.optim.Adam( # filter(lambda p: p.requires_grad, model.parameters()), lr=args["lr"] # ), # factor=args["decay_factor"], # patience=args["decay_patience"] # ) # optimizer = scheduler.optimizer parameters = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameters, args["lr"], weight_decay=args["wdecay"]) print('Training and evaluating...') logger.info( "Batch Size: %d, Dropout: %.2f, RNN Weight Dropout: %.2f, RNN Layer Dropout: %.2f, Input Embedding Layers Dropout: %.2f, " "Remove Words Embedding Dropout: %.2f" % (args["batch_size"], args["dropout"], args["wdrop"], args["dropouth"], args["dropouti"], args["dropoute"])) if args["ac_re"]: logger.info( "Alpha L2 Regularization: %d, Beta Slowness Regularization: %d" % ( args["alpha"], args["beta"], )) start_time = time.time() if not os.path.exists(args["exp_dir"]): os.makedirs(args["exp_dir"]) best_ppl = 9999999 last_improved = 0 require_improvement = 5 with open(args["exp_dir"] + "params.json", "w") as outfile: json.dump(args, outfile, indent=4) for epoch in range(args["epochs"]): print('=============== Epoch %d=================' % (epoch + 1)) logger.info("Optimizer: %s" % (optimizer)) train_loss, train_ppl, task1_loss, task1_ppl, task2_loss, task2_ppl = train_epoch( epoch, train_dataloader, model, loss_fn, optimizer, device, args["clip"], args["ac_re"], args["alpha"], args["beta"]) # train_loss, train_ppl, task1_loss, task1_ppl, task2_loss, task2_ppl = train_shunxu( # epoch, train_dataloader, model, loss_fn, optimizer, device, args["clip"] # ) valid1_loss, valid1_ppl, valid2_loss, valid2_ppl = test( model, args["label_type"], valid_dataloader, device) if valid2_ppl < best_ppl: best_ppl = valid2_ppl last_improved = epoch torch.save( model.state_dict(), args["exp_dir"] + 'hir_shared_%s_min_ppl.pkl' % (epoch + 1)) improved_str = '*' else: improved_str = '' time_dif = get_time_dif(start_time) msg = 'Epoch: {0:>6},Total Train Loss: {1:>6.6}, Total Train Ppl: {2:>6.6},Time:{3}' print(msg.format(epoch + 1, train_loss, train_ppl, time_dif) + "\n") task1_msg = 'Task1: {0:>6},Valid Loss: {1:>6.6}, Valid Ppl: {2:>6.6}' task2_msg = 'Task2: {0:>6},Valid Loss: {1:>6.6}, Vliad Ppl: {2:>6.6} {3}' print( task1_msg.format(args["label_type"][0], valid1_loss, valid1_ppl) + "\n") print( task2_msg.format(args["label_type"][1], valid2_loss, valid2_ppl, improved_str) + "\n") if epoch - last_improved > require_improvement: print("No optimization for a long time, auto-stopping...") break return 1