def main(): # load settings for training parser = argparse.ArgumentParser( description='train.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) opt = parser.parse_args() opt = process_opt(opt) opt.input_feeding = False opt.copy_input_feeding = False logging = config.init_logging(logger_name=None, log_file=opt.exp_path + '/output.log', stdout=True) logging.info('Parameters:') [logging.info('%s : %s' % (k, str(v))) for k, v in opt.__dict__.items()] try: train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab(opt) model = init_model(opt) optimizer_ml, optimizer_rl, criterion = init_optimizer_criterion(model, opt) train_model(model, optimizer_ml, optimizer_rl, criterion, train_data_loader, valid_data_loader, test_data_loader, opt) except Exception as e: logging.exception("message")
def main(): # load settings for training parser = argparse.ArgumentParser( description='train.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) config.transformer_opts(parser) opt = parser.parse_args() opt = process_opt(opt) opt.input_feeding = False opt.copy_input_feeding = False logging = config.init_logging(logger_name=None, log_file=opt.exp_path + '/output.log', stdout=True) try: # print(opt.bidirectional) # exit(0) # opt.train_from = 'model/kp20k.ml.copy.uni-directional.20180817-021054/kp20k.ml.copy.uni-directional.epoch=6.batch=6735.total_batch=57300.model' train_data_loader, word2id, id2word, vocab, eval_dataloader = load_data_vocab( opt) model = init_model(opt) optimizer_ml, _, criterion = init_optimizer_criterion(model, opt) train_model(model, optimizer_ml, _, criterion, train_data_loader, opt, eval_dataloader) except Exception as e: logging.exception("message")
def main(): # load settings for training parser = argparse.ArgumentParser( description='predict.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) config.transformer_opts(parser) opt = parser.parse_args() if opt.seed > 0: torch.manual_seed(opt.seed) # print(opt.gpuid) if torch.cuda.is_available() and not opt.gpuid: opt.gpuid = 0 opt.exp = 'predict.' + opt.exp if hasattr(opt, 'copy_model') and opt.copy_model: opt.exp += '.copy' if hasattr(opt, 'bidirectional'): if opt.bidirectional: opt.exp += '.bi-directional' else: opt.exp += '.uni-directional' # fill time into the name if opt.exp_path.find('%s') > 0: opt.exp_path = opt.exp_path % (opt.exp, opt.timemark) opt.pred_path = opt.pred_path % (opt.exp, opt.timemark) if not os.path.exists(opt.exp_path): os.makedirs(opt.exp_path) if not os.path.exists(opt.pred_path): os.makedirs(opt.pred_path) logging = config.init_logging(logger_name=None, log_file=opt.exp_path + '/output.log', stdout=True) try: opt.train_from = 'model/kp20k.ml.copy.bi-directional.20180908-054257/kp20k.ml.copy.bi-directional.epoch=9.batch=2932.model' test_data_loader, word2id, id2word, vocab = load_data_vocab(opt, load_train=False) model = init_model(opt) generator = SequenceGenerator(model,opt, eos_id=opt.word2id[pykp.io.EOS_WORD], beam_size=opt.beam_size, max_sequence_length=opt.max_sent_length, ) evaluate_beam_search(generator, test_data_loader, opt, title='predict', save_path=opt.pred_path + '/[epoch=%d,batch=%d,total_batch=%d]test_result.csv' % (0, 0, 0)) except Exception as e: logging.exception("message")
def main(): # load settings for training parser = argparse.ArgumentParser( description='train.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) opt = parser.parse_args() if opt.seed > 0: torch.manual_seed(opt.seed) print(opt.gpuid) if torch.cuda.is_available() and not opt.gpuid: opt.gpuid = 0 if hasattr(opt, 'copy_model') and opt.copy_model: opt.exp += '.copy' if hasattr(opt, 'bidirectional'): if opt.bidirectional: opt.exp += '.bi-directional' else: opt.exp += '.uni-directional' # fill time into the name if opt.exp_path.find('%s') > 0: opt.exp_path = opt.exp_path % (opt.exp, opt.timemark) opt.save_path = opt.save_path % (opt.exp, opt.timemark) if not os.path.exists(opt.exp_path): os.makedirs(opt.exp_path) if not os.path.exists(opt.save_path): os.makedirs(opt.save_path) config.init_logging(opt.exp_path + '/output.log') logging.info('Parameters:') [ logging.info('%s : %s' % (k, str(v))) for k, v in opt.__dict__.items() ] try: train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab( opt) model = init_model(opt) optimizer, criterion = init_optimizer_criterion(model, opt) train_model(model, optimizer, criterion, train_data_loader, valid_data_loader, test_data_loader, opt) except Exception as e: logging.exception("message")
def main(): # load settings for training parser = argparse.ArgumentParser( description='train.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) opt = parser.parse_args() if opt.seed > 0: torch.manual_seed(opt.seed) print(opt.gpuid) if torch.cuda.is_available() and not opt.gpuid: opt.gpuid = 0 if hasattr(opt, 'copy_model') and opt.copy_model: opt.exp += '.copy' if hasattr(opt, 'bidirectional'): if opt.bidirectional: opt.exp += '.bi-directional' else: opt.exp += '.uni-directional' # fill time into the name if opt.exp_path.find('%s') > 0: opt.exp_path = opt.exp_path % (opt.exp, opt.timemark) opt.save_path = opt.save_path % (opt.exp, opt.timemark) if not os.path.exists(opt.exp_path): os.makedirs(opt.exp_path) if not os.path.exists(opt.save_path): os.makedirs(opt.save_path) config.init_logging(opt.exp_path + '/output.log') logging.info('Parameters:') [logging.info('%s : %s' % (k, str(v))) for k, v in opt.__dict__.items()] try: train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab(opt) model = init_model(opt) optimizer, criterion = init_optimizer_criterion(model, opt) train_model(model, optimizer, criterion, train_data_loader, valid_data_loader, test_data_loader, opt) except Exception as e: logging.exception("message")
def main(): # load settings for training parser = argparse.ArgumentParser( description='train.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) config.transformer_opts(parser) opt = parser.parse_args() opt = process_opt(opt) opt.input_feeding = False opt.copy_input_feeding = False logging = config.init_logging(logger_name=None, log_file=opt.exp_path + '/output.log', stdout=True) try: # opt.train_from = 'model/kp20k.ml.copy.bi-directional.20180901-025437/kp20k.ml.copy.bi-directional.epoch=9.batch=938.model' train_data_loader, word2id, id2word, vocab, eval_dataloader = load_data_vocab( opt) model = init_model(opt) # embedding=make_embedding(word2id,id2word) embedding = torch.load('embedding50004.pt') model.init_embedding(embedding) opt.learning_rate = 0.001 optimizer_ml, criterion = init_optimizer_criterion(model, opt) train_model(model, optimizer_ml, criterion, train_data_loader, opt, eval_dataloader) except Exception as e: logging.exception("message")
def main(opt): # build test dataset test_loader = build_test_dataset(opt) # init the pretrained model model = init_pretrained_model(opt) # Print out predict path print("Prediction path: %s" % opt.pred_path) # predict the keyphrases of the src file and output it to opt.pred_path/predictions.txt predict(test_loader, model, opt) return opt.pred_path if __name__ == '__main__': # load settings for training parser = argparse.ArgumentParser( description='predict.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.model_opts(parser) config.predict_opts(parser) config.vocab_opts(parser) opt = parser.parse_args() opt = arrange_opt(opt, stage="prediction") # if opt.n_best < 0: # opt.n_best = None main(opt)
def main(): # load settings for training parser = argparse.ArgumentParser( description='predict.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) config.preprocess_opts(parser) config.model_opts(parser) config.train_opts(parser) config.predict_opts(parser) opt = parser.parse_args() if opt.seed > 0: torch.manual_seed(opt.seed) print(opt.gpuid) if torch.cuda.is_available() and not opt.gpuid: opt.gpuid = 0 opt.exp = 'predict.' + opt.exp if hasattr(opt, 'copy_model') and opt.copy_model: opt.exp += '.copy' if hasattr(opt, 'bidirectional'): if opt.bidirectional: opt.exp += '.bi-directional' else: opt.exp += '.uni-directional' # fill time into the name if opt.exp_path.find('%s') > 0: opt.exp_path = opt.exp_path % (opt.exp, opt.timemark) opt.pred_path = opt.pred_path % (opt.exp, opt.timemark) if not os.path.exists(opt.exp_path): os.makedirs(opt.exp_path) if not os.path.exists(opt.pred_path): os.makedirs(opt.pred_path) logging = config.init_logging('train', opt.exp_path + '/output.log') logging.info('Parameters:') [ logging.info('%s : %s' % (k, str(v))) for k, v in opt.__dict__.items() ] try: train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab( opt, load_train=False) model = init_model(opt) # optimizer, criterion = init_optimizer_criterion(model, opt) generator = SequenceGenerator(model, eos_id=opt.word2id[pykp.io.EOS_WORD], beam_size=opt.beam_size, max_sequence_length=opt.max_sent_length) # import time # start_time = time.time() evaluate_beam_search( generator, test_data_loader, opt, title='predict', save_path=opt.pred_path + '/[epoch=%d,batch=%d,total_batch=%d]test_result.csv' % (0, 0, 0)) # print("--- %s seconds --- Complete Beam Search" % (time.time() - start_time)) # predict_greedy(model, test_data_loader, test_examples, opt) except Exception as e: logging.exception("message")