def main():
# load settings for training
parser = argparse.ArgumentParser(
description='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
opt = parser.parse_args()
opt = process_opt(opt)
opt.input_feeding = False
opt.copy_input_feeding = False
logging = config.init_logging(logger_name=None, log_file=opt.exp_path + '/output.log', stdout=True)
logging.info('Parameters:')
[logging.info('%s : %s' % (k, str(v))) for k, v in opt.__dict__.items()]
try:
train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab(opt)
model = init_model(opt)
optimizer_ml, optimizer_rl, criterion = init_optimizer_criterion(model, opt)
train_model(model, optimizer_ml, optimizer_rl, criterion, train_data_loader, valid_data_loader, test_data_loader, opt)
except Exception as e:
logging.exception("message")
def main():
# load settings for training
parser = argparse.ArgumentParser(
description='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
config.transformer_opts(parser)
opt = parser.parse_args()
opt = process_opt(opt)
opt.input_feeding = False
opt.copy_input_feeding = False
logging = config.init_logging(logger_name=None,
log_file=opt.exp_path + '/output.log',
stdout=True)
try:
# print(opt.bidirectional)
# exit(0)
# opt.train_from = 'model/kp20k.ml.copy.uni-directional.20180817-021054/kp20k.ml.copy.uni-directional.epoch=6.batch=6735.total_batch=57300.model'
train_data_loader, word2id, id2word, vocab, eval_dataloader = load_data_vocab(
opt)
model = init_model(opt)
optimizer_ml, _, criterion = init_optimizer_criterion(model, opt)
train_model(model, optimizer_ml, _, criterion, train_data_loader, opt,
eval_dataloader)
except Exception as e:
logging.exception("message")
def main():
# load settings for training
parser = argparse.ArgumentParser(
description='predict.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
config.transformer_opts(parser)
opt = parser.parse_args()
if opt.seed > 0:
torch.manual_seed(opt.seed)
# print(opt.gpuid)
if torch.cuda.is_available() and not opt.gpuid:
opt.gpuid = 0
opt.exp = 'predict.' + opt.exp
if hasattr(opt, 'copy_model') and opt.copy_model:
opt.exp += '.copy'
if hasattr(opt, 'bidirectional'):
if opt.bidirectional:
opt.exp += '.bi-directional'
else:
opt.exp += '.uni-directional'
# fill time into the name
if opt.exp_path.find('%s') > 0:
opt.exp_path = opt.exp_path % (opt.exp, opt.timemark)
opt.pred_path = opt.pred_path % (opt.exp, opt.timemark)
if not os.path.exists(opt.exp_path):
os.makedirs(opt.exp_path)
if not os.path.exists(opt.pred_path):
os.makedirs(opt.pred_path)
logging = config.init_logging(logger_name=None, log_file=opt.exp_path + '/output.log', stdout=True)
try:
opt.train_from = 'model/kp20k.ml.copy.bi-directional.20180908-054257/kp20k.ml.copy.bi-directional.epoch=9.batch=2932.model'
test_data_loader, word2id, id2word, vocab = load_data_vocab(opt, load_train=False)
model = init_model(opt)
generator = SequenceGenerator(model,opt,
eos_id=opt.word2id[pykp.io.EOS_WORD],
beam_size=opt.beam_size,
max_sequence_length=opt.max_sent_length,
)
evaluate_beam_search(generator, test_data_loader, opt, title='predict', save_path=opt.pred_path + '/[epoch=%d,batch=%d,total_batch=%d]test_result.csv' % (0, 0, 0))
except Exception as e:
logging.exception("message")
def main():
# load settings for training
parser = argparse.ArgumentParser(
description='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
opt = parser.parse_args()
if opt.seed > 0:
torch.manual_seed(opt.seed)
print(opt.gpuid)
if torch.cuda.is_available() and not opt.gpuid:
opt.gpuid = 0
if hasattr(opt, 'copy_model') and opt.copy_model:
opt.exp += '.copy'
if hasattr(opt, 'bidirectional'):
if opt.bidirectional:
opt.exp += '.bi-directional'
else:
opt.exp += '.uni-directional'
# fill time into the name
if opt.exp_path.find('%s') > 0:
opt.exp_path = opt.exp_path % (opt.exp, opt.timemark)
opt.save_path = opt.save_path % (opt.exp, opt.timemark)
if not os.path.exists(opt.exp_path):
os.makedirs(opt.exp_path)
if not os.path.exists(opt.save_path):
os.makedirs(opt.save_path)
config.init_logging(opt.exp_path + '/output.log')
logging.info('Parameters:')
[
logging.info('%s : %s' % (k, str(v)))
for k, v in opt.__dict__.items()
]
try:
train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab(
opt)
model = init_model(opt)
optimizer, criterion = init_optimizer_criterion(model, opt)
train_model(model, optimizer, criterion, train_data_loader,
valid_data_loader, test_data_loader, opt)
except Exception as e:
logging.exception("message")
def main():
# load settings for training
parser = argparse.ArgumentParser(
description='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
opt = parser.parse_args()
if opt.seed > 0:
torch.manual_seed(opt.seed)
print(opt.gpuid)
if torch.cuda.is_available() and not opt.gpuid:
opt.gpuid = 0
if hasattr(opt, 'copy_model') and opt.copy_model:
opt.exp += '.copy'
if hasattr(opt, 'bidirectional'):
if opt.bidirectional:
opt.exp += '.bi-directional'
else:
opt.exp += '.uni-directional'
# fill time into the name
if opt.exp_path.find('%s') > 0:
opt.exp_path = opt.exp_path % (opt.exp, opt.timemark)
opt.save_path = opt.save_path % (opt.exp, opt.timemark)
if not os.path.exists(opt.exp_path):
os.makedirs(opt.exp_path)
if not os.path.exists(opt.save_path):
os.makedirs(opt.save_path)
config.init_logging(opt.exp_path + '/output.log')
logging.info('Parameters:')
[logging.info('%s : %s' % (k, str(v))) for k, v in opt.__dict__.items()]
try:
train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab(opt)
model = init_model(opt)
optimizer, criterion = init_optimizer_criterion(model, opt)
train_model(model, optimizer, criterion, train_data_loader, valid_data_loader, test_data_loader, opt)
except Exception as e:
logging.exception("message")
def main():
# load settings for training
parser = argparse.ArgumentParser(
description='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
config.transformer_opts(parser)
opt = parser.parse_args()
opt = process_opt(opt)
opt.input_feeding = False
opt.copy_input_feeding = False
logging = config.init_logging(logger_name=None,
log_file=opt.exp_path + '/output.log',
stdout=True)
try:
# opt.train_from = 'model/kp20k.ml.copy.bi-directional.20180901-025437/kp20k.ml.copy.bi-directional.epoch=9.batch=938.model'
train_data_loader, word2id, id2word, vocab, eval_dataloader = load_data_vocab(
opt)
model = init_model(opt)
# embedding=make_embedding(word2id,id2word)
embedding = torch.load('embedding50004.pt')
model.init_embedding(embedding)
opt.learning_rate = 0.001
optimizer_ml, criterion = init_optimizer_criterion(model, opt)
train_model(model, optimizer_ml, criterion, train_data_loader, opt,
eval_dataloader)
except Exception as e:
logging.exception("message")
def main(opt):
# build test dataset
test_loader = build_test_dataset(opt)
# init the pretrained model
model = init_pretrained_model(opt)
# Print out predict path
print("Prediction path: %s" % opt.pred_path)
# predict the keyphrases of the src file and output it to opt.pred_path/predictions.txt
predict(test_loader, model, opt)
return opt.pred_path
if __name__ == '__main__':
# load settings for training
parser = argparse.ArgumentParser(
description='predict.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.model_opts(parser)
config.predict_opts(parser)
config.vocab_opts(parser)
opt = parser.parse_args()
opt = arrange_opt(opt, stage="prediction")
# if opt.n_best < 0:
# opt.n_best = None
main(opt)
def main():
# load settings for training
parser = argparse.ArgumentParser(
description='predict.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config.preprocess_opts(parser)
config.model_opts(parser)
config.train_opts(parser)
config.predict_opts(parser)
opt = parser.parse_args()
if opt.seed > 0:
torch.manual_seed(opt.seed)
print(opt.gpuid)
if torch.cuda.is_available() and not opt.gpuid:
opt.gpuid = 0
opt.exp = 'predict.' + opt.exp
if hasattr(opt, 'copy_model') and opt.copy_model:
opt.exp += '.copy'
if hasattr(opt, 'bidirectional'):
if opt.bidirectional:
opt.exp += '.bi-directional'
else:
opt.exp += '.uni-directional'
# fill time into the name
if opt.exp_path.find('%s') > 0:
opt.exp_path = opt.exp_path % (opt.exp, opt.timemark)
opt.pred_path = opt.pred_path % (opt.exp, opt.timemark)
if not os.path.exists(opt.exp_path):
os.makedirs(opt.exp_path)
if not os.path.exists(opt.pred_path):
os.makedirs(opt.pred_path)
logging = config.init_logging('train', opt.exp_path + '/output.log')
logging.info('Parameters:')
[
logging.info('%s : %s' % (k, str(v)))
for k, v in opt.__dict__.items()
]
try:
train_data_loader, valid_data_loader, test_data_loader, word2id, id2word, vocab = load_data_vocab(
opt, load_train=False)
model = init_model(opt)
# optimizer, criterion = init_optimizer_criterion(model, opt)
generator = SequenceGenerator(model,
eos_id=opt.word2id[pykp.io.EOS_WORD],
beam_size=opt.beam_size,
max_sequence_length=opt.max_sent_length)
# import time
# start_time = time.time()
evaluate_beam_search(
generator,
test_data_loader,
opt,
title='predict',
save_path=opt.pred_path +
'/[epoch=%d,batch=%d,total_batch=%d]test_result.csv' % (0, 0, 0))
# print("--- %s seconds --- Complete Beam Search" % (time.time() - start_time))
# predict_greedy(model, test_data_loader, test_examples, opt)
except Exception as e:
logging.exception("message")