def setup_training(model, train_loader, valid_loader, hps):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(hps.save_root, "train")
if not os.path.exists(train_dir): os.makedirs(train_dir)
if hps.restore_model != 'None':
logger.info("[INFO] Restoring %s for training...", hps.restore_model)
bestmodel_file = os.path.join(train_dir, hps.restore_model)
loader = ModelLoader()
loader.load_pytorch(model, bestmodel_file)
else:
logger.info("[INFO] Create new model for training...")
try:
run_training(model, train_loader, valid_loader,
hps) # this is an infinite loop until interrupted
except KeyboardInterrupt:
logger.error(
"[Error] Caught keyboard interrupt on worker. Stopping supervisor..."
)
save_file = os.path.join(train_dir, "earlystop.pkl")
saver = ModelSaver(save_file)
saver.save_pytorch(model)
logger.info('[INFO] Saving early stop model to %s', save_file)
def infer():
# Config Loader
test_args = ConfigSection()
ConfigLoader().load_config(cfgfile, {"POS_test": test_args})
# fetch dictionary size and number of labels from pickle files
word2index = load_pickle(pickle_path, "word2id.pkl")
test_args["vocab_size"] = len(word2index)
index2label = load_pickle(pickle_path, "label2id.pkl")
test_args["num_classes"] = len(index2label)
# Define the same model
model = AdvSeqLabel(test_args)
try:
ModelLoader.load_pytorch(model, "./save/trained_model.pkl")
print('model loaded!')
except Exception as e:
print('cannot load model!')
raise
# Data Loader
infer_data = SeqLabelDataSet(load_func=BaseLoader.load_lines)
infer_data.load(data_infer_path,
vocabs={"word_vocab": word2index},
infer=True)
print('data loaded')
# Inference interface
infer = SeqLabelInfer(pickle_path)
results = infer.predict(model, infer_data)
print(results)
print("Inference finished!")
def predict():
# Config Loader
test_args = ConfigSection()
ConfigLoader().load_config(cfgfile, {"POS_test": test_args})
# fetch dictionary size and number of labels from pickle files
word2index = load_pickle(pickle_path, "word2id.pkl")
test_args["vocab_size"] = len(word2index)
index2label = load_pickle(pickle_path, "label2id.pkl")
test_args["num_classes"] = len(index2label)
# load dev data
dev_data = load_pickle(pickle_path, "data_dev.pkl")
# Define the same model
model = AdvSeqLabel(test_args)
# Dump trained parameters into the model
ModelLoader.load_pytorch(model, "./save/trained_model.pkl")
print("model loaded!")
# Tester
test_args["evaluator"] = SeqLabelEvaluator()
tester = SeqLabelTester(**test_args.data)
# Start testing
tester.test(model, dev_data)
def run_test(model, loader, hps, limited=False):
"""Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far."""
test_dir = os.path.join(hps.save_root, "test") # make a subdir of the root dir for eval data
eval_dir = os.path.join(hps.save_root, "eval")
if not os.path.exists(test_dir) : os.makedirs(test_dir)
if not os.path.exists(eval_dir) :
logger.exception("[Error] eval_dir %s doesn't exist. Run in train mode to create it.", eval_dir)
raise Exception("[Error] eval_dir %s doesn't exist. Run in train mode to create it." % (eval_dir))
if hps.test_model == "evalbestmodel":
bestmodel_load_path = os.path.join(eval_dir, 'bestmodel.pkl') # this is where checkpoints of best models are saved
elif hps.test_model == "earlystop":
train_dir = os.path.join(hps.save_root, "train")
bestmodel_load_path = os.path.join(train_dir, 'earlystop.pkl')
else:
logger.error("None of such model! Must be one of evalbestmodel/trainbestmodel/earlystop")
raise ValueError("None of such model! Must be one of evalbestmodel/trainbestmodel/earlystop")
logger.info("[INFO] Restoring %s for testing...The path is %s", hps.test_model, bestmodel_load_path)
modelloader = ModelLoader()
modelloader.load_pytorch(model, bestmodel_load_path)
if hps.use_pyrouge:
logger.info("[INFO] Use PyRougeMetric for testing")
tester = Tester(data=loader, model=model,
metrics=[LabelFMetric(pred="prediction"), PyRougeMetric(hps, pred="prediction")],
batch_size=hps.batch_size)
else:
logger.info("[INFO] Use FastRougeMetric for testing")
tester = Tester(data=loader, model=model,
metrics=[LabelFMetric(pred="prediction"), FastRougeMetric(hps, pred="prediction")],
batch_size=hps.batch_size)
test_info = tester.test()
logger.info(test_info)
def train():
# Config Loader
train_args = ConfigSection()
test_args = ConfigSection()
ConfigLoader().load_config(cfgfile, {
"train": train_args,
"test": test_args
})
print("loading data set...")
data = SeqLabelDataSet(load_func=TokenizeDataSetLoader.load)
data.load(cws_data_path)
data_train, data_dev = data.split(ratio=0.3)
train_args["vocab_size"] = len(data.word_vocab)
train_args["num_classes"] = len(data.label_vocab)
print("vocab size={}, num_classes={}".format(len(data.word_vocab),
len(data.label_vocab)))
change_field_is_target(data_dev, "truth", True)
save_pickle(data_dev, "./save/", "data_dev.pkl")
save_pickle(data.word_vocab, "./save/", "word2id.pkl")
save_pickle(data.label_vocab, "./save/", "label2id.pkl")
# Trainer
trainer = SeqLabelTrainer(epochs=train_args["epochs"],
batch_size=train_args["batch_size"],
validate=train_args["validate"],
use_cuda=train_args["use_cuda"],
pickle_path=train_args["pickle_path"],
save_best_dev=True,
print_every_step=10,
model_name="trained_model.pkl",
evaluator=SeqLabelEvaluator())
# Model
model = AdvSeqLabel(train_args)
try:
ModelLoader.load_pytorch(model, "./save/saved_model.pkl")
print('model parameter loaded!')
except Exception as e:
print("No saved model. Continue.")
pass
# Start training
trainer.train(model, data_train, data_dev)
print("Training finished!")
# Saver
saver = ModelSaver("./save/trained_model.pkl")
saver.save_pytorch(model)
print("Model saved!")
def on_epoch_end(self, cur_epoch, n_epoch, optimizer):
if cur_epoch == 1:
self.opt.param_groups[0]["lr"] = self.best_lr
self.find = False
# reset model
ModelLoader().load_pytorch(self.trainer.model, "tmp")
print("Model reset. \nFind best lr={}".format(self.best_lr))
def setup_training(model, train_loader, valid_loader, hps):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(hps.save_root, "train")
if not os.path.exists(train_dir): os.makedirs(train_dir)
if hps.restore_model != 'None':
logger.info("[INFO] Restoring %s for training...", hps.restore_model)
bestmodel_file = os.path.join(train_dir, hps.restore_model)
loader = ModelLoader()
loader.load_pytorch(model, bestmodel_file)
else:
logger.info("[INFO] Create new model for training...")
run_training(model, train_loader, valid_loader,
hps) # this is an infinite loop until interrupted
def test(path):
# Tester
tester = Tester(**test_args.data, evaluator=ParserEvaluator(ignore_label))
# Model
model = BiaffineParser(**model_args.data)
model.eval()
try:
ModelLoader.load_pytorch(model, path)
print('model parameter loaded!')
except Exception as _:
print("No saved model. Abort test.")
raise
# Start training
print("Testing Train data")
tester.test(model, train_data)
print("Testing Dev data")
tester.test(model, dev_data)
print("Testing Test data")
tester.test(model, test_data)
def load_model(model_dir, device):
"""load trained model and its parameters, model name must as flat_ner.model"""
model = ModelLoader.load_pytorch_model(model_dir + '/flat_ner.model')
if 'cpu' in device.type:
model.to("cpu")
else:
model.to("cuda")
model.eval()
return model
def __init__(self, modelFile, vocabFile, addTarget2Vocab=False):
# CHAR_INPUT="chars", 并且会转化为word_index
self._vocabFile = vocabFile
self._addTarget2Vocab = addTarget2Vocab
self._CONST_CHAR = Const.CHAR_INPUT
self._CONST_WORDS = Const.INPUT
self._CONST_TARGET = Const.TARGET
self._input_fields = [self._CONST_WORDS, Const.INPUT_LEN]
self._word_counter, self._word_vocab, self._target_counter, \
self._target_vocab, self._target = self._get_vocabs()
self._vocab4word = Vocabulary()
self._update_word()
if self._addTarget2Vocab:
self._vocab4target = Vocabulary(unknown=None, padding=None)
self._input_fields.append(self._CONST_TARGET)
self._update_target()
self._model = Predictor(ModelLoader().load_pytorch_model(modelFile))
def __init__(self, n_batch, start_lr=1e-6, end_lr=10):
"""用第一个 epoch 找最佳的学习率，从第二个epoch开始应用它
:param n_batch: 一个epoch内的iteration数
:param start_lr: 学习率下界
:param end_lr: 学习率上界
"""
super(LRFinder, self).__init__()
self.start_lr, self.end_lr = start_lr, end_lr
self.num_it = n_batch
self.stop = False
self.best_loss = 0.
self.best_lr = None
self.loss_history = []
self.smooth_value = SmoothValue(0.8)
self.opt = None
scale = (self.end_lr - self.start_lr) / self.num_it
self.lr_gen = (self.start_lr + scale * (step + 1)
for step in range(self.num_it))
self.find = None
self.loader = ModelLoader()
def run_test(model, loader, hps, limited=False):
"""Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far."""
test_dir = os.path.join(
hps.save_root, "test") # make a subdir of the root dir for eval data
eval_dir = os.path.join(hps.save_root, "eval")
if not os.path.exists(test_dir): os.makedirs(test_dir)
if not os.path.exists(eval_dir):
logger.exception(
"[Error] eval_dir %s doesn't exist. Run in train mode to create it.",
eval_dir)
raise Exception(
"[Error] eval_dir %s doesn't exist. Run in train mode to create it."
% (eval_dir))
if hps.test_model == "evalbestmodel":
bestmodel_load_path = os.path.join(
eval_dir, 'bestmodel.pkl'
) # this is where checkpoints of best models are saved
elif hps.test_model == "evalbestFmodel":
bestmodel_load_path = os.path.join(eval_dir, 'bestFmodel.pkl')
elif hps.test_model == "trainbestmodel":
train_dir = os.path.join(hps.save_root, "train")
bestmodel_load_path = os.path.join(train_dir, 'bestmodel.pkl')
elif hps.test_model == "trainbestFmodel":
train_dir = os.path.join(hps.save_root, "train")
bestmodel_load_path = os.path.join(train_dir, 'bestFmodel.pkl')
elif hps.test_model == "earlystop":
train_dir = os.path.join(hps.save_root, "train")
bestmodel_load_path = os.path.join(train_dir, 'earlystop,pkl')
else:
logger.error(
"None of such model! Must be one of evalbestmodel/trainbestmodel/earlystop"
)
raise ValueError(
"None of such model! Must be one of evalbestmodel/trainbestmodel/earlystop"
)
logger.info("[INFO] Restoring %s for testing...The path is %s",
hps.test_model, bestmodel_load_path)
modelloader = ModelLoader()
modelloader.load_pytorch(model, bestmodel_load_path)
import datetime
nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') #现在
if hps.save_label:
log_dir = os.path.join(test_dir, hps.data_path.split("/")[-1])
resfile = open(log_dir, "w")
else:
log_dir = os.path.join(test_dir, nowTime)
resfile = open(log_dir, "wb")
logger.info("[INFO] Write the Evaluation into %s", log_dir)
model.eval()
match, pred, true, match_true = 0.0, 0.0, 0.0, 0.0
total_example_num = 0.0
pairs = {}
pairs["hyps"] = []
pairs["refer"] = []
pred_list = []
iter_start_time = time.time()
with torch.no_grad():
for i, (batch_x, batch_y) in enumerate(loader):
input, input_len = batch_x[Const.INPUT], batch_x[Const.INPUT_LEN]
label = batch_y[Const.TARGET]
if hps.cuda:
input = input.cuda() # [batch, N, seq_len]
label = label.cuda()
input_len = input_len.cuda()
batch_size, N, _ = input.size()
input = Variable(input)
input_len = Variable(input_len, requires_grad=False)
model_outputs = model.forward(input, input_len) # [batch, N, 2]
prediction = model_outputs["prediction"]
if hps.save_label:
pred_list.extend(
model_outputs["pred_idx"].data.cpu().view(-1).tolist())
continue
pred += prediction.sum()
true += label.sum()
match_true += ((prediction == label) & (prediction == 1)).sum()
match += (prediction == label).sum()
total_example_num += batch_size * N
for j in range(batch_size):
original_article_sents = batch_x["text"][j]
sent_max_number = len(original_article_sents)
refer = "\n".join(batch_x["summary"][j])
hyps = "\n".join(
original_article_sents[id].replace("\n", "")
for id in range(len(prediction[j]))
if prediction[j][id] == 1 and id < sent_max_number)
if limited:
k = len(refer.split())
hyps = " ".join(hyps.split()[:k])
logger.info((len(refer.split()), len(hyps.split())))
resfile.write(b"Original_article:")
resfile.write("\n".join(batch_x["text"][j]).encode('utf-8'))
resfile.write(b"\n")
resfile.write(b"Reference:")
if isinstance(refer, list):
for ref in refer:
resfile.write(ref.encode('utf-8'))
resfile.write(b"\n")
resfile.write(b'*' * 40)
resfile.write(b"\n")
else:
resfile.write(refer.encode('utf-8'))
resfile.write(b"\n")
resfile.write(b"hypothesis:")
resfile.write(hyps.encode('utf-8'))
resfile.write(b"\n")
if hps.use_pyrouge:
pairs["hyps"].append(hyps)
pairs["refer"].append(refer)
else:
try:
scores = utils.rouge_all(hyps, refer)
pairs["hyps"].append(hyps)
pairs["refer"].append(refer)
except ValueError:
logger.error("Do not select any sentences!")
logger.debug("sent_max_number:%d", sent_max_number)
logger.debug(original_article_sents)
logger.debug("label:")
logger.debug(label[j])
continue
# single example res writer
res = "Rouge1:\n\tp:%.6f, r:%.6f, f:%.6f\n" % (scores['rouge-1']['p'], scores['rouge-1']['r'], scores['rouge-1']['f']) \
+ "Rouge2:\n\tp:%.6f, r:%.6f, f:%.6f\n" % (scores['rouge-2']['p'], scores['rouge-2']['r'], scores['rouge-2']['f']) \
+ "Rougel:\n\tp:%.6f, r:%.6f, f:%.6f\n" % (scores['rouge-l']['p'], scores['rouge-l']['r'], scores['rouge-l']['f'])
resfile.write(res.encode('utf-8'))
resfile.write(b'-' * 89)
resfile.write(b"\n")
if hps.save_label:
import json
json.dump(pred_list, resfile)
logger.info(' | end of test | time: {:5.2f}s | '.format(
(time.time() - iter_start_time)))
return
resfile.write(b"\n")
resfile.write(b'=' * 89)
resfile.write(b"\n")
if hps.use_pyrouge:
logger.info("The number of pairs is %d", len(pairs["hyps"]))
if not len(pairs["hyps"]):
logger.error("During testing, no hyps is selected!")
return
if isinstance(pairs["refer"][0], list):
logger.info("Multi Reference summaries!")
scores_all = utils.pyrouge_score_all_multi(pairs["hyps"],
pairs["refer"])
else:
scores_all = utils.pyrouge_score_all(pairs["hyps"], pairs["refer"])
else:
logger.info("The number of pairs is %d", len(pairs["hyps"]))
if not len(pairs["hyps"]):
logger.error("During testing, no hyps is selected!")
return
rouge = Rouge()
scores_all = rouge.get_scores(pairs["hyps"], pairs["refer"], avg=True)
# the whole model res writer
resfile.write(b"The total testset is:")
res = "Rouge1:\n\tp:%.6f, r:%.6f, f:%.6f\n" % (scores_all['rouge-1']['p'], scores_all['rouge-1']['r'], scores_all['rouge-1']['f']) \
+ "Rouge2:\n\tp:%.6f, r:%.6f, f:%.6f\n" % (scores_all['rouge-2']['p'], scores_all['rouge-2']['r'], scores_all['rouge-2']['f']) \
+ "Rougel:\n\tp:%.6f, r:%.6f, f:%.6f\n" % (scores_all['rouge-l']['p'], scores_all['rouge-l']['r'], scores_all['rouge-l']['f'])
resfile.write(res.encode("utf-8"))
logger.info(res)
logger.info(' | end of test | time: {:5.2f}s | '.format(
(time.time() - iter_start_time)))
# label prediction
logger.info("match_true %d, pred %d, true %d, total %d, match %d", match,
pred, true, total_example_num, match)
accu, precision, recall, F = utils.eval_label(match_true, pred, true,
total_example_num, match)
res = "The size of totalset is %d, accu is %f, precision is %f, recall is %f, F is %f" % (
total_example_num / hps.doc_max_timesteps, accu, precision, recall, F)
resfile.write(res.encode('utf-8'))
logger.info(
"The size of totalset is %d, accu is %f, precision is %f, recall is %f, F is %f",
len(loader), accu, precision, recall, F)
from utils import *
from model import *
from dataloader import *
from fastNLP import Tester
from fastNLP.core.metrics import AccuracyMetric
# from fastNLP.core.utils import load_pickle
from fastNLP.io.model_io import ModelLoader
# define model
model=DPCNN(max_features=len(vocab),word_embedding_dimension=word_embedding_dimension,max_sentence_length = max_sentence_length,num_classes=num_classes,weight=weight)
# load checkpoint to model
# load_model = load_pickle(pickle_path=pickle_path, file_name='model_ckpt_100.pkl')
load_model_ = ModelLoader()
load_model = load_model_.load_pytorch_model('./model_backup/train_best_model.pkl')
# print(load_model)
# print(type(load_model))
# use Tester to evaluate
tester=Tester(data=dataset_test,model=load_model,metrics=AccuracyMetric(pred="predict",target="label_seq"),batch_size=64)
acc=tester.test()
print(acc)
