class Train(object):
"""
Train
"""
def __init__(self, **kwargs):
"""
:param kwargs:
Args of data:
train_iter : train batch data iterator
dev_iter : dev batch data iterator
test_iter : test batch data iterator
Args of train:
model : nn model
config : config
"""
print("Training Start......")
# for k, v in kwargs.items():
# self.__setattr__(k, v)
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.model = kwargs["model"]
self.config = kwargs["config"]
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(name=self.config.learning_algorithm,
model=self.model,
lr=self.config.learning_rate,
weight_decay=self.config.weight_decay,
grad_clip=self.config.clip_max_norm)
self.loss_function = self._loss(
learning_algorithm=self.config.learning_algorithm)
print(self.optimizer)
print(self.loss_function)
self.best_score = Best_Result()
self.train_iter_len = len(self.train_iter)
@staticmethod
def _loss(learning_algorithm):
"""
:param learning_algorithm:
:return:
"""
if learning_algorithm == "SGD":
loss_function = nn.CrossEntropyLoss(reduction="sum")
return loss_function
else:
loss_function = nn.CrossEntropyLoss(reduction="mean")
return loss_function
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
"""
:param clip_max_norm_use: whether to use clip max norm for nn model
:param clip_max_norm: clip max norm max values [float or None]
:return:
"""
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm_(self.model.parameters(), max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
"""
:param config: config
:param epoch: epoch
:param new_lr: learning rate
:return:
"""
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, config, epoch, init_lr):
"""lr decay
Args:
epoch: int, epoch
init_lr: initial lr
"""
if config.use_lr_decay:
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
"""
:return:
"""
if backward_count % config.backward_batch_size == 0 or backward_count == self.train_iter_len:
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch):
"""
:param epoch:
:return:
"""
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
print("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
print(
"Early Stop Train. Best Score Locate on {} Epoch.".format(
self.best_score.best_epoch))
exit()
@staticmethod
def _get_model_args(batch_features):
"""
:param batch_features: Batch Instance
:return:
"""
inst = batch_features.inst
word = batch_features.word_features
mask = word > 0
sentence_length = batch_features.sentence_length
labels = batch_features.label_features
batch_size = batch_features.batch_length
return inst, word, mask, sentence_length, labels, batch_size
def _calculate_loss(self, feats, labels):
"""
Args:
feats: size = (batch_size, seq_len, tag_size)
labels: size = (batch_size, seq_len)
"""
loss_value = self.loss_function(feats, labels)
return loss_value
def train(self):
"""
:return:
"""
epochs = self.config.epochs
clip_max_norm_use = self.config.clip_max_norm_use
clip_max_norm = self.config.clip_max_norm
new_lr = self.config.learning_rate
for epoch in range(1, epochs + 1):
print("\n## The {} Epoch, All {} Epochs ! ##".format(
epoch, epochs))
new_lr = self._dynamic_lr(config=self.config,
epoch=epoch,
new_lr=new_lr)
# self.optimizer = self._decay_learning_rate(config=self.config, epoch=epoch - 1, init_lr=self.config.learning_rate)
print("now lr is {}".format(
self.optimizer.param_groups[0].get("lr")),
end="")
start_time = time.time()
random.shuffle(self.train_iter)
self.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
for batch_count, batch_features in enumerate(self.train_iter):
backward_count += 1
# self.optimizer.zero_grad()
inst, word, mask, sentence_length, labels, batch_size = self._get_model_args(
batch_features)
logit = self.model(word, sentence_length, train=True)
loss = self._calculate_loss(logit, labels)
loss.backward()
self._clip_model_norm(clip_max_norm_use, clip_max_norm)
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
# self.optimizer.step()
steps += 1
if (steps - 1) % self.config.log_interval == 0:
accuracy = self.getAcc(logit, labels, batch_size)
sys.stdout.write(
"\nbatch_count = [{}] , loss is {:.6f}, [accuracy is {:.6f}%]"
.format(batch_count + 1, loss.item(), accuracy))
end_time = time.time()
print("\nTrain Time {:.3f}".format(end_time - start_time), end="")
self.eval(model=self.model, epoch=epoch, config=self.config)
self._model2file(model=self.model, config=self.config, epoch=epoch)
self._early_stop(epoch=epoch)
def eval(self, model, epoch, config):
"""
:param model: nn model
:param epoch: epoch
:param config: config
:return:
"""
eval_start_time = time.time()
print('\nmistakes for dev_iter')
self.eval_batch(self.dev_iter,
model,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
print("Dev Time {:.3f}".format(eval_end_time - eval_start_time))
print('mistakes for test_iter')
eval_start_time = time.time()
self.eval_batch(self.test_iter,
model,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
print("Test Time {:.3f}".format(eval_end_time - eval_start_time))
def _model2file(self, model, config, epoch):
"""
:param model: nn model
:param config: config
:param epoch: epoch
:return:
"""
if config.save_model and config.save_all_model:
save_model_all(model, config.save_dir, config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config.save_best_model_path,
config.model_name, self.best_score)
else:
print()
def eval_batch(self,
data_iter,
model,
best_score,
epoch,
config,
test=False):
"""
:param data_iter: eval batch data iterator
:param model: eval model
:param best_score:
:param epoch:
:param config: config
:param test: whether to test
:return: None
"""
model.eval()
# eval time
corrects = 0
size = 0
loss = 0
Truelabel = []
Words = []
d = []
for batch_features in data_iter:
inst, word, mask, sentence_length, labels, batch_size = self._get_model_args(
batch_features)
logit = self.model(word, sentence_length,
train=False) # 加入拼音model需要修改
loss += self._calculate_loss(logit, labels)
size += batch_features.batch_length
t = torch.max(logit, 1)[1].view(labels.size()).data
p = t.cpu().numpy()
p = p.tolist()
for i in p:
d.append([i])
for k in inst:
Truelabel.append(k.label_index)
Words.append(k.words)
# torch.max(logit, 1)[1] 返回tensor logit每一行最大值的索引
corrects += (torch.max(logit, 1)[1].view(
labels.size()).data == labels.data).sum()
print("更加详细的评估指标:\n", classification_report(Truelabel, d, digits=5))
assert size is not 0, print("Error")
accuracy = float(corrects) / size * 100.0
average_loss = float(loss) / size
test_flag = "Test"
if test is False:
print()
test_flag = "Dev"
best_score.current_dev_score = accuracy
if accuracy >= best_score.best_dev_score:
best_score.best_dev_score = accuracy
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True:
best_score.p = accuracy
print("{} eval: average_loss = {:.6f}, accuracy = {:.6f}%".format(
test_flag, average_loss, accuracy))
if test is True:
print("The Current Best Dev Accuracy: {:.6f}, Locate on {} Epoch.".
format(best_score.best_dev_score, best_score.best_epoch))
print("The Current Best Test Accuracy: accuracy = {:.6f}%".format(
best_score.p))
if test is True:
best_score.best_test = False
@staticmethod
def getAcc(logit, target, batch_size):
"""
:param logit: model predict
:param target: gold label
:param batch_size: batch size
:param config: config
:return:
"""
corrects = (torch.max(logit, 1)[1].view(
target.size()).data == target.data).sum()
accuracy = float(corrects) / batch_size * 100.0
return accuracy
class Train(object):
"""
Train
"""
def __init__(self, **kwargs):
"""
:param kwargs:
Args of data:
train_iter : train batch data iterator
dev_iter : dev batch data iterator
test_iter : test batch data iterator
Args of train:
model : nn model
config : config
"""
print("Training Start......")
# for k, v in kwargs.items():
# self.__setattr__(k, v)
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.model = kwargs["model"]
self.config = kwargs["config"]
self.use_crf = self.config.use_crf
self.average_batch = self.config.average_batch
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(name=self.config.learning_algorithm,
model=self.model,
lr=self.config.learning_rate,
weight_decay=self.config.weight_decay,
grad_clip=self.config.clip_max_norm)
self.loss_function = self._loss(
learning_algorithm=self.config.learning_algorithm,
label_paddingId=self.config.label_paddingId,
use_crf=self.use_crf)
print(self.optimizer)
print(self.loss_function)
self.best_score = Best_Result()
self.train_eval, self.dev_eval, self.test_eval = Eval(), Eval(), Eval()
self.train_iter_len = len(self.train_iter)
def _loss(self, learning_algorithm, label_paddingId, use_crf=False):
if use_crf:
loss_function = self.model.crf_layer.neg_log_likelihood_loss
return loss_function
elif learning_algorithm == "SGD":
loss_function = nn.CrossEntropyLoss(ignore_index=label_paddingId,
size_average=False)
return loss_function
else:
loss_function = nn.CrossEntropyLoss(ignore_index=label_paddingId,
size_average=True)
return loss_function
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
"""
:param clip_max_norm_use: whether to use clip max norm for nn model
:param clip_max_norm: clip max norm max values [float or None]
:return:
"""
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm(self.model.parameters(), max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
"""
:param config: config
:param epoch: epoch
:param new_lr: learning rate
:return:
"""
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, epoch, init_lr):
"""lr decay
Args:
epoch: int, epoch
init_lr: initial lr
"""
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
"""
:return:
"""
if backward_count % config.backward_batch_size == 0 or backward_count == self.train_iter_len:
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch):
"""
:param epoch:
:return:
"""
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
print("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
print(
"Early Stop Train. Best Score Locate on {} Epoch.".format(
self.best_score.best_epoch))
exit()
@staticmethod
def _get_model_args(batch_features):
"""
:param batch_features: Batch Instance
:return:
"""
word = batch_features.word_features
char = batch_features.char_features
mask = word > 0
sentence_length = batch_features.sentence_length
# desorted_indices = batch_features.desorted_indices
tags = batch_features.label_features
return word, char, mask, sentence_length, tags
def _calculate_loss(self, feats, mask, tags):
"""
Args:
feats: size = (batch_size, seq_len, tag_size)
mask: size = (batch_size, seq_len)
tags: size = (batch_size, seq_len)
"""
if not self.use_crf:
batch_size, max_len = feats.size(0), feats.size(1)
lstm_feats = feats.view(batch_size * max_len, -1)
tags = tags.view(-1)
return self.loss_function(lstm_feats, tags)
else:
loss_value = self.loss_function(feats, mask, tags)
if self.average_batch:
batch_size = feats.size(0)
loss_value /= float(batch_size)
return loss_value
def train(self):
"""
:return:
"""
epochs = self.config.epochs
clip_max_norm_use = self.config.clip_max_norm_use
clip_max_norm = self.config.clip_max_norm
new_lr = self.config.learning_rate
for epoch in range(1, epochs + 1):
print("\n## The {} Epoch, All {} Epochs ! ##".format(
epoch, epochs))
# new_lr = self._dynamic_lr(config=self.config, epoch=epoch, new_lr=new_lr)
self.optimizer = self._decay_learning_rate(
epoch=epoch - 1, init_lr=self.config.learning_rate)
print("now lr is {}".format(
self.optimizer.param_groups[0].get("lr")),
end="")
start_time = time.time()
random.shuffle(self.train_iter)
self.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
for batch_count, batch_features in enumerate(self.train_iter):
backward_count += 1
# self.optimizer.zero_grad()
word, char, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = self.model(word, char, sentence_length, train=True)
loss = self._calculate_loss(logit, mask, tags)
loss.backward()
self._clip_model_norm(clip_max_norm_use, clip_max_norm)
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
# self.optimizer.step()
steps += 1
if (steps - 1) % self.config.log_interval == 0:
self.getAcc(self.train_eval, batch_features, logit,
self.config)
sys.stdout.write(
"\nbatch_count = [{}] , loss is {:.6f}, [TAG-ACC is {:.6f}%]"
.format(batch_count + 1, loss.data[0],
self.train_eval.acc()))
end_time = time.time()
print("\nTrain Time {:.3f}".format(end_time - start_time), end="")
self.eval(model=self.model, epoch=epoch, config=self.config)
self._model2file(model=self.model, config=self.config, epoch=epoch)
self._early_stop(epoch=epoch)
def eval(self, model, epoch, config):
"""
:param model: nn model
:param epoch: epoch
:param config: config
:return:
"""
self.dev_eval.clear_PRF()
eval_start_time = time.time()
self.eval_batch(self.dev_iter,
model,
self.dev_eval,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
print("Dev Time {:.3f}".format(eval_end_time - eval_start_time))
self.test_eval.clear_PRF()
eval_start_time = time.time()
self.eval_batch(self.test_iter,
model,
self.test_eval,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
print("Test Time {:.3f}".format(eval_end_time - eval_start_time))
def _model2file(self, model, config, epoch):
"""
:param model: nn model
:param config: config
:param epoch: epoch
:return:
"""
if config.save_model and config.save_all_model:
save_model_all(model, config.save_dir, config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config.save_best_model_path,
config.model_name, self.best_score)
else:
print()
def eval_batch(self,
data_iter,
model,
eval_instance,
best_score,
epoch,
config,
test=False):
"""
:param data_iter: eval batch data iterator
:param model: eval model
:param eval_instance:
:param best_score:
:param epoch:
:param config: config
:param test: whether to test
:return: None
"""
model.eval()
# eval time
eval_acc = Eval()
eval_PRF = EvalPRF()
gold_labels = []
predict_labels = []
for batch_features in data_iter:
word, char, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = model(word, char, sentence_length, train=False)
if self.use_crf is False:
predict_ids = torch_max(logit)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
for id_word in range(inst.words_size):
predict_label.append(
config.create_alphabet.label_alphabet.from_id(
label_ids[id_word]))
gold_labels.append(inst.labels)
predict_labels.append(predict_label)
else:
path_score, best_paths = model.crf_layer(logit, mask)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
gold_labels.append(inst.labels)
label_ids = best_paths[id_batch].cpu().data.numpy(
)[:inst.words_size]
label = []
for i in label_ids:
label.append(
config.create_alphabet.label_alphabet.from_id(i))
predict_labels.append(label)
for p_label, g_label in zip(predict_labels, gold_labels):
eval_PRF.evalPRF(predict_labels=p_label,
gold_labels=g_label,
eval=eval_instance)
if eval_acc.gold_num == 0:
eval_acc.gold_num = 1
p, r, f = eval_instance.getFscore()
# p, r, f = entity_evalPRF_exact(gold_labels=gold_labels, predict_labels=predict_labels)
# p, r, f = entity_evalPRF_propor(gold_labels=gold_labels, predict_labels=predict_labels)
# p, r, f = entity_evalPRF_binary(gold_labels=gold_labels, predict_labels=predict_labels)
test_flag = "Test"
if test is False:
print()
test_flag = "Dev"
best_score.current_dev_score = f
if f >= best_score.best_dev_score:
best_score.best_dev_score = f
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True:
best_score.p = p
best_score.r = r
best_score.f = f
print(
"{} eval: precision = {:.6f}% recall = {:.6f}% , f-score = {:.6f}%, [TAG-ACC = {:.6f}%]"
.format(test_flag, p, r, f, 0.0000))
if test is True:
print("The Current Best Dev F-score: {:.6f}, Locate on {} Epoch.".
format(best_score.best_dev_score, best_score.best_epoch))
print(
"The Current Best Test Result: precision = {:.6f}% recall = {:.6f}% , f-score = {:.6f}%"
.format(best_score.p, best_score.r, best_score.f))
if test is True:
best_score.best_test = False
@staticmethod
def getAcc(eval_acc, batch_features, logit, config):
"""
:param eval_acc: eval instance
:param batch_features: batch data feature
:param logit: model output
:param config: config
:return:
"""
eval_acc.clear_PRF()
predict_ids = torch_max(logit)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
gold_lable = inst.labels
for id_word in range(inst.words_size):
predict_label.append(
config.create_alphabet.label_alphabet.from_id(
label_ids[id_word]))
assert len(predict_label) == len(gold_lable)
cor = 0
for p_lable, g_lable in zip(predict_label, gold_lable):
if p_lable == g_lable:
cor += 1
eval_acc.correct_num += cor
eval_acc.gold_num += len(gold_lable)
class Train(object):
"""
Train
"""
def __init__(self, **kwargs):
"""
:param kwargs:
Args of data:
train_iter : train batch data iterator
dev_iter : dev batch data iterator
test_iter : test batch data iterator
Args of train:
model : nn model
config : config
"""
print("Training Start......")
# for k, v in kwargs.items():
# self.__setattr__(k, v)
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.parser = kwargs["model"]
self.config = kwargs["config"]
self.device = self.config.device
self.cuda = False
if self.device != cpu_device:
self.cuda = True
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(
name=self.config.learning_algorithm,
model=self.parser.model,
lr=self.config.learning_rate,
# weight_decay=self.config.weight_decay, grad_clip=self.config.clip_max_norm,
weight_decay=self.config.weight_decay,
grad_clip="None",
betas=(0.9, 0.9),
eps=1.0e-12)
if self.config.learning_algorithm == "SGD":
self.loss_function = nn.CrossEntropyLoss(reduction="sum")
else:
self.loss_function = nn.CrossEntropyLoss(reduction="mean")
print(self.optimizer)
self.best_score = Best_Result()
self.train_iter_len = len(self.train_iter)
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
"""
:param clip_max_norm_use: whether to use clip max norm for nn model
:param clip_max_norm: clip max norm max values [float or None]
:return:
"""
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm_(self.parser.model.parameters(),
max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
"""
:param config: config
:param epoch: epoch
:param new_lr: learning rate
:return:
"""
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
# print("epoch", epoch)
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, epoch, init_lr):
"""
Args:
epoch: int, epoch
init_lr: initial lr
"""
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
# print('learning rate: {0}'.format(lr))
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
"""
:return:
"""
if backward_count % config.update_batch_size == 0 or backward_count == self.train_iter_len:
self._clip_model_norm(self.config.clip_max_norm_use,
self.config.clip_max_norm)
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch):
"""
:param epoch:
:return:
"""
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
print("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
print(
"Early Stop Train. Best Score Locate on {} Epoch.".format(
self.best_score.best_epoch))
exit()
def _model2file(self, model, config, epoch):
"""
:param model: nn model
:param config: config
:param epoch: epoch
:return:
"""
if config.save_model and config.save_all_model:
save_model_all(model, config.save_dir, config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config.save_best_model_path,
config.model_name, self.best_score)
else:
print()
def train(self):
"""
:return:
"""
epochs = self.config.epochs
new_lr = self.config.learning_rate
for epoch in range(1, epochs + 1):
print("\n## The {} Epoch, All {} Epochs ! ##".format(
epoch, epochs))
new_lr = self._dynamic_lr(config=self.config,
epoch=epoch,
new_lr=new_lr)
# self.optimizer = self._decay_learning_rate(epoch=epoch - 1, init_lr=self.config.learning_rate)
print("now lr is {}".format(
self.optimizer.param_groups[0].get("lr")),
end="")
start_time = time.time()
random.shuffle(self.train_iter)
self.parser.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
overall_arc_correct, overall_label_correct, overall_total_arcs = 0, 0, 0
for batch_count, batch_features in enumerate(self.train_iter):
backward_count += 1
words, ext_words, tags, masks = batch_features.words, batch_features.ext_words, batch_features.tags, \
batch_features.masks
heads, rels, lengths = batch_features.heads, batch_features.rels, batch_features.lengths
sumLength = sum(lengths)
self.parser.forward(words, ext_words, tags, masks)
loss = self.parser.compute_loss(heads, rels, lengths)
loss = loss / self.config.update_batch_size
loss_value = loss.data.cpu().numpy()
loss.backward()
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
steps += 1
if (steps - 1) % self.config.log_interval == 0:
arc_correct, label_correct, total_arcs = self.parser.compute_accuracy(
heads, rels)
overall_arc_correct += arc_correct
overall_label_correct += label_correct
overall_total_arcs += total_arcs
uas = overall_arc_correct.item(
) * 100.0 / overall_total_arcs
las = overall_label_correct.item(
) * 100.0 / overall_total_arcs
sys.stdout.write(
"\nbatch_count = [{}/{}] , loss is {:.6f}, length: {}, ARC: {:.6f}, REL: {:.6f}"
.format(batch_count + 1, self.train_iter_len,
float(loss_value), sumLength, float(uas),
float(las)))
end_time = time.time()
print("\nTrain Time {:.3f}".format(end_time - start_time), end="")
self.eval(parser=self.parser, epoch=epoch, config=self.config)
self._model2file(model=self.parser.model,
config=self.config,
epoch=epoch)
self._early_stop(epoch=epoch)
def eval(self, parser, epoch, config):
"""
:param parser:
:param epoch:
:param config:
:return:
"""
eval_start_time = time.time()
self._eval_batch(self.dev_iter,
parser,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
print("Dev Time {:.3f}".format(eval_end_time - eval_start_time))
eval_start_time = time.time()
self._eval_batch(self.test_iter,
parser,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
print("Test Time {:.3f}".format(eval_end_time - eval_start_time))
# self.get_one_batch(batch_features.insts)
def get_one_batch(self, insts):
"""
:param insts:
:return:
"""
batch = []
for inst in insts:
batch.append(inst.sentence)
return batch
def _eval_batch(self,
data_iter,
parser,
best_score,
epoch,
config,
test=False):
"""
:param data_iter:
:param parser:
:param vocab:
:param best_score:
:param epoch:
:param config:
:param test:
:return:
"""
parser.model.eval()
arc_total_test, arc_correct_test, rel_total_test, rel_correct_test = 0, 0, 0, 0
alphabet = config.alphabet
for batch_count, batch_features in enumerate(data_iter):
one_batch = self.get_one_batch(batch_features.insts)
words, ext_words, tags, masks = batch_features.words, batch_features.ext_words, batch_features.tags, batch_features.masks
heads, rels, lengths = batch_features.heads, batch_features.rels, batch_features.lengths
# print()
# print(heads)
# print(rels)
# print(lengths)
# exit()
sumLength = sum(lengths)
count = 0
arcs_batch, rels_batch = parser.parse(words, ext_words, tags,
lengths, masks)
# print(arcs_batch)
# print(rels_batch)
# exit()
for tree in batch_variable_depTree(one_batch, arcs_batch,
rels_batch, lengths, alphabet):
# printDepTree(output, tree)
# arc_total, arc_correct, rel_total, rel_correct = evalDepTree(tree, one_batch[count])
arc_total, arc_correct, rel_total, rel_correct = evalDepTree(
one_batch[count], tree)
arc_total_test += arc_total
arc_correct_test += arc_correct
rel_total_test += rel_total
rel_correct_test += rel_correct
count += 1
uas = arc_correct_test * 100.0 / arc_total_test
las = rel_correct_test * 100.0 / rel_total_test
f = uas
# p, r, f = law_p, law_r, law_f
test_flag = "Test"
if test is False:
print()
test_flag = "Dev"
best_score.current_dev_score = f
if f >= best_score.best_dev_score:
best_score.best_dev_score = f
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True:
best_score.f = f
print("{}:".format(test_flag))
print("UAS = %d/%d = %.2f, LAS = %d/%d =%.2f" %
(arc_correct_test, arc_total_test, uas, rel_correct_test,
rel_total_test, las))
if test is True:
print("The Current Best Dev score: {:.6f}, Locate on {} Epoch.".
format(best_score.best_dev_score, best_score.best_epoch))
if test is True:
best_score.best_test = False
class Train(object):
"""
Train
"""
def __init__(self, **kwargs):
"""
:param kwargs:
Args of data:
train_iter : train batch data iterator
dev_iter : dev batch data iterator
test_iter : test batch data iterator
Args of train:
model : nn model
config : config
"""
print("Training Start......")
# for k, v in kwargs.items():
# self.__setattr__(k, v)
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.model = kwargs["model"]
self.config = kwargs["config"]
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(name=self.config.learning_algorithm,
model=self.model,
lr=self.config.learning_rate,
weight_decay=self.config.weight_decay,
grad_clip=self.config.clip_max_norm)
if self.config.learning_algorithm == "SGD":
self.loss_function = nn.CrossEntropyLoss(size_average=False)
else:
self.loss_function = nn.CrossEntropyLoss(size_average=True)
print(self.optimizer)
self.best_score = Best_Result()
self.train_eval, self.dev_eval_seg, self.dev_eval_pos, self.test_eval_seg, self.test_eval_pos = Eval(
), Eval(), Eval(), Eval(), Eval()
self.train_iter_len = len(self.train_iter)
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
"""
:param clip_max_norm_use: whether to use clip max norm for nn model
:param clip_max_norm: clip max norm max values [float or None]
:return:
"""
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm(self.model.parameters(), max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
"""
:param config: config
:param epoch: epoch
:param new_lr: learning rate
:return:
"""
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
# print("epoch", epoch)
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, epoch, init_lr):
"""衰减学习率
Args:
epoch: int, 迭代次数
init_lr: 初始学习率
"""
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
# print('learning rate: {0}'.format(lr))
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
"""
:return:
"""
if backward_count % config.backward_batch_size == 0 or backward_count == self.train_iter_len:
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch):
"""
:param epoch:
:return:
"""
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
print("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
print(
"Early Stop Train. Best Score Locate on {} Epoch.".format(
self.best_score.best_epoch))
exit()
def _model2file(self, model, config, epoch):
"""
:param model: nn model
:param config: config
:param epoch: epoch
:return:
"""
if config.save_model and config.save_all_model:
save_model_all(model, config.save_dir, config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config.save_best_model_path,
config.model_name, self.best_score)
else:
print()
def train(self):
"""
:return:
"""
epochs = self.config.epochs
clip_max_norm_use = self.config.clip_max_norm_use
clip_max_norm = self.config.clip_max_norm
new_lr = self.config.learning_rate
for epoch in range(1, epochs + 1):
print("\n## The {} Epoch, All {} Epochs ! ##".format(
epoch, epochs))
new_lr = self._dynamic_lr(config=self.config,
epoch=epoch,
new_lr=new_lr)
# self.optimizer = self._decay_learning_rate(epoch=epoch - 1, init_lr=self.config.learning_rate)
print("now lr is {}".format(
self.optimizer.param_groups[0].get("lr")),
end="")
start_time = time.time()
random.shuffle(self.train_iter)
self.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
for batch_count, batch_features in enumerate(self.train_iter):
backward_count += 1
# self.optimizer.zero_grad()
maxCharSize = batch_features.char_features.size()[1]
decoder_out, state = self.model(batch_features, train=True)
self.cal_train_acc(batch_features, self.train_eval,
batch_count, decoder_out, maxCharSize,
self.config)
loss = torch.nn.functional.nll_loss(
decoder_out, batch_features.gold_features)
loss.backward()
self._clip_model_norm(clip_max_norm_use, clip_max_norm)
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
# self.optimizer.step()
steps += 1
if (steps - 1) % self.config.log_interval == 0:
sys.stdout.write(
"\nBatch_count = [{}/{}] , Loss is {:.6f} , (Correct/Total_num) = Accuracy ({}/{})"
" = {:.6f}%".format(batch_count + 1,
self.train_iter_len, loss.data[0],
self.train_eval.correct_num,
self.train_eval.gold_num,
self.train_eval.acc() * 100))
end_time = time.time()
# print("\nTrain Time {:.3f}".format(end_time - start_time), end="")
print("\nTrain Time {:.4f}".format(end_time - start_time))
self.eval(model=self.model, epoch=epoch, config=self.config)
self._model2file(model=self.model, config=self.config, epoch=epoch)
self._early_stop(epoch=epoch)
def eval(self, model, epoch, config):
"""
:param model: nn model
:param epoch: epoch
:param config: config
:return:
"""
self.dev_eval_pos.clear()
self.dev_eval_seg.clear()
eval_start_time = time.time()
self.eval_batch(self.dev_iter,
model,
self.dev_eval_seg,
self.dev_eval_pos,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
print("Dev Time {:.4f}".format(eval_end_time - eval_start_time))
self.test_eval_pos.clear()
self.test_eval_seg.clear()
eval_start_time = time.time()
self.eval_batch(self.test_iter,
model,
self.test_eval_seg,
self.test_eval_pos,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
print("Test Time {:.4f}".format(eval_end_time - eval_start_time))
def eval_batch(self,
data_iter,
model,
eval_seg,
eval_pos,
best_score,
epoch,
config,
test=False):
"""
:param data_iter: eval data iterator
:param model: nn model
:param eval_seg: seg eval
:param eval_pos: pos eval
:param best_score: best score
:param epoch: current epoch
:param config: config
:param test: test
:return:
"""
model.eval()
for batch_features in data_iter:
decoder_out, state = model(batch_features, train=False)
for i in range(batch_features.batch_length):
self.jointPRF_Batch(batch_features.inst[i], state.words[i],
state.pos_labels[i], eval_seg, eval_pos)
# calculate the F-Score
seg_p, seg_r, seg_f = eval_seg.getFscore()
pos_p, pos_r, pos_f = eval_pos.getFscore()
test_flag = "Test"
if test is False:
# print()
test_flag = "Dev"
best_score.current_dev_score = pos_f
if pos_f >= best_score.best_dev_score:
best_score.best_dev_score = pos_f
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True:
best_score.p = pos_p
best_score.r = pos_r
best_score.f = pos_f
print(test_flag + " ---->")
print("seg: precision = {:.4f}% recall = {:.4f}% , f-score = {:.4f}%".
format(seg_p, seg_r, seg_f))
print("pos: precision = {:.4f}% recall = {:.4f}% , f-score = {:.4f}%".
format(pos_p, pos_r, pos_f))
if test is True:
print("The Current Best Dev F-score: {:.4f}%, Locate on {} Epoch.".
format(best_score.best_dev_score, best_score.best_epoch))
if test is True:
best_score.best_test = False
@staticmethod
def jointPRF_Batch(inst, state_words, state_posLabel, seg_eval, pos_eval):
"""
:param inst:
:param state_words:
:param state_posLabel:
:param seg_eval:
:param pos_eval:
:return:
"""
words = state_words
posLabels = state_posLabel
count = 0
predict_seg = []
predict_pos = []
for idx in range(len(words)):
w = words[idx]
posLabel = posLabels[idx]
predict_seg.append('[' + str(count) + ',' + str(count + len(w)) +
']')
predict_pos.append('[' + str(count) + ',' + str(count + len(w)) +
']' + posLabel)
count += len(w)
seg_eval.gold_num += len(inst.gold_seg)
seg_eval.predict_num += len(predict_seg)
for p in predict_seg:
if p in inst.gold_seg:
seg_eval.correct_num += 1
pos_eval.gold_num += len(inst.gold_pos)
pos_eval.predict_num += len(predict_pos)
for p in predict_pos:
if p in inst.gold_pos:
pos_eval.correct_num += 1
def cal_train_acc(self, batch_features, train_eval, batch_count,
decoder_out, maxCharSize, args):
"""
:param batch_features:
:param train_eval:
:param batch_count:
:param decoder_out:
:param maxCharSize:
:param args:
:return:
"""
train_eval.clear()
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
for id_char in range(inst.chars_size):
actionID = self.getMaxindex(
decoder_out[id_batch * maxCharSize + id_char], args)
if actionID == inst.gold_index[id_char]:
train_eval.correct_num += 1
train_eval.gold_num += inst.chars_size
@staticmethod
def getMaxindex(decode_out_acc, config):
"""
:param decode_out_acc:
:param config:
:return:
"""
max = decode_out_acc.data[0]
maxIndex = 0
for idx in range(1, config.label_size):
if decode_out_acc.data[idx] > max:
max = decode_out_acc.data[idx]
maxIndex = idx
return maxIndex
class Train(object):
def __init__(self, **kwargs):
self.config = kwargs["config"]
self.config.logger.info("Training Start......")
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.model = kwargs["model"]
self.use_crf = self.config.use_crf
self.target = kwargs["target"]
self.average_batch = self.config.average_batch
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(name=self.config.learning_algorithm,
model=self.model,
lr=self.config.learning_rate,
weight_decay=self.config.weight_decay,
grad_clip=self.config.clip_max_norm)
self.loss_function = self._loss(
learning_algorithm=self.config.learning_algorithm,
label_paddingId=self.config.arg_paddingId,
use_crf=self.use_crf)
self.config.logger.info(self.optimizer)
self.config.logger.info(self.loss_function)
self.best_score = Best_Result()
self.train_eval, self.dev_eval, self.test_eval = Eval(), Eval(), Eval()
def _loss(self, learning_algorithm, label_paddingId, use_crf=False):
"""
:param learning_algorithm:
:param label_paddingId:
:param use_crf:
:return:
"""
if use_crf:
loss_function = self.model.crf_layer.neg_log_likelihood_loss
return loss_function
elif learning_algorithm == "SGD":
loss_function = nn.CrossEntropyLoss(ignore_index=label_paddingId,
reduction="sum")
return loss_function
else:
loss_function = nn.CrossEntropyLoss(ignore_index=label_paddingId,
reduction="mean")
return loss_function
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
"""
:param clip_max_norm_use: whether to use clip max norm for nn model
:param clip_max_norm: clip max norm max values [float or None]
:return:
"""
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm_(self.model.parameters(), max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
"""
:param config: config
:param epoch: epoch
:param new_lr: learning rate
:return:
"""
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, epoch, init_lr):
"""lr decay
Args:
epoch: int, epoch
init_lr: initial lr
"""
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
"""
:param config:
:param backward_count:
:return:
"""
if backward_count % config.backward_batch_size == 0: # or backward_count == self.train_iter_len:
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch, config):
"""
:param epoch:
:return:
"""
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
self.config.logger.info("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
self.end_of_epoch = epoch
self.config.logger.info(
"\n\nEarly Stop Train. Best Score Locate on {} Epoch.".
format(self.best_score.best_epoch))
self.save_training_summary()
return True # exit()
else:
return False
else:
return False
@staticmethod
def _get_model_args(batch_features):
"""
:param batch_features: Batch Instance
:return:
"""
elmo_char_seqs = batch_features.elmo_char_seqs
elmo_word_seqs = batch_features.elmo_word_seqs
word = batch_features.word_features
lang = batch_features.lang
pos = batch_features.pos_features
prd = batch_features.prd_features
x_prd_posi = batch_features.prd_posi_features
mask = batch_features.mask
sentence_length = batch_features.sentence_length
tags = batch_features.label_features
return elmo_char_seqs, elmo_word_seqs, word, lang, pos, prd, x_prd_posi, mask, sentence_length, tags
def _calculate_loss(self, feats, mask, tags):
"""
Args:
feats: size = (batch_size, seq_len, tag_size)
mask: size = (batch_size, seq_len)
tags: size = (batch_size, seq_len)
"""
if not self.use_crf:
batch_size, max_len = feats.size(0), feats.size(1)
lstm_feats = feats.view(batch_size * max_len, -1)
tags = tags.view(-1)
return self.loss_function(lstm_feats, tags)
else:
loss_value = self.loss_function(feats, mask, tags)
if self.average_batch:
batch_size = feats.size(0)
loss_value /= float(batch_size)
return loss_value
def train(self):
"""
:return:
"""
epochs = self.config.epochs
clip_max_norm_use = self.config.clip_max_norm_use
clip_max_norm = self.config.clip_max_norm
self.config.logger.info('\n\n')
self.config.logger.info('=-' * 50)
for epoch in range(1, epochs + 1):
self.train_iter.reset_flag4trainset()
self.config.logger.info("\n\n### Epoch: {}/{} ###".format(
epoch, epochs))
self.optimizer = self._decay_learning_rate(
epoch=epoch - 1, init_lr=self.config.learning_rate)
self.config.logger.info("current lr: {}".format(
self.optimizer.param_groups[0].get("lr")))
start_time = time.time()
self.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
self.config.logger.info('=-' * 10)
batch_count = 0
for batch_features in tqdm.tqdm(self.train_iter):
batch_count += 1
backward_count += 1
elmo_char_seqs, elmo_word_seqs, word, lang, pos, prd, x_prd_posi, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = self.model(elmo_char_seqs,
elmo_word_seqs,
word,
lang,
pos,
prd,
x_prd_posi,
mask,
sentence_length,
train=True)
loss = self._calculate_loss(logit, mask, tags)
loss.backward()
self._clip_model_norm(clip_max_norm_use, clip_max_norm)
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
steps += 1
if self.use_crf is True:
p, r, f, acc_ = self.getAccCRF(self.train_eval, batch_features,
logit, mask, self.config)
else:
p, r, f, acc_ = self.getAcc(self.train_eval, batch_features,
logit, self.config)
self.config.logger.info(
"batch_count:{} , loss: {:.4f}, p: {:.4f}% r: {:.4f}% , f: {:.4f}%, ACC: {:.4f}%"
.format(batch_count, loss.item(), p, r, f, acc_))
end_time = time.time()
self.config.logger.info("Train Time {:.3f}".format(end_time -
start_time))
self.config.logger.info('=-' * 10)
self.eval(model=self.model, epoch=epoch, config=self.config)
self.config.logger.info('=-' * 10)
if self._early_stop(epoch=epoch, config=self.config):
return
self.config.logger.info('=-' * 15)
self.save_training_summary()
def save_training_summary(self):
self.config.logger.info(
"Copy the last model ckps to {} as backup.".format(
self.config.save_dir))
self.config.logger.info(
"save the training summary at end of the log file.")
self.config.logger.info("\n")
self.config.logger.info("*" * 25)
self.config.logger.info("*" * 10)
self.config.logger.info("features:")
if self.config.is_predicate:
self.config.logger.info("\tpredicate, dim: %d" %
self.config.prd_embed_dim)
self.config.logger.info("*" * 10)
self.config.logger.info("model:")
self.config.logger.info(self.model)
self.config.logger.info("*" * 10)
self.config.logger.info("training:")
self.config.logger.info('\tbatch size: %d' % self.config.batch_size)
self.config.logger.info("*" * 10)
self.config.logger.info("best performance:")
self.config.logger.info("\tbest at epoch: %d" %
self.best_score.best_epoch)
self.config.logger.info("\tdev(%):")
self.config.logger.info("\t\tprecision, %.5f" %
self.best_score.best_dev_p_score)
self.config.logger.info("\t\trecall, %.5f" %
self.best_score.best_dev_r_score)
self.config.logger.info("\t\tf1, %.5f" %
self.best_score.best_dev_f1_score)
self.config.logger.info("\ttest(%):")
self.config.logger.info("\t\tprecision, %.5f" % self.best_score.p)
self.config.logger.info("\t\trecall, %.5f" % self.best_score.r)
self.config.logger.info("\t\tf1, %.5f" % self.best_score.f)
self.config.logger.info("*" * 25)
def eval(self, model, epoch, config):
"""
:param model: nn model
:param epoch: epoch
:param config: config
:return:
"""
self.dev_eval.clear_PRF()
eval_start_time = time.time()
self.eval_batch(self.dev_iter,
model,
self.dev_eval,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
self.config.logger.info("Dev Time: {:.3f}".format(eval_end_time -
eval_start_time))
self.config.logger.info('=-' * 10)
self.test_eval.clear_PRF()
eval_start_time = time.time()
self.eval_batch(self.test_iter,
model,
self.test_eval,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
self.config.logger.info("Test Time: {:.3f}".format(eval_end_time -
eval_start_time))
def _model2file(self, model, config, epoch):
"""
:param model: nn model
:param config: config
:param epoch: epoch
:return:
"""
if config.save_model and config.save_all_model:
save_model_all(model, config, config.save_model_dir,
config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config, config.save_model_dir,
config.model_name, self.best_score)
else:
self.config.logger.info()
def eval_batch(self,
data_iter,
model,
eval_instance,
best_score,
epoch,
config,
test=False):
"""
:param data_iter: eval batch data iterator
:param model: eval model
:param eval_instance:
:param best_score:
:param epoch:
:param config: config
:param test: whether to test
:return: None
"""
test_flag = "Test"
if test is False:
test_flag = "Dev"
model.eval()
gold_labels = []
predict_labels = []
all_sentence_length = []
for batch_features in tqdm.tqdm(data_iter):
elmo_char_seqs, elmo_word_seqs, word, lang, pos, prd, x_prd_posi, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = model(elmo_char_seqs,
elmo_word_seqs,
word,
lang,
pos,
prd,
x_prd_posi,
mask,
sentence_length,
train=False)
all_sentence_length.extend(sentence_length)
if self.use_crf is False:
predict_ids = torch_max(logit)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
for id_word in range(inst.words_size):
predict_label.append(config.argvocab.i2c[int(i)])
gold_labels.append(inst.labels)
predict_labels.append(predict_label)
else:
path_score, best_paths = model.crf_layer(logit, mask)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
gold_labels.append(inst.labels)
label_ids = best_paths[id_batch].cpu().data.numpy(
)[:inst.words_size]
label = []
for i in label_ids:
label.append(config.argvocab.i2c[int(i)])
predict_labels.append(label)
p, r, f, acc_ = eval_instance.getFscore(predict_labels, gold_labels,
all_sentence_length)
if test is False:
best_score.current_dev_score = f
if f >= best_score.best_dev_f1_score:
best_score.best_dev_f1_score = f
best_score.best_dev_p_score = p
best_score.best_dev_r_score = r
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True: # test
best_score.p = p
best_score.r = r
best_score.f = f
self.config.logger.info(
"{} at current epoch, p: {:.4f}% r: {:.4f}% , f: {:.4f}%, ACC: {:.3f}%"
.format(test_flag, p, r, f, acc_))
if test is False:
self.config.logger.info(
"Till now, The Best Dev Result: p: {:.4f}% r: {:.4f}% , f: {:.4f}%, Locate on {} Epoch."
.format(best_score.best_dev_p_score,
best_score.best_dev_r_score,
best_score.best_dev_f1_score, best_score.best_epoch))
elif test is True:
self.config.logger.info(
"Till now, The Best Test Result: p: {:.4f}% r: {:.4f}% , f: {:.4f}%, Locate on {} Epoch."
.format(best_score.p, best_score.r, best_score.f,
best_score.best_epoch))
best_score.best_test = False
def eval_external_batch(self, data_iter, config, meta_info=''):
"""
:param data_iter: eval batch data iterator
:param model: eval model
:param eval_instance:
:param best_score:
:param epoch:
:param config: config
:param test: whether to test
:return: None
"""
eval = Eval()
self.model.eval()
gold_labels = []
predict_labels = []
all_sentence_length = []
for batch_features in tqdm.tqdm(data_iter):
elmo_char_seqs, elmo_word_seqs, word, lang, pos, prd, x_prd_posi, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = self.model(elmo_char_seqs,
elmo_word_seqs,
word,
lang,
pos,
prd,
x_prd_posi,
mask,
sentence_length,
train=False)
all_sentence_length.extend(sentence_length)
if self.use_crf is False:
predict_ids = torch_max(logit)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
for id_word in range(inst.words_size):
predict_label.append(config.argvocab.i2c[int(i)])
gold_labels.append(inst.labels)
predict_labels.append(predict_label)
else:
path_score, best_paths = self.model.crf_layer(logit, mask)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
gold_labels.append(inst.labels)
label_ids = best_paths[id_batch].cpu().data.numpy(
)[:inst.words_size]
label = []
for i in label_ids:
label.append(config.argvocab.i2c[int(i)])
predict_labels.append(label)
p, r, f, acc_ = eval.getFscore(predict_labels, gold_labels,
all_sentence_length)
self.config.logger.info(
"eval on {}%, p: {:.4f}% r: {:.4f}% , f: {:.4f}%, ACC: {:.4f}%".
format(meta_info, p, r, f, acc_))
@staticmethod
def getAcc(eval_train, batch_features, logit, config):
"""
:param eval_acc: eval instance
:param batch_features: batch data feature
:param logit: model output
:param config: config
:return:
"""
eval_train.clear_PRF()
predict_ids = torch_max(logit)
predict_labels = []
gold_labels = []
batch_length = []
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
gold_label = inst.labels
for id_word in range(inst.words_size):
predict_label.append(config.argvocab.i2c[label_ids[id_word]])
predict_labels.append(predict_label)
gold_labels.append(gold_label)
batch_length.append(inst.words_size)
assert len(predict_label) == len(gold_label)
p, r, f, acc_ = eval_train.getFscore(predict_labels, gold_labels,
batch_length)
return p, r, f, acc_
def getAccCRF(self, eval_train, batch_features, logit, mask, config):
eval_train.clear_PRF()
predict_labels = []
gold_labels = []
batch_length = []
path_score, best_paths = self.model.crf_layer(logit, mask)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
gold_labels.append(inst.labels)
label_ids = best_paths[id_batch].cpu().data.numpy()[:inst.
words_size]
label = []
for i in label_ids:
label.append(config.argvocab.i2c[int(i)])
predict_labels.append(label)
batch_length.append(inst.words_size)
assert len(label) == len(inst.labels)
p, r, f, acc_ = eval_train.getFscore(predict_labels, gold_labels,
batch_length)
return p, r, f, acc_
class Train(object):
"""
Train
"""
def __init__(self, **kwargs):
"""
:param kwargs:
Args of data:
train_iter : train batch data iterator
dev_iter : dev batch data iterator
test_iter : test batch data iterator
Args of train:
model : nn model
config : config
"""
print("Training Start......")
# for k, v in kwargs.items():
# self.__setattr__(k, v)
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.model = kwargs["model"]
self.config = kwargs["config"]
self.device = self.config.device
self.cuda = False
if self.device != cpu_device:
self.cuda = True
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(name=self.config.learning_algorithm,
model=self.model,
lr=self.config.learning_rate,
weight_decay=self.config.weight_decay,
grad_clip=self.config.clip_max_norm)
if self.config.learning_algorithm == "SGD":
self.loss_function = nn.CrossEntropyLoss(reduction="sum")
else:
self.loss_function = nn.CrossEntropyLoss(reduction="mean")
# self.loss_function = nn.MultiLabelSoftMarginLoss(size_average=True)
print(self.optimizer)
self.best_score = Best_Result()
self.train_iter_len = len(self.train_iter)
# define accu eval
self.accu_train_eval_micro, self.accu_dev_eval_micro, self.accu_test_eval_micro = Eval(
), Eval(), Eval()
self.accu_train_eval_macro, self.accu_dev_eval_macro, self.accu_test_eval_macro = [], [], []
for i in range(self.config.accu_class_num):
self.accu_train_eval_macro.append(Eval())
self.accu_dev_eval_macro.append(Eval())
self.accu_test_eval_macro.append(Eval())
# define law eval
self.law_train_eval_micro, self.law_dev_eval_micro, self.law_test_eval_micro = Eval(
), Eval(), Eval()
self.law_train_eval_macro, self.law_dev_eval_macro, self.law_test_eval_macro = [], [], []
for i in range(self.config.law_class_num):
self.law_train_eval_macro.append(Eval())
self.law_dev_eval_macro.append(Eval())
self.law_test_eval_macro.append(Eval())
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
"""
:param clip_max_norm_use: whether to use clip max norm for nn model
:param clip_max_norm: clip max norm max values [float or None]
:return:
"""
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm_(self.model.parameters(), max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
"""
:param config: config
:param epoch: epoch
:param new_lr: learning rate
:return:
"""
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
# print("epoch", epoch)
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, epoch, init_lr):
"""
Args:
epoch: int, epoch
init_lr: initial lr
"""
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
# print('learning rate: {0}'.format(lr))
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
"""
:return:
"""
if backward_count % config.backward_batch_size == 0 or backward_count == self.train_iter_len:
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch):
"""
:param epoch:
:return:
"""
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
print("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
print(
"Early Stop Train. Best Score Locate on {} Epoch.".format(
self.best_score.best_epoch))
exit()
def train(self):
"""
:return:
"""
epochs = self.config.epochs
clip_max_norm_use = self.config.clip_max_norm_use
clip_max_norm = self.config.clip_max_norm
new_lr = self.config.learning_rate
for epoch in range(1, epochs + 1):
print("\n## The {} Epoch, All {} Epochs ! ##".format(
epoch, epochs))
new_lr = self._dynamic_lr(config=self.config,
epoch=epoch,
new_lr=new_lr)
# self.optimizer = self._decay_learning_rate(epoch=epoch - 1, init_lr=self.config.learning_rate)
print("now lr is {}".format(
self.optimizer.param_groups[0].get("lr")),
end="")
start_time = time.time()
random.shuffle(self.train_iter)
self.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
for batch_count, batch_features in enumerate(self.train_iter):
backward_count += 1
# self.optimizer.zero_grad()
accu, law, e_time, d_time = self.model(batch_features)
accu_logit = accu.view(
accu.size(0) * accu.size(1), accu.size(2))
law_logit = law.view(law.size(0) * law.size(1), law.size(2))
# print(accu_logit.size())
# accu_logit = torch_max_one(accu_logit)
# law_logit = torch_max_one(law_logit)
# print(batch_features.accu_label_features.size())
loss_accu = self.loss_function(
accu_logit, batch_features.accu_label_features)
loss_law = self.loss_function(
law_logit, batch_features.law_label_features)
# total_loss = (loss_accu + loss_law)
total_loss = (loss_accu + loss_law) / 2
# loss.backward()
total_loss.backward()
self._clip_model_norm(clip_max_norm_use, clip_max_norm)
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
# self.optimizer.step()
steps += 1
if (steps - 1) % self.config.log_interval == 0:
self.accu_train_eval_micro.clear_PRF()
for i in range(self.config.accu_class_num):
self.accu_train_eval_macro[i].clear_PRF()
F1_measure(accu,
batch_features.accu_label_features,
self.accu_train_eval_micro,
self.accu_train_eval_macro,
cuda=self.cuda)
(accu_p_avg, accu_r_avg,
accu_f_avg), (p_micro, r_micro,
f1_micro), (p_macro_avg, r_macro_avg,
f1_macro_avg) = getFscore_Avg(
self.accu_train_eval_micro,
self.accu_train_eval_macro,
accu.size(1))
sys.stdout.write(
"\nbatch_count = [{}/{}] , total_loss is {:.6f}, [accu-Micro-F1 is {:.6f}%]"
.format(batch_count + 1, self.train_iter_len,
total_loss.item(), f1_micro))
end_time = time.time()
print("\nTrain Time {:.3f}".format(end_time - start_time), end="")
self.eval(model=self.model, epoch=epoch, config=self.config)
self._model2file(model=self.model, config=self.config, epoch=epoch)
self._early_stop(epoch=epoch)
def eval(self, model, epoch, config):
"""
:param model: nn model
:param epoch: epoch
:param config: config
:return:
"""
self.accu_dev_eval_micro.clear_PRF()
for i in range(self.config.accu_class_num):
self.accu_dev_eval_macro[i].clear_PRF()
self.law_dev_eval_micro.clear_PRF()
for i in range(self.config.law_class_num):
self.law_dev_eval_macro[i].clear_PRF()
eval_start_time = time.time()
self._eval_batch(self.dev_iter,
model,
self.accu_dev_eval_micro,
self.accu_dev_eval_macro,
self.law_dev_eval_micro,
self.law_dev_eval_macro,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
print("Dev Time {:.3f}".format(eval_end_time - eval_start_time))
self.accu_test_eval_micro.clear_PRF()
for i in range(self.config.accu_class_num):
self.accu_test_eval_macro[i].clear_PRF()
self.law_test_eval_micro.clear_PRF()
for i in range(self.config.law_class_num):
self.law_test_eval_macro[i].clear_PRF()
eval_start_time = time.time()
self._eval_batch(self.test_iter,
model,
self.accu_test_eval_micro,
self.accu_test_eval_macro,
self.law_test_eval_micro,
self.law_test_eval_macro,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
print("Test Time {:.3f}".format(eval_end_time - eval_start_time))
def _model2file(self, model, config, epoch):
"""
:param model: nn model
:param config: config
:param epoch: epoch
:return:
"""
if config.save_model and config.save_all_model:
save_model_all(model, config.save_dir, config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config.save_best_model_path,
config.model_name, self.best_score)
else:
print()
def _eval_batch(self,
data_iter,
model,
accu_eval_micro,
accu_eval_macro,
law_eval_micro,
law_eval_macro,
best_score,
epoch,
config,
test=False):
"""
:param data_iter:
:param model:
:param accu_eval_micro:
:param accu_eval_macro:
:param best_score:
:param epoch:
:param config:
:param test:
:return:
"""
model.eval()
for batch_count, batch_features in enumerate(data_iter):
accu, law, e_time, d_time = model(batch_features)
F1_measure(accu,
batch_features.accu_label_features,
accu_eval_micro,
accu_eval_macro,
cuda=self.cuda)
F1_measure(law,
batch_features.law_label_features,
law_eval_micro,
law_eval_macro,
cuda=self.cuda)
# get f-score
accu_macro_micro_avg, accu_micro, accu_macro = getFscore_Avg(
accu_eval_micro, accu_eval_macro, accu.size(1))
law_macro_micro_avg, law_micro, law_macro = getFscore_Avg(
law_eval_micro, law_eval_macro, law.size(1))
accu_p, accu_r, accu_f = accu_macro_micro_avg
accu_p_ma, accu_r_ma, accu_f_ma = accu_macro
accu_p_mi, accu_r_mi, accu_f_mi = accu_micro
law_p, law_r, law_f = law_macro_micro_avg
law_p_ma, law_r_ma, law_f_ma = law_macro
law_p_mi, law_r_mi, law_f_mi = law_micro
p, r, f = accu_p, accu_r, accu_f
# p, r, f = law_p, law_r, law_f
test_flag = "Test"
if test is False:
print()
test_flag = "Dev"
best_score.current_dev_score = f
if f >= best_score.best_dev_score:
best_score.best_dev_score = f
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True:
best_score.p = p
best_score.r = r
best_score.f = f
print("{}:".format(test_flag))
print("Macro_Micro_Avg ===>>> ")
print(
"Eval: accu --- Precision = {:.6f}% Recall = {:.6f}% , F-Score = {:.6f}%"
.format(accu_p, accu_r, accu_f))
print(
"Eval: law --- Precision = {:.6f}% Recall = {:.6f}% , F-Score = {:.6f}%"
.format(law_p, law_r, law_f))
print("Macro ===>>> ")
print(
"Eval: accu --- Precision = {:.6f}% Recall = {:.6f}% , F-Score = {:.6f}%"
.format(accu_p_ma, accu_r_ma, accu_f_ma))
print(
"Eval: law --- Precision = {:.6f}% Recall = {:.6f}% , F-Score = {:.6f}%"
.format(law_p_ma, law_r_ma, law_f_ma))
print("Micro ===>>> ")
print(
"Eval: accu --- Precision = {:.6f}% Recall = {:.6f}% , F-Score = {:.6f}%"
.format(accu_p_mi, accu_r_mi, accu_f_mi))
print(
"Eval: law --- Precision = {:.6f}% Recall = {:.6f}% , F-Score = {:.6f}%"
.format(law_p_mi, law_r_mi, law_f_mi))
if test is True:
print(
"The Current Best accu Dev F-score: {:.6f}, Locate on {} Epoch."
.format(best_score.best_dev_score, best_score.best_epoch))
# print("The Current Best Law Dev F-score: {:.6f}, Locate on {} Epoch.".format(best_score.best_dev_score, best_score.best_epoch))
if test is True:
best_score.best_test = False
class Train(object):
def __init__(self, **kwargs):
self.config = kwargs["config"]
self.config.logger.info("Training Start......")
self.train_iter = kwargs["train_iter"]
self.dev_iter = kwargs["dev_iter"]
self.test_iter = kwargs["test_iter"]
self.model = kwargs["model"]
self.use_crf = self.config.use_crf
self.average_batch = self.config.average_batch
self.early_max_patience = self.config.early_max_patience
self.optimizer = Optimizer(name=self.config.learning_algorithm,
model=self.model,
lr=self.config.learning_rate,
weight_decay=self.config.weight_decay,
grad_clip=self.config.clip_max_norm)
self.loss_function = self._loss(
learning_algorithm=self.config.learning_algorithm,
label_paddingId=self.config.label_paddingId,
use_crf=self.use_crf)
self.config.logger.info(self.optimizer)
self.config.logger.info(self.loss_function)
self.best_score = Best_Result()
self.train_eval, self.dev_eval, self.test_eval = Eval(), Eval(), Eval()
self.train_iter_len = len(self.train_iter)
def _loss(self, learning_algorithm, label_paddingId, use_crf=False):
if use_crf:
loss_function = self.model.crf_layer.neg_log_likelihood_loss
return loss_function
elif learning_algorithm == "SGD":
loss_function = nn.CrossEntropyLoss(ignore_index=label_paddingId,
reduction="sum")
return loss_function
else:
loss_function = nn.CrossEntropyLoss(ignore_index=label_paddingId,
reduction="mean")
return loss_function
def _clip_model_norm(self, clip_max_norm_use, clip_max_norm):
if clip_max_norm_use is True:
gclip = None if clip_max_norm == "None" else float(clip_max_norm)
assert isinstance(gclip, float)
utils.clip_grad_norm_(self.model.parameters(), max_norm=gclip)
def _dynamic_lr(self, config, epoch, new_lr):
if config.use_lr_decay is True and epoch > config.max_patience and (
epoch -
1) % config.max_patience == 0 and new_lr > config.min_lrate:
new_lr = max(new_lr * config.lr_rate_decay, config.min_lrate)
set_lrate(self.optimizer, new_lr)
return new_lr
def _decay_learning_rate(self, epoch, init_lr):
lr = init_lr / (1 + self.config.lr_rate_decay * epoch)
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
return self.optimizer
def _optimizer_batch_step(self, config, backward_count):
if backward_count % config.backward_batch_size == 0 or backward_count == self.train_iter_len:
self.optimizer.step()
self.optimizer.zero_grad()
def _early_stop(self, epoch, config):
best_epoch = self.best_score.best_epoch
if epoch > best_epoch:
self.best_score.early_current_patience += 1
self.config.logger.info("Dev Has Not Promote {} / {}".format(
self.best_score.early_current_patience,
self.early_max_patience))
if self.best_score.early_current_patience >= self.early_max_patience:
self.end_of_epoch = epoch
self.config.logger.info(
"\n\nEarly Stop Train. Best Score Locate on {} Epoch.".
format(self.best_score.best_epoch))
self.save_training_summary()
exit()
@staticmethod
def _get_model_args(batch_features):
word = batch_features.word_features
mask = word > 0
sentence_length = batch_features.sentence_length
tags = batch_features.label_features
return word, mask, sentence_length, tags
def _calculate_loss(self, feats, mask, tags):
if not self.use_crf:
batch_size, max_len = feats.size(0), feats.size(1)
lstm_feats = feats.view(batch_size * max_len, -1)
tags = tags.view(-1)
return self.loss_function(lstm_feats, tags)
else:
loss_value = self.loss_function(feats, mask, tags)
if self.average_batch:
batch_size = feats.size(0)
loss_value /= float(batch_size)
return loss_value
def train(self):
epochs = self.config.epochs
clip_max_norm_use = self.config.clip_max_norm_use
clip_max_norm = self.config.clip_max_norm
new_lr = self.config.learning_rate
self.config.logger.info('\n\n')
self.config.logger.info('=-' * 50)
self.config.logger.info('batch number: %d' % len(self.train_iter))
for epoch in range(1, epochs + 1):
self.config.logger.info("\n\n### Epoch: {}/{} ###".format(
epoch, epochs))
self.optimizer = self._decay_learning_rate(
epoch=epoch - 1, init_lr=self.config.learning_rate)
self.config.logger.info("current lr: {}".format(
self.optimizer.param_groups[0].get("lr")))
start_time = time.time()
random.shuffle(self.train_iter)
self.model.train()
steps = 1
backward_count = 0
self.optimizer.zero_grad()
self.config.logger.info('=-' * 10)
for batch_count, batch_features in enumerate(self.train_iter):
backward_count += 1
word, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = self.model(word, sentence_length, train=True)
loss = self._calculate_loss(logit, mask, tags)
loss.backward()
self._clip_model_norm(clip_max_norm_use, clip_max_norm)
self._optimizer_batch_step(config=self.config,
backward_count=backward_count)
steps += 1
if (steps - 1) % self.config.log_interval == 0:
self.getAcc(self.train_eval, batch_features, logit,
self.config)
self.config.logger.info(
"batch_count:{} , loss: {:.4f}, [TAG-ACC: {:.4f}%]".
format(batch_count + 1, loss.item(),
self.train_eval.acc()))
end_time = time.time()
self.config.logger.info("Train Time {:.3f}".format(end_time -
start_time))
self.config.logger.info('=-' * 10)
self.eval(model=self.model, epoch=epoch, config=self.config)
self.config.logger.info('=-' * 10)
self._model2file(model=self.model, config=self.config, epoch=epoch)
self._early_stop(epoch=epoch, config=self.config)
self.config.logger.info('=-' * 15)
self.save_training_summary()
def save_training_summary(self):
self.config.logger.info(
"Copy the last model ckps to {} as backup.".format(
self.config.save_dir))
shutil.copytree(
self.config.save_model_dir, "/".join(
[self.config.save_dir, self.config.save_model_dir + "_bak"]))
self.config.logger.info(
"save the training summary at end of the log file.")
self.config.logger.info("\n")
self.config.logger.info("*" * 25)
par_path = os.path.dirname(self.config.train_file)
self.config.logger.info("dataset:\n\t %s" % par_path)
self.config.logger.info("\ttrain set count: %d" %
self.config.train_cnt)
self.config.logger.info("\tdev set count: %d" % self.config.dev_cnt)
self.config.logger.info("\ttest set count: %d" % self.config.test_cnt)
self.config.logger.info("*" * 10)
self.config.logger.info("model:")
self.config.logger.info(self.model)
self.config.logger.info("*" * 10)
self.config.logger.info("training:")
self.config.logger.info('\tbatch size: %d' % self.config.batch_size)
self.config.logger.info('\tbatch count: %d' % len(self.train_iter))
self.config.logger.info("*" * 10)
self.config.logger.info("best performance:")
self.config.logger.info("\tend at epoch: %d" % self.end_of_epoch)
self.config.logger.info("\tbest at epoch: %d" %
self.best_score.best_epoch)
self.config.logger.info("\tdev(%):")
self.config.logger.info("\t\tprecision, %.5f" %
self.best_score.best_dev_p_score)
self.config.logger.info("\t\trecall, %.5f" %
self.best_score.best_dev_r_score)
self.config.logger.info("\t\tf1, %.5f" %
self.best_score.best_dev_f1_score)
self.config.logger.info("\ttest(%):")
self.config.logger.info("\t\tprecision, %.5f" % self.best_score.p)
self.config.logger.info("\t\trecall, %.5f" % self.best_score.r)
self.config.logger.info("\t\tf1, %.5f" % self.best_score.f)
self.config.logger.info("*" * 25)
def eval(self, model, epoch, config):
self.dev_eval.clear_PRF()
eval_start_time = time.time()
self.eval_batch(self.dev_iter,
model,
self.dev_eval,
self.best_score,
epoch,
config,
test=False)
eval_end_time = time.time()
self.config.logger.info("Dev Time: {:.3f}".format(eval_end_time -
eval_start_time))
self.config.logger.info('=-' * 10)
self.test_eval.clear_PRF()
eval_start_time = time.time()
self.eval_batch(self.test_iter,
model,
self.test_eval,
self.best_score,
epoch,
config,
test=True)
eval_end_time = time.time()
self.config.logger.info("Test Time: {:.3f}".format(eval_end_time -
eval_start_time))
def _model2file(self, model, config, epoch):
if config.save_model and config.save_all_model:
save_model_all(model, config, config.save_model_dir,
config.model_name, epoch)
elif config.save_model and config.save_best_model:
save_best_model(model, config, config.save_model_dir,
config.model_name, self.best_score)
else:
self.config.logger.info()
def eval_batch(self,
data_iter,
model,
eval_instance,
best_score,
epoch,
config,
test=False):
test_flag = "Test"
if test is False: # dev
test_flag = "Dev"
model.eval() # set flag for pytorch
eval_PRF = EvalPRF()
gold_labels = []
predict_labels = []
for batch_features in data_iter:
word, mask, sentence_length, tags = self._get_model_args(
batch_features)
logit = model(word, sentence_length, train=False)
if self.use_crf is False:
predict_ids = torch_max(logit)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
for id_word in range(inst.words_size):
predict_label.append(
config.create_alphabet.label_alphabet.from_id(
label_ids[id_word]))
gold_labels.append(inst.labels)
predict_labels.append(predict_label)
else:
path_score, best_paths = model.crf_layer(logit, mask)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
gold_labels.append(inst.labels)
label_ids = best_paths[id_batch].cpu().data.numpy(
)[:inst.words_size]
label = []
for i in label_ids:
# self.config.logger.info("\n", i)
label.append(
config.create_alphabet.label_alphabet.from_id(
int(i)))
predict_labels.append(label)
for p_label, g_label in zip(predict_labels, gold_labels):
eval_PRF.evalPRF(predict_labels=p_label,
gold_labels=g_label,
eval=eval_instance)
cor = 0
totol_leng = sum(
[len(predict_label) for predict_label in predict_labels])
for p_lable, g_lable in zip(predict_labels, gold_labels):
for p_lable_, g_lable_ in zip(p_lable, g_lable):
if p_lable_ == g_lable_:
cor += 1
acc_ = cor / totol_leng * 100
p, r, f = eval_instance.getFscore()
if test is False: # dev
best_score.current_dev_score = f
if f >= best_score.best_dev_f1_score:
best_score.best_dev_f1_score = f
best_score.best_dev_p_score = p
best_score.best_dev_r_score = r
best_score.best_epoch = epoch
best_score.best_test = True
if test is True and best_score.best_test is True: # test
best_score.p = p
best_score.r = r
best_score.f = f
self.config.logger.info(
"{} at current epoch, precision: {:.4f}% recall: {:.4f}% , f-score: {:.4f}%, [TAG-ACC: {:.3f}%]"
.format(test_flag, p, r, f, acc_))
if test is False:
self.config.logger.info(
"Till now, The Best Dev Result: precision: {:.4f}% recall: {:.4f}% , f-score: {:.4f}%, Locate on {} Epoch."
.format(best_score.best_dev_p_score,
best_score.best_dev_r_score,
best_score.best_dev_f1_score, best_score.best_epoch))
elif test is True:
self.config.logger.info(
"Till now, The Best Test Result: precision: {:.4f}% recall: {:.4f}% , f-score: {:.4f}%, Locate on {} Epoch."
.format(best_score.p, best_score.r, best_score.f,
best_score.best_epoch))
best_score.best_test = False
@staticmethod
def getAcc(eval_acc, batch_features, logit, config):
eval_acc.clear_PRF()
predict_ids = torch_max(logit)
for id_batch in range(batch_features.batch_length):
inst = batch_features.inst[id_batch]
label_ids = predict_ids[id_batch]
predict_label = []
gold_lable = inst.labels
for id_word in range(inst.words_size):
predict_label.append(
config.create_alphabet.label_alphabet.from_id(
label_ids[id_word]))
assert len(predict_label) == len(gold_lable)
cor = 0
for p_lable, g_lable in zip(predict_label, gold_lable):
if p_lable == g_lable:
cor += 1
eval_acc.correct_num += cor
eval_acc.gold_num += len(gold_lable)