def test(self):
# turn on the testing mode; clean up the history
network = self._model
self._mode(network, is_test=True)
data_iterator = Batch(self.data, self.batch_size, sampler=SequentialSampler(), as_numpy=False)
eval_results = {}
try:
with torch.no_grad():
for batch_x, batch_y in data_iterator:
_move_dict_value_to_device(batch_x, batch_y, device=self._model_device)
pred_dict = self._data_forward(self._predict_func, batch_x)
if not isinstance(pred_dict, dict):
raise TypeError(f"The return value of {get_func_signature(self._predict_func)} "
f"must be `dict`, got {type(pred_dict)}.")
for metric in self.metrics:
metric(pred_dict, batch_y)
for metric in self.metrics:
eval_result = metric.get_metric()
if not isinstance(eval_result, dict):
raise TypeError(f"The return value of {get_func_signature(metric.get_metric)} must be "
f"`dict`, got {type(eval_result)}")
metric_name = metric.__class__.__name__
eval_results[metric_name] = eval_result
except CheckError as e:
prev_func_signature = get_func_signature(self._predict_func)
_check_loss_evaluate(prev_func_signature=prev_func_signature, func_signature=e.func_signature,
check_res=e.check_res, pred_dict=pred_dict, target_dict=batch_y,
dataset=self.data, check_level=0)
if self.verbose >= 1:
print("[tester] \n{}".format(self._format_eval_results(eval_results)))
self._mode(network, is_test=False)
return eval_results
def get_predictions(pred_model, input_data, batch_size, num_workers=4):
texts = list(list(map(lambda x: vocabs['char'].to_word(x), sample['chars'])) for sample in input_data)
seq_lens = [sample['seq_len'] for sample in input_data]
pred_model.to(device)
sampler = SequentialSampler()
data_iterator = DataSetIter(dataset=input_data, batch_size=batch_size, sampler=sampler,
num_workers=num_workers)
with torch.no_grad():
preds, golds = [], []
pred_model.eval()
for batch_x, batch_y in data_iterator:
_move_dict_value_to_device(batch_x, batch_y, device=device)
x = _build_args(pred_model.forward, **batch_x)
with torch.no_grad():
y = pred_model.forward(**x)
preds.extend(list(map(list, y['pred'].cpu().numpy())))
golds.extend(list(map(list, batch_y['target'].cpu().numpy())))
pred_seqs = list(list(map(lambda _y: vocabs['label'].to_word(_y), pred)) for pred in preds)
gold_seqs = list(list(map(lambda _y: vocabs['label'].to_word(_y), pred)) for pred in golds)
case_result = []
for pred_seq, gold_seq, word_seq, seq_len in zip(pred_seqs, gold_seqs, texts, seq_lens):
pred_seq = pred_seq[:seq_len]
gold_seq = gold_seq[:seq_len]
case_result.append((''.join(word_seq), extract_kvpairs_in_bmoes(gold_seq, word_seq),
extract_kvpairs_in_bmoes(pred_seq, word_seq)))
# output for case study
os.makedirs(f'../output/case_study/{args.dataset}', exist_ok=True)
fout = open(f'../output/case_study/{args.dataset}/{args.dataset}_bert{args.use_bert}_scheme{args.new_tag_scheme}_ple{args.ple_channel_num}_plstm{int(args.use_ple_lstm)}_trainrate{args.train_dataset_rate}.casestudy', 'w', encoding='utf8')
for word_seq, gold_pair, pred_pair in case_result:
fout.write(word_seq + '\n' + str(gold_pair) + '\n' + str(pred_pair) + '\n\n')
def is_phrase_match_BERT(phrase1, phrase2):
"""
Determine if two phrases match
:param phrase1: phrase1
:param phrase2: phrase2
"""
from fastNLP import DataSetIter, DataSet
from fastNLP.core.utils import _move_dict_value_to_device
from my_bert_match import addWords, addWordPiece, processItem, processNum, addSeqlen
# 0 for not match,1 for match
testset = DataSet({"raw_words": [f"{phrase1}::{phrase2}"]})
testset.apply(addWords, new_field_name="p_words")
testset.apply(addWordPiece, new_field_name="t_words")
testset.apply(processItem, new_field_name="word_pieces")
testset.apply(processNum, new_field_name="word_nums")
testset.apply(addSeqlen, new_field_name="seq_len")
testset.field_arrays["word_pieces"].is_input = True
testset.field_arrays["seq_len"].is_input = True
testset.field_arrays["word_nums"].is_input = True
# print(testset)
with torch.no_grad():
bert_model.eval()
test_batch = DataSetIter(batch_size=1, dataset=testset, sampler=None)
outputs = []
for batch_x, batch_y in test_batch:
_move_dict_value_to_device(batch_x, batch_y, device=device)
outputs.append(bert_model.forward(batch_x["word_pieces"], batch_x["word_nums"], batch_x["seq_len"])['pred'])
outputs = torch.cat(outputs)
outputs = torch.nn.functional.softmax(outputs, dim=1)
return ["Not Match", "Related", "Match"][outputs.argmax().item()]
def test():
from fastNLP import DataSetIter, DataSet
# 0 for not match,1 for match
testset = DataSet({"raw_words": ["5::five"]})
testset.apply(addWords, new_field_name="p_words")
testset.apply(addWordPiece, new_field_name="t_words")
testset.apply(processItem, new_field_name="word_pieces")
testset.apply(processNum, new_field_name="word_nums")
testset.apply(addSeqlen, new_field_name="seq_len")
testset.field_arrays["word_pieces"].is_input = True
testset.field_arrays["seq_len"].is_input = True
testset.field_arrays["word_nums"].is_input = True
# print(testset)
from fastNLP.io import ModelLoader
loader = ModelLoader()
if torch.cuda.is_available():
model = loader.load_pytorch_model(
"../models/bert_model_max_triple.pkl")
else:
model = torch.load("../models/bert_model_max_triple.pkl",
map_location="cpu")
model.eval()
test_batch = DataSetIter(batch_size=1, dataset=testset, sampler=None)
outputs = []
for batch_x, batch_y in test_batch:
_move_dict_value_to_device(batch_x,
batch_y,
device=_get_model_device(model))
outputs.append(
model.forward(batch_x["word_pieces"], batch_x["word_nums"],
batch_x["seq_len"])['pred'])
outputs = torch.cat(outputs)
outputs = torch.nn.functional.softmax(outputs, dim=1)
return outputs
def _tqdm_train(self):
self.step = 0
data_iterator = Batch(self.train_data,
batch_size=self.batch_size,
sampler=self.sampler,
as_numpy=False)
total_steps = data_iterator.num_batches * self.n_epochs
with tqdm(total=total_steps,
postfix='loss:{0:<6.5f}',
leave=False,
dynamic_ncols=True) as pbar:
avg_loss = 0
for epoch in range(1, self.n_epochs + 1):
pbar.set_description_str(
desc="Epoch {}/{}".format(epoch, self.n_epochs))
for batch_x, batch_y in data_iterator:
_move_dict_value_to_device(batch_x,
batch_y,
device=self._model_device)
prediction = self._data_forward(self.model, batch_x)
loss = self._compute_loss(prediction, batch_y)
avg_loss += loss.item()
self._grad_backward(loss)
self._update()
self._summary_writer.add_scalar("loss",
loss.item(),
global_step=self.step)
for name, param in self.model.named_parameters():
if param.requires_grad:
self._summary_writer.add_scalar(
name + "_mean",
param.mean(),
global_step=self.step)
# self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.step)
# self._summary_writer.add_scalar(name + "_grad_sum", param.sum(), global_step=self.step)
if (self.step + 1) % self.print_every == 0:
pbar.set_postfix_str("loss:{0:<6.5f}".format(
avg_loss / self.print_every))
avg_loss = 0
pbar.update(self.print_every)
self.step += 1
if self.validate_every > 0 and self.step % self.validate_every == 0 \
and self.dev_data is not None:
eval_res = self._do_validation(epoch=epoch,
step=self.step)
eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \
self.tester._format_eval_results(eval_res)
pbar.write(eval_str)
if self.validate_every < 0 and self.dev_data:
eval_res = self._do_validation(epoch=epoch, step=self.step)
eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \
self.tester._format_eval_results(eval_res)
pbar.write(eval_str)
if epoch != self.n_epochs:
data_iterator = Batch(self.train_data,
batch_size=self.batch_size,
sampler=self.sampler,
as_numpy=False)
pbar.close()
def predict(self, data: DataSet, seq_len_field_name=None):
r"""用已经训练好的模型进行inference.
:param fastNLP.DataSet data: 待预测的数据集
:param str seq_len_field_name: 表示序列长度信息的field名字
:return: dict dict里面的内容为模型预测的结果
"""
if not isinstance(data, DataSet):
raise ValueError("Only Dataset class is allowed, not {}.".format(
type(data)))
if seq_len_field_name is not None and seq_len_field_name not in data.field_arrays:
raise ValueError("Field name {} not found in DataSet {}.".format(
seq_len_field_name, data))
prev_training = self.network.training
self.network.eval()
network_device = _get_model_device(self.network)
batch_output = defaultdict(list)
data_iterator = DataSetIter(data,
batch_size=self.batch_size,
sampler=SequentialSampler(),
as_numpy=False)
if hasattr(self.network, "predict"):
predict_func = self.network.predict
else:
predict_func = self.network.forward
with torch.no_grad():
for batch_x, _ in data_iterator:
_move_dict_value_to_device(batch_x, _, device=network_device)
refined_batch_x = _build_args(predict_func, **batch_x)
prediction = predict_func(**refined_batch_x)
if seq_len_field_name is not None:
seq_lens = batch_x[seq_len_field_name].tolist()
for key, value in prediction.items():
value = value.cpu().numpy()
if len(value.shape) == 1 or (len(value.shape) == 2
and value.shape[1] == 1):
batch_output[key].extend(value.tolist())
else:
if seq_len_field_name is not None:
tmp_batch = []
for idx, seq_len in enumerate(seq_lens):
tmp_batch.append(value[idx, :seq_len])
batch_output[key].extend(tmp_batch)
else:
batch_output[key].append(value)
self.network.train(prev_training)
return batch_output
def produceCandidateTripleSlow(raw_phrase, Candidate_phrases, model,
Candidate_hpos_sub, threshold):
"""
使用BERT判断Candidate_phrases中哪个与raw_phrase语义最接近；基于最大值方式；适用于单个处理
"""
from fastNLP.core.utils import _move_dict_value_to_device
from fastNLP.core.utils import _get_model_device
from fastNLP import DataSet
from fastNLP import DataSetIter
from my_bert_match import addWordPiece, addSeqlen, addWords, processItem, processNum
p_Candidate_phrases = [
raw_phrase + "::" + item for item in Candidate_phrases
]
Candidate_dataset = DataSet({"raw_words": p_Candidate_phrases})
Candidate_dataset.apply(addWords, new_field_name="p_words")
Candidate_dataset.apply(addWordPiece, new_field_name="t_words")
Candidate_dataset.apply(processItem, new_field_name="word_pieces")
Candidate_dataset.apply(processNum, new_field_name="word_nums")
Candidate_dataset.apply(addSeqlen, new_field_name="seq_len")
Candidate_dataset.field_arrays["word_pieces"].is_input = True
Candidate_dataset.field_arrays["seq_len"].is_input = True
Candidate_dataset.field_arrays["word_nums"].is_input = True
test_batch = DataSetIter(batch_size=10,
dataset=Candidate_dataset,
sampler=None)
outputs = []
for batch_x, batch_y in test_batch:
_move_dict_value_to_device(batch_x,
batch_y,
device=_get_model_device(model))
outputs.append(
model.forward(batch_x["word_pieces"], batch_x["word_nums"],
batch_x["seq_len"])['pred'])
outputs = torch.cat(outputs)
outputs = torch.nn.functional.softmax(outputs,
dim=1).cpu().detach().numpy()
results_2 = np.array([item[2] for item in outputs])
results_1 = np.array([item[1] for item in outputs])
# 如果这里已经能找到精确匹配的就直接输出
if max(results_2) >= threshold:
return Candidate_hpos_sub[int(
np.argmax(results_2))], max(results_2), "2"
if max(results_1) >= threshold:
return Candidate_hpos_sub[int(
np.argmax(results_1))], max(results_1), "1"
return "None", None, "0"
def _print_train(self):
epoch = 1
start = time.time()
while epoch <= self.n_epochs:
data_iterator = Batch(self.train_data,
batch_size=self.batch_size,
sampler=self.sampler,
as_numpy=False)
for batch_x, batch_y in data_iterator:
# TODO 这里可能会遇到问题，万一用户在model内部修改了prediction的device就会有问题
_move_dict_value_to_device(batch_x,
batch_y,
device=self._model_device)
prediction = self._data_forward(self.model, batch_x)
loss = self._compute_loss(prediction, batch_y)
self._grad_backward(loss)
self._update()
self._summary_writer.add_scalar("loss",
loss.item(),
global_step=self.step)
for name, param in self.model.named_parameters():
if param.requires_grad:
self._summary_writer.add_scalar(name + "_mean",
param.mean(),
global_step=self.step)
# self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.step)
# self._summary_writer.add_scalar(name + "_grad_sum", param.sum(), global_step=self.step)
if self.print_every > 0 and self.step % self.print_every == 0:
end = time.time()
diff = timedelta(seconds=round(end - start))
print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format(
epoch, self.step, loss.data, diff)
print(print_output)
if (self.validate_every > 0
and self.step % self.validate_every == 0
and self.dev_data is not None):
self._do_validation(epoch=epoch, step=self.step)
self.step += 1
# validate_every override validation at end of epochs
if self.dev_data and self.validate_every <= 0:
self._do_validation(epoch=epoch, step=self.step)
epoch += 1
def predict(self, data: DataSet, seq_len_field_name=None):
r"""
"""
if not isinstance(data, DataSet):
raise ValueError(
"Only Dataset class is allowed, not {}.".format(type(data)))
if seq_len_field_name is not None and seq_len_field_name not in data.field_arrays:
raise ValueError("Field name {} not found in DataSet {}.".format(
seq_len_field_name, data))
self.network.eval() # self.network.module for multi-GPU
network_device = _get_model_device(self.network)
batch_output = defaultdict(list)
data_iterator = DataSetIter(
data, batch_size=self.batch_size, sampler=SequentialSampler(), as_numpy=False)
# predict_func = self.network.module.predict # self.network.module for
# multi-GPU
try:
predict_func = self.network.predict
except ModuleAttributeError:
predict_func = self.network.module.predict
with torch.no_grad():
# for batch_x, _ in tqdm(data_iterator):
for batch_x, _ in tqdm(data_iterator, total=len(data_iterator)):
_move_dict_value_to_device(batch_x, _, device=network_device)
refined_batch_x = _build_args(predict_func, **batch_x)
prediction = predict_func(**refined_batch_x)
if seq_len_field_name is not None:
seq_lens = batch_x[seq_len_field_name].tolist()
for key, value in prediction.items():
value = value.cpu().numpy()
if len(value.shape) == 1 or (
len(value.shape) == 2 and value.shape[1] == 1):
batch_output[key].extend(value.tolist())
else:
if seq_len_field_name is not None:
tmp_batch = []
for idx, seq_len in enumerate(seq_lens):
tmp_batch.append(value[idx, :seq_len])
batch_output[key].extend(tmp_batch)
else:
batch_output[key].append(value)
return batch_output
def produceCandidateTriple(Candidate_hpos_sub_total, model, hpo_tree,
threshold):
"""
使用BERT判断Candidate_phrases中哪个与raw_phrase语义最接近；基于最大值方式
:param Candidate_hpos_sub_total: 输出的短语及候选HPO嵌套列表
:param model:
:param hpo_tree:
:param threshold: 用作该模型输出阈值
:return:
"""
from fastNLP.core.utils import _move_dict_value_to_device
from fastNLP.core.utils import _get_model_device
from fastNLP import DataSet
from fastNLP import DataSetIter
from my_bert_match import addWordPiece, addSeqlen, addWords, processItem, processNum
p_Candidate_phrases = []
phrase_nums_per_hpo = []
Candidate_hpos = []
for raw_phrase, Candidate_phrase, Candidate_hpos_sub in Candidate_hpos_sub_total:
p_Candidate_phrases.extend(
[raw_phrase + "::" + item for item in Candidate_phrase])
phrase_nums_per_hpo.append(len(Candidate_phrase))
Candidate_hpos.append(Candidate_hpos_sub)
Candidate_dataset = DataSet({"raw_words": p_Candidate_phrases})
Candidate_dataset.apply(addWords, new_field_name="p_words")
Candidate_dataset.apply(addWordPiece, new_field_name="t_words")
Candidate_dataset.apply(processItem, new_field_name="word_pieces")
Candidate_dataset.apply(processNum, new_field_name="word_nums")
Candidate_dataset.apply(addSeqlen, new_field_name="seq_len")
Candidate_dataset.field_arrays["word_pieces"].is_input = True
Candidate_dataset.field_arrays["seq_len"].is_input = True
Candidate_dataset.field_arrays["word_nums"].is_input = True
test_batch = DataSetIter(batch_size=128,
dataset=Candidate_dataset,
sampler=None)
outputs = []
for batch_x, batch_y in test_batch:
_move_dict_value_to_device(batch_x,
batch_y,
device=_get_model_device(model))
outputs.append(
model.forward(batch_x["word_pieces"], batch_x["word_nums"],
batch_x["seq_len"])['pred'])
outputs = torch.cat(outputs)
outputs = torch.nn.functional.softmax(outputs,
dim=1).cpu().detach().numpy()
# print(outputs.size)
results_2 = np.array([item[2] for item in outputs])
results_1 = np.array([item[1] for item in outputs])
# 按短语分组
count = 0
index = 0
ans = []
for group_num in phrase_nums_per_hpo:
g_results_2 = results_2[index:index + group_num]
g_results_1 = results_1[index:index + group_num]
Candidate_hpos_sub = Candidate_hpos[count]
index += group_num
count += 1
# 如果这里已经能找到精确匹配的就直接输出
if max(g_results_2) >= threshold:
ans.append([
Candidate_hpos_sub[int(np.argmax(g_results_2))],
max(g_results_2), "2"
])
continue
if max(g_results_1) >= threshold:
ans.append([
Candidate_hpos_sub[int(np.argmax(g_results_1))],
max(g_results_1), "1"
])
continue
ans.append(["None", None, "0"])
return ans
def _check_code(dataset,
model,
losser,
metrics,
batch_size=DEFAULT_CHECK_BATCH_SIZE,
dev_data=None,
metric_key=None,
check_level=0):
# check get_loss 方法
model_devcie = model.parameters().__next__().device
batch = Batch(dataset=dataset,
batch_size=batch_size,
sampler=SequentialSampler())
for batch_count, (batch_x, batch_y) in enumerate(batch):
_move_dict_value_to_device(batch_x, batch_y, device=model_devcie)
# forward check
if batch_count == 0:
info_str = ""
input_fields = _get_value_info(batch_x)
target_fields = _get_value_info(batch_y)
if len(input_fields) > 0:
info_str += "input fields after batch(if batch size is {}):\n".format(
batch_size)
info_str += "\n".join(input_fields)
info_str += '\n'
else:
raise RuntimeError("There is no input field.")
if len(target_fields) > 0:
info_str += "target fields after batch(if batch size is {}):\n".format(
batch_size)
info_str += "\n".join(target_fields)
info_str += '\n'
else:
info_str += 'There is no target field.'
print(info_str)
_check_forward_error(forward_func=model.forward,
dataset=dataset,
batch_x=batch_x,
check_level=check_level)
refined_batch_x = _build_args(model.forward, **batch_x)
pred_dict = model(**refined_batch_x)
func_signature = get_func_signature(model.forward)
if not isinstance(pred_dict, dict):
raise TypeError(
f"The return value of {func_signature} should be `dict`, not `{type(pred_dict)}`."
)
# loss check
try:
loss = losser(pred_dict, batch_y)
# check loss output
if batch_count == 0:
if not isinstance(loss, torch.Tensor):
raise TypeError(
f"The return value of {get_func_signature(losser.get_loss)} should be `torch.Tensor`, "
f"but got `{type(loss)}`.")
if len(loss.size()) != 0:
raise ValueError(
f"The size of return value of {get_func_signature(losser.get_loss)} is {loss.size()}, "
f"should be torch.size([])")
loss.backward()
except CheckError as e:
# TODO: another error raised if CheckError caught
pre_func_signature = get_func_signature(model.forward)
_check_loss_evaluate(prev_func_signature=pre_func_signature,
func_signature=e.func_signature,
check_res=e.check_res,
pred_dict=pred_dict,
target_dict=batch_y,
dataset=dataset,
check_level=check_level)
model.zero_grad()
if batch_count + 1 >= DEFAULT_CHECK_NUM_BATCH:
break
if dev_data is not None:
tester = Tester(data=dataset[:batch_size * DEFAULT_CHECK_NUM_BATCH],
model=model,
metrics=metrics,
batch_size=batch_size,
verbose=-1)
evaluate_results = tester.test()
_check_eval_results(metrics=evaluate_results,
metric_key=metric_key,
metric_list=metrics)
def train_shared(self, pbar=None, max_step=None, dag=None):
"""Train the language model for 400 steps of minibatches of 64
examples.
Args:
max_step: Used to run extra training steps as a warm-up.
dag: If not None, is used instead of calling sample().
BPTT is truncated at 35 timesteps.
For each weight update, gradients are estimated by sampling M models
from the fixed controller policy, and averaging their gradients
computed on a batch of training data.
"""
model = self.shared
model.train()
self.controller.eval()
hidden = self.shared.init_hidden(self.batch_size)
abs_max_grad = 0
abs_max_hidden_norm = 0
step = 0
raw_total_loss = 0
total_loss = 0
train_idx = 0
avg_loss = 0
data_iterator = Batch(self.train_data,
batch_size=self.batch_size,
sampler=self.sampler,
as_numpy=False,
prefetch=self.prefetch)
for batch_x, batch_y in data_iterator:
_move_dict_value_to_device(batch_x,
batch_y,
device=self._model_device)
indices = data_iterator.get_batch_indices()
# negative sampling; replace unknown; re-weight batch_y
self.callback_manager.on_batch_begin(batch_x, batch_y, indices)
# prediction = self._data_forward(self.model, batch_x)
dags = self.controller.sample(1)
inputs, targets = batch_x, batch_y
# self.callback_manager.on_loss_begin(batch_y, prediction)
loss, hidden, extra_out = self.get_loss(inputs, targets, hidden,
dags)
hidden.detach_()
avg_loss += loss.item()
# Is loss NaN or inf? requires_grad = False
self.callback_manager.on_backward_begin(loss, self.model)
self._grad_backward(loss)
self.callback_manager.on_backward_end(self.model)
self._update()
self.callback_manager.on_step_end(self.optimizer)
if (self.step + 1) % self.print_every == 0:
if self.use_tqdm:
print_output = "loss:{0:<6.5f}".format(avg_loss /
self.print_every)
pbar.update(self.print_every)
else:
end = time.time()
diff = timedelta(seconds=round(end - start))
print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format(
epoch, self.step, avg_loss, diff)
pbar.set_postfix_str(print_output)
avg_loss = 0
self.step += 1
step += 1
self.shared_step += 1
self.callback_manager.on_batch_end()
def _train(self):
if not self.use_tqdm:
from fastNLP.core.utils import pseudo_tqdm as inner_tqdm
else:
inner_tqdm = tqdm
self.step = 0
start = time.time()
data_iterator = Batch(self.train_data,
batch_size=self.batch_size,
sampler=self.sampler,
as_numpy=False)
total_steps = data_iterator.num_batches * self.n_epochs
with inner_tqdm(total=total_steps,
postfix='loss:{0:<6.5f}',
leave=False,
dynamic_ncols=True) as pbar:
avg_loss = 0
for epoch in range(1, self.n_epochs + 1):
pbar.set_description_str(
desc="Epoch {}/{}".format(epoch, self.n_epochs))
# early stopping
self.callback_manager.before_epoch(epoch, self.n_epochs)
for batch_x, batch_y in data_iterator:
indices = data_iterator.get_batch_indices()
# negative sampling; replace unknown; re-weight batch_y
self.callback_manager.before_batch(batch_x, batch_y,
indices)
_move_dict_value_to_device(batch_x,
batch_y,
device=self._model_device)
prediction = self._data_forward(self.model, batch_x)
# edit prediction
self.callback_manager.before_loss(batch_y, prediction)
loss = self._compute_loss(prediction, batch_y)
avg_loss += loss.item()
# Is loss NaN or inf? requires_grad = False
self.callback_manager.before_backward(loss, self.model)
self._grad_backward(loss)
# gradient clipping
self.callback_manager.after_backward(self.model)
self._update()
# lr scheduler; lr_finder; one_cycle
self.callback_manager.after_step(self.optimizer)
self._summary_writer.add_scalar("loss",
loss.item(),
global_step=self.step)
for name, param in self.model.named_parameters():
if param.requires_grad:
self._summary_writer.add_scalar(
name + "_mean",
param.mean(),
global_step=self.step)
# self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.step)
# self._summary_writer.add_scalar(name + "_grad_sum", param.sum(), global_step=self.step)
if (self.step + 1) % self.print_every == 0:
if self.use_tqdm:
print_output = "loss:{0:<6.5f}".format(
avg_loss / self.print_every)
pbar.update(self.print_every)
else:
end = time.time()
diff = timedelta(seconds=round(end - start))
print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format(
epoch, self.step, avg_loss, diff)
pbar.set_postfix_str(print_output)
avg_loss = 0
self.step += 1
# do nothing
self.callback_manager.after_batch()
if ((self.validate_every > 0 and self.step % self.validate_every == 0) or
(self.validate_every < 0 and self.step % len(data_iterator)) == 0) \
and self.dev_data is not None:
eval_res = self._do_validation(epoch=epoch,
step=self.step)
eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step,
total_steps) + \
self.tester._format_eval_results(eval_res)
pbar.write(eval_str)
# if self.validate_every < 0 and self.dev_data:
# eval_res = self._do_validation(epoch=epoch, step=self.step)
# eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \
# self.tester._format_eval_results(eval_res)
# pbar.write(eval_str)
if epoch != self.n_epochs:
data_iterator = Batch(self.train_data,
batch_size=self.batch_size,
sampler=self.sampler,
as_numpy=False)
# lr decay; early stopping
self.callback_manager.after_epoch(epoch, self.n_epochs,
self.optimizer)
pbar.close()
def _train(self):
if not self.use_tqdm:
from fastNLP.core.utils import pseudo_tqdm as inner_tqdm
else:
inner_tqdm = tqdm
self.step = 0
start = time.time()
total_steps = (len(self.train_data) // self.batch_size + int(
len(self.train_data) % self.batch_size != 0)) * self.n_epochs
with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar:
avg_loss = 0
data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False,
prefetch=self.prefetch)
for epoch in range(1, self.n_epochs+1):
pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs))
# early stopping
self.callback_manager.on_epoch_begin(epoch, self.n_epochs)
for batch_x, batch_y in data_iterator:
_move_dict_value_to_device(batch_x, batch_y, device=self._model_device)
indices = data_iterator.get_batch_indices()
# negative sampling; replace unknown; re-weight batch_y
self.callback_manager.on_batch_begin(batch_x, batch_y, indices)
prediction = self._data_forward(self.model, batch_x)
# edit prediction
self.callback_manager.on_loss_begin(batch_y, prediction)
loss = self._compute_loss(prediction, batch_y)
avg_loss += loss.item()
# Is loss NaN or inf? requires_grad = False
self.callback_manager.on_backward_begin(loss, self.model)
self._grad_backward(loss)
self.callback_manager.on_backward_end(self.model)
self._update()
self.callback_manager.on_step_end(self.optimizer)
if (self.step+1) % self.print_every == 0:
if self.use_tqdm:
print_output = "loss:{0:<6.5f}".format(avg_loss / self.print_every)
pbar.update(self.print_every)
else:
end = time.time()
diff = timedelta(seconds=round(end - start))
print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format(
epoch, self.step, avg_loss, diff)
pbar.set_postfix_str(print_output)
avg_loss = 0
self.step += 1
self.callback_manager.on_batch_end()
if ((self.validate_every > 0 and self.step % self.validate_every == 0) or
(self.validate_every < 0 and self.step % len(data_iterator) == 0)) \
and self.dev_data is not None:
eval_res = self._do_validation(epoch=epoch, step=self.step)
eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step,
total_steps) + \
self.tester._format_eval_results(eval_res)
pbar.write(eval_str)
# ================= mini-batch end ==================== #
# lr decay; early stopping
self.callback_manager.on_epoch_end(epoch, self.n_epochs, self.optimizer)
# =============== epochs end =================== #
pbar.close()
def train():
n_epochs = 10
train_set = data_set_loader._load('../models/all4bert_new_triple.txt')
train_set, tmp_set = train_set.split(0.2)
val_set, test_set = tmp_set.split(0.5)
data_bundle = [train_set, val_set, test_set]
for dataset in data_bundle:
dataset.apply(addWords, new_field_name="p_words")
dataset.apply(addWordPiece, new_field_name="t_words")
dataset.apply(processItem, new_field_name="word_pieces")
dataset.apply(processNum, new_field_name="word_nums")
dataset.apply(addSeqlen, new_field_name="seq_len")
dataset.apply(processTarget, new_field_name="target")
for dataset in data_bundle:
dataset.field_arrays["word_pieces"].is_input = True
dataset.field_arrays["seq_len"].is_input = True
dataset.field_arrays["word_nums"].is_input = True
dataset.field_arrays["target"].is_target = True
print("In total " + str(len(data_bundle)) + " datasets:")
print("Trainset has " + str(len(train_set)) + " instances.")
print("Validateset has " + str(len(val_set)) + " instances.")
print("Testset has " + str(len(test_set)) + " instances.")
train_set.print_field_meta()
# print(train_set)
from fastNLP.models.Mybert import BertForSentenceMatching
from fastNLP import AccuracyMetric, DataSetIter
from fastNLP.core.utils import _pseudo_tqdm as tqdm
# 注意这里是表明分的类数
model = BertForSentenceMatching(embed, 3)
if torch.cuda.is_available():
model = _move_model_to_device(model, device=0)
# print(model)
train_batch = DataSetIter(batch_size=16, dataset=train_set, sampler=None)
optimizer = torch.optim.Adam(model.parameters(), lr=2e-5)
Lossfunc = torch.nn.CrossEntropyLoss()
with tqdm(total=n_epochs,
postfix='loss:{0:<6.5f}',
leave=False,
dynamic_ncols=True) as pbar:
print_every = 10
for epoch in range(1, n_epochs + 1):
pbar.set_description_str(
desc="Epoch {}/{}".format(epoch, n_epochs))
avg_loss = 0
step = 0
for batch_x, batch_y in train_batch:
step += 1
_move_dict_value_to_device(batch_x,
batch_y,
device=_get_model_device(model))
optimizer.zero_grad()
output = model.forward(batch_x["word_pieces"],
batch_x["word_nums"],
batch_x["seq_len"])
loss = Lossfunc(output['pred'], batch_y['target'])
loss.backward()
optimizer.step()
avg_loss += loss.item()
if step % print_every == 0:
avg_loss = float(avg_loss) / print_every
print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6}".format(
epoch, step, avg_loss)
pbar.update(print_every)
pbar.set_postfix_str(print_output)
avg_loss = 0
metric = AccuracyMetric()
val_batch = DataSetIter(batch_size=8,
dataset=val_set,
sampler=None)
for batch_x, batch_y in val_batch:
_move_dict_value_to_device(batch_x,
batch_y,
device=_get_model_device(model))
output = model.predict(batch_x["word_pieces"],
batch_x["word_nums"],
batch_x["seq_len"])
metric(output, batch_y)
eval_result = metric.get_metric()
print("ACC on Validate Set:", eval_result)
from fastNLP.io import ModelSaver
saver = ModelSaver("../models/bert_model_max_triple.pkl")
saver.save_pytorch(model, param_only=False)
pbar.close()
metric = AccuracyMetric()
test_batch = DataSetIter(batch_size=8, dataset=test_set, sampler=None)
for batch_x, batch_y in test_batch:
_move_dict_value_to_device(batch_x,
batch_y,
device=_get_model_device(model))
output = model.predict(batch_x["word_pieces"], batch_x["word_nums"],
batch_x["seq_len"])
metric(output, batch_y)
eval_result = metric.get_metric()
print("ACC on Test Set:", eval_result)
from fastNLP.io import ModelSaver
saver = ModelSaver("../models/bert_model_max_triple.pkl")
saver.save_pytorch(model, param_only=False)
