def infer(args):
id2word_dict = reader.load_dict(args.word_dict_path)
word2id_dict = reader.load_reverse_dict(args.word_dict_path)
id2label_dict = reader.load_dict(args.label_dict_path)
label2id_dict = reader.load_reverse_dict(args.label_dict_path)
q2b_dict = reader.load_dict(args.word_rep_dict_path)
test_data = paddle.batch(reader.test_reader(args.test_data_dir,
word2id_dict, label2id_dict,
q2b_dict),
batch_size=args.batch_size)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
inference_scope = fluid.core.Scope()
with fluid.scope_guard(inference_scope):
[inference_program, feed_target_names,
fetch_targets] = fluid.io.load_inference_model(args.model_path, exe)
for data in test_data():
full_out_str = ""
word_idx = to_lodtensor([x[0] for x in data], place)
word_list = [x[1] for x in data]
(crf_decode, ) = exe.run(inference_program,
feed={"word": word_idx},
fetch_list=fetch_targets,
return_numpy=False)
lod_info = (crf_decode.lod())[0]
np_data = np.array(crf_decode)
assert len(data) == len(lod_info) - 1
for sen_index in xrange(len(data)):
assert len(
data[sen_index][0]) == lod_info[sen_index +
1] - lod_info[sen_index]
word_index = 0
outstr = ""
cur_full_word = ""
cur_full_tag = ""
words = word_list[sen_index]
for tag_index in xrange(lod_info[sen_index],
lod_info[sen_index + 1]):
cur_word = words[word_index]
cur_tag = id2label_dict[str(np_data[tag_index][0])]
if cur_tag.endswith("-B") or cur_tag.endswith("O"):
if len(cur_full_word) != 0:
outstr += cur_full_word.encode(
'utf8') + "/" + cur_full_tag.encode(
'utf8') + " "
cur_full_word = cur_word
cur_full_tag = get_real_tag(cur_tag)
else:
cur_full_word += cur_word
word_index += 1
outstr += cur_full_word.encode(
'utf8') + "/" + cur_full_tag.encode('utf8') + " "
outstr = outstr.strip()
full_out_str += outstr + "\n"
print full_out_str.strip()
def train(args):
"""
Train the network.
"""
if not os.path.exists(args.model_save_dir):
os.mkdir(args.model_save_dir)
word2id_dict = reader.load_reverse_dict(args.word_dict_path)
label2id_dict = reader.load_reverse_dict(args.label_dict_path)
word_rep_dict = reader.load_dict(args.word_rep_dict_path)
word_dict_len = max(map(int, word2id_dict.values())) + 1
label_dict_len = max(map(int, label2id_dict.values())) + 1
avg_cost, crf_decode, word, target = lex_net(args, word_dict_len,
label_dict_len)
sgd_optimizer = fluid.optimizer.SGD(learning_rate=args.base_learning_rate)
sgd_optimizer.minimize(avg_cost)
(precision, recall, f1_score, num_infer_chunks, num_label_chunks,
num_correct_chunks) = fluid.layers.chunk_eval(
input=crf_decode,
label=target,
chunk_scheme="IOB",
num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0)))
chunk_evaluator = fluid.metrics.ChunkEvaluator()
chunk_evaluator.reset()
train_reader_list = []
corpus_num = len(args.corpus_type_list)
for i in xrange(corpus_num):
train_reader = paddle.batch(
paddle.reader.shuffle(reader.file_reader(args.traindata_dir,
word2id_dict,
label2id_dict,
word_rep_dict,
args.corpus_type_list[i]),
buf_size=args.traindata_shuffle_buffer),
batch_size=int(args.batch_size * args.corpus_proportion_list[i]))
train_reader_list.append(train_reader)
test_reader = paddle.batch(reader.file_reader(args.testdata_dir,
word2id_dict, label2id_dict,
word_rep_dict),
batch_size=args.batch_size)
train_reader_itr_list = []
for train_reader in train_reader_list:
cur_reader_itr = train_reader()
train_reader_itr_list.append(cur_reader_itr)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, target], place=place)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
batch_id = 0
start_time = time.time()
eval_list = []
iter = 0
while True:
full_batch = []
cur_batch = []
for i in xrange(corpus_num):
reader_itr = train_reader_itr_list[i]
try:
cur_batch = next(reader_itr)
except StopIteration:
print(args.corpus_type_list[i] +
" corpus finish a pass of training")
new_reader = train_reader_list[i]
train_reader_itr_list[i] = new_reader()
cur_batch = next(train_reader_itr_list[i])
full_batch += cur_batch
random.shuffle(full_batch)
cost_var, nums_infer, nums_label, nums_correct = exe.run(
fluid.default_main_program(),
fetch_list=[
avg_cost, num_infer_chunks, num_label_chunks,
num_correct_chunks
],
feed=feeder.feed(full_batch))
print("batch_id:" + str(batch_id) + ", avg_cost:" + str(cost_var[0]))
chunk_evaluator.update(nums_infer, nums_label, nums_correct)
batch_id += 1
if (batch_id % args.save_model_per_batchs == 1):
save_exe = fluid.Executor(place)
save_dirname = os.path.join(args.model_save_dir,
"params_batch_%d" % batch_id)
fluid.io.save_inference_model(save_dirname, ['word'], [crf_decode],
save_exe)
temp_save_model = os.path.join(args.model_save_dir,
"temp_model_for_test")
fluid.io.save_inference_model(
temp_save_model, ['word', 'target'],
[num_infer_chunks, num_label_chunks, num_correct_chunks],
save_exe)
precision, recall, f1_score = chunk_evaluator.eval()
print("[train] batch_id:" + str(batch_id) + ", precision:" +
str(precision) + ", recall:" + str(recall) + ", f1:" +
str(f1_score))
chunk_evaluator.reset()
p, r, f1 = test(exe, chunk_evaluator, temp_save_model, test_reader,
place)
chunk_evaluator.reset()
print("[test] batch_id:" + str(batch_id) + ", precision:" +
str(p) + ", recall:" + str(r) + ", f1:" + str(f1))
end_time = time.time()
print("cur_batch_id:" + str(batch_id) + ", last " +
str(args.save_model_per_batchs) + " batchs, time_cost:" +
str(end_time - start_time))
start_time = time.time()
if len(eval_list) < 2 * args.eval_window:
eval_list.append(f1)
else:
eval_list.pop(0)
eval_list.append(f1)
last_avg_f1 = sum(
eval_list[0:args.eval_window]) / args.eval_window
cur_avg_f1 = sum(
eval_list[args.eval_window:2 *
args.eval_window]) / args.eval_window
if cur_avg_f1 <= last_avg_f1:
return
else:
print "keep training!"
iter += 1
if (iter == args.num_iterations):
return
predictor.zero_copy_run()
results = []
# get out data from output tensor
output_names = predictor.get_output_names()
for i, name in enumerate(output_names):
output_tensor = predictor.get_output_tensor(name)
output_data = output_tensor.copy_to_cpu()
results.append(output_data)
return results
if __name__ == '__main__':
args = parse_args()
word2id_dict = reader.load_reverse_dict(args.word_dict_path)
label2id_dict = reader.load_reverse_dict(args.label_dict_path)
word_rep_dict = reader.load_dict(args.word_rep_dict_path)
word_dict_len = max(map(int, word2id_dict.values())) + 1
label_dict_len = max(map(int, label2id_dict.values())) + 1
pred = create_predictor(args)
test_data = paddle.batch(reader.file_reader(args.testdata_dir,
word2id_dict, label2id_dict,
word_rep_dict),
batch_size=1)
batch_id = 0
id2word = {v: k for k, v in word2id_dict.items()}
id2label = {v: k for k, v in label2id_dict.items()}
for data in test_data():
