def infer(model_path, batch_size, test_data_file):
word = fluid.layers.data(name='word', shape=[1], dtype='int64', lod_level=1)
mention = fluid.layers.data(
name='mention', shape=[1], dtype='int64', lod_level=1)
target = fluid.layers.data(
name='target', shape=[1], dtype='int64', lod_level=1)
test_data = paddle.batch(
reader.file_reader(test_data_file), batch_size=batch_size)
place = fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, mention], place=place)
exe = fluid.Executor(place)
inference_scope = fluid.core.Scope()
with fluid.scope_guard(inference_scope):
[inference_program, feed_target_names,
fetch_targets] = fluid.io.load_inference_model(model_path, exe)
print_res = ""
for data in test_data():
crf_decode = exe.run(inference_program,
feed=feeder.feed(data),
fetch_list=fetch_targets,
return_numpy=False)
lod_info = (crf_decode[0].lod())[0]
np_data = np.array(crf_decode[0])
assert len(data) == len(lod_info) - 1
for sen_index in xrange(len(data)):
assert len(data[sen_index][0]) == lod_info[
sen_index + 1] - lod_info[sen_index]
word_index = 0
for tag_index in xrange(lod_info[sen_index],
lod_info[sen_index + 1]):
word = str(data[sen_index][0][word_index])
print_res += str(np_data[tag_index][0]) +" "
word_index += 1
#print_res +="\n"
print print_res
def main(args):
if not os.path.exists(args.model_save_dir):
os.makedirs(args.model_save_dir)
main = fluid.Program()
startup = fluid.Program()
with fluid.program_guard(main, startup):
avg_cost, feature_out, word, mention, target = ner_net(
args.word_dict_len, args.label_dict_len)
crf_decode = fluid.layers.crf_decoding(
input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))
(precision, recall, f1_score, num_infer_chunks, num_label_chunks,
num_correct_chunks) = fluid.layers.chunk_eval(
input=crf_decode,
label=target,
chunk_scheme="IOB",
num_chunk_types=int(math.ceil((args.label_dict_len - 1) / 2.0)))
inference_program = fluid.default_main_program().clone(for_test=True)
sgd_optimizer = fluid.optimizer.SGD(learning_rate=1e-3)
sgd_optimizer.minimize(avg_cost)
chunk_evaluator = fluid.metrics.ChunkEvaluator()
train_reader = paddle.batch(paddle.reader.shuffle(reader.file_reader(
args.train_data_dir),
buf_size=2000000),
batch_size=args.batch_size)
test_reader = paddle.batch(paddle.reader.shuffle(reader.file_reader(
args.test_data_dir),
buf_size=2000000),
batch_size=args.batch_size)
place = fluid.CUDAPlace(
0) if args.device == 'GPU' else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, mention, target],
place=place)
exe = fluid.Executor(place)
exe.run(startup)
if args.parallel:
train_exe = fluid.ParallelExecutor(loss_name=avg_cost.name,
use_cuda=(args.device == 'GPU'))
test_exe = fluid.ParallelExecutor(use_cuda=(args.device == 'GPU'),
main_program=inference_program,
share_vars_from=train_exe)
else:
train_exe = exe
test_exe = exe
batch_id = 0
for pass_id in range(args.num_passes):
chunk_evaluator.reset()
train_reader_iter = train_reader()
start_time = time.time()
while True:
try:
cur_batch = next(train_reader_iter)
cost, nums_infer, nums_label, nums_correct = train_exe.run(
fetch_list=[
avg_cost.name, num_infer_chunks.name,
num_label_chunks.name, num_correct_chunks.name
],
feed=feeder.feed(cur_batch))
chunk_evaluator.update(
np.array(nums_infer).sum().astype("int64"),
np.array(nums_label).sum().astype("int64"),
np.array(nums_correct).sum().astype("int64"))
cost_list = np.array(cost)
batch_id += 1
except StopIteration:
break
end_time = time.time()
print("pass_id:" + str(pass_id) + ", time_cost:" +
str(end_time - start_time) + "s")
precision, recall, f1_score = chunk_evaluator.eval()
print("[Train] precision:" + str(precision) + ", recall:" +
str(recall) + ", f1:" + str(f1_score))
p, r, f1 = test2(
exe, chunk_evaluator, inference_program, test_reader, place,
[num_infer_chunks, num_label_chunks, num_correct_chunks])
print("[Test] precision:" + str(p) + ", recall:" + str(r) +
", f1:" + str(f1))
save_dirname = os.path.join(args.model_save_dir,
"params_pass_%d" % pass_id)
fluid.io.save_inference_model(save_dirname, ['word', 'mention'],
[crf_decode], exe)
def infer(args):
word = fluid.layers.data(name='word',
shape=[1],
dtype='int64',
lod_level=1)
mention = fluid.layers.data(name='mention',
shape=[1],
dtype='int64',
lod_level=1)
target = fluid.layers.data(name='target',
shape=[1],
dtype='int64',
lod_level=1)
label_reverse_dict = load_reverse_dict(args.test_label_file)
test_data = paddle.batch(reader.file_reader(args.test_data_dir),
batch_size=args.batch_size)
place = fluid.CUDAPlace(0) if args.device == 'GPU' else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, mention, target], place=place)
exe = fluid.Executor(place)
inference_scope = fluid.Scope()
with fluid.scope_guard(inference_scope):
[inference_program, feed_target_names,
fetch_targets] = fluid.io.load_inference_model(args.model_path, exe)
total_passes = args.num_passes + args.skip_pass_num
batch_times = [0] * total_passes
word_counts = [0] * total_passes
wpses = [0] * total_passes
all_iters = 0
for pass_id in range(total_passes):
if pass_id < args.skip_pass_num:
print("Warm-up pass")
if pass_id == args.skip_pass_num:
profiler.reset_profiler()
iters = 0
for data in test_data():
word = to_lodtensor(list(map(lambda x: x[0], data)), place)
mention = to_lodtensor(list(map(lambda x: x[1], data)), place)
start = time.time()
crf_decode = exe.run(inference_program,
feed={
"word": word,
"mention": mention
},
fetch_list=fetch_targets,
return_numpy=False)
batch_time = time.time() - start
lod_info = (crf_decode[0].lod())[0]
np_data = np.array(crf_decode[0])
word_count = 0
assert len(data) == len(lod_info) - 1
for sen_index in range(len(data)):
assert len(
data[sen_index]
[0]) == lod_info[sen_index + 1] - lod_info[sen_index]
word_index = 0
for tag_index in range(lod_info[sen_index],
lod_info[sen_index + 1]):
word = str(data[sen_index][0][word_index])
gold_tag = label_reverse_dict[data[sen_index][2]
[word_index]]
tag = label_reverse_dict[np_data[tag_index][0]]
word_index += 1
word_count += word_index
batch_times[pass_id] += batch_time
word_counts[pass_id] += word_count
iters += 1
all_iters += 1
batch_times[pass_id] /= iters
word_counts[pass_id] /= iters
wps = word_counts[pass_id] / batch_times[pass_id]
wpses[pass_id] = wps
print(
"Pass: %d, iterations (total): %d (%d), latency: %.5f s, words: %d, wps: %f"
% (pass_id, iters, all_iters, batch_times[pass_id],
word_counts[pass_id], wps))
# Postprocess benchmark data
latencies = batch_times[args.skip_pass_num:]
latency_avg = np.average(latencies)
latency_std = np.std(latencies)
latency_pc99 = np.percentile(latencies, 99)
wps_avg = np.average(wpses)
wps_std = np.std(wpses)
wps_pc01 = np.percentile(wpses, 1)
# Benchmark output
print('\nTotal passes (incl. warm-up): %d' % (total_passes))
print('Total iterations (incl. warm-up): %d' % (all_iters))
print('Total examples (incl. warm-up): %d' % (all_iters * args.batch_size))
print('avg latency: %.5f, std latency: %.5f, 99pc latency: %.5f' %
(latency_avg, latency_std, latency_pc99))
print('avg wps: %.5f, std wps: %.5f, wps for 99pc latency: %.5f' %
(wps_avg, wps_std, wps_pc01))
def train(args):
"""
Train the network.
"""
if not os.path.exists(args.model_save_dir):
os.mkdir(args.model_save_dir)
word2id_dict = reader.load_reverse_dict(args.word_dict_path)
label2id_dict = reader.load_reverse_dict(args.label_dict_path)
word_rep_dict = reader.load_dict(args.word_rep_dict_path)
word_dict_len = max(map(int, word2id_dict.values())) + 1
label_dict_len = max(map(int, label2id_dict.values())) + 1
avg_cost, crf_decode, word, target = lex_net(args, word_dict_len,
label_dict_len)
sgd_optimizer = fluid.optimizer.SGD(learning_rate=args.base_learning_rate)
sgd_optimizer.minimize(avg_cost)
(precision, recall, f1_score, num_infer_chunks, num_label_chunks,
num_correct_chunks) = fluid.layers.chunk_eval(
input=crf_decode,
label=target,
chunk_scheme="IOB",
num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0)))
chunk_evaluator = fluid.metrics.ChunkEvaluator()
chunk_evaluator.reset()
train_reader_list = []
corpus_num = len(args.corpus_type_list)
for i in xrange(corpus_num):
train_reader = paddle.batch(
paddle.reader.shuffle(reader.file_reader(args.traindata_dir,
word2id_dict,
label2id_dict,
word_rep_dict,
args.corpus_type_list[i]),
buf_size=args.traindata_shuffle_buffer),
batch_size=int(args.batch_size * args.corpus_proportion_list[i]))
train_reader_list.append(train_reader)
test_reader = paddle.batch(reader.file_reader(args.testdata_dir,
word2id_dict, label2id_dict,
word_rep_dict),
batch_size=args.batch_size)
train_reader_itr_list = []
for train_reader in train_reader_list:
cur_reader_itr = train_reader()
train_reader_itr_list.append(cur_reader_itr)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, target], place=place)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
batch_id = 0
start_time = time.time()
eval_list = []
iter = 0
while True:
full_batch = []
cur_batch = []
for i in xrange(corpus_num):
reader_itr = train_reader_itr_list[i]
try:
cur_batch = next(reader_itr)
except StopIteration:
print(args.corpus_type_list[i] +
" corpus finish a pass of training")
new_reader = train_reader_list[i]
train_reader_itr_list[i] = new_reader()
cur_batch = next(train_reader_itr_list[i])
full_batch += cur_batch
random.shuffle(full_batch)
cost_var, nums_infer, nums_label, nums_correct = exe.run(
fluid.default_main_program(),
fetch_list=[
avg_cost, num_infer_chunks, num_label_chunks,
num_correct_chunks
],
feed=feeder.feed(full_batch))
print("batch_id:" + str(batch_id) + ", avg_cost:" + str(cost_var[0]))
chunk_evaluator.update(nums_infer, nums_label, nums_correct)
batch_id += 1
if (batch_id % args.save_model_per_batchs == 1):
save_exe = fluid.Executor(place)
save_dirname = os.path.join(args.model_save_dir,
"params_batch_%d" % batch_id)
fluid.io.save_inference_model(save_dirname, ['word'], [crf_decode],
save_exe)
temp_save_model = os.path.join(args.model_save_dir,
"temp_model_for_test")
fluid.io.save_inference_model(
temp_save_model, ['word', 'target'],
[num_infer_chunks, num_label_chunks, num_correct_chunks],
save_exe)
precision, recall, f1_score = chunk_evaluator.eval()
print("[train] batch_id:" + str(batch_id) + ", precision:" +
str(precision) + ", recall:" + str(recall) + ", f1:" +
str(f1_score))
chunk_evaluator.reset()
p, r, f1 = test(exe, chunk_evaluator, temp_save_model, test_reader,
place)
chunk_evaluator.reset()
print("[test] batch_id:" + str(batch_id) + ", precision:" +
str(p) + ", recall:" + str(r) + ", f1:" + str(f1))
end_time = time.time()
print("cur_batch_id:" + str(batch_id) + ", last " +
str(args.save_model_per_batchs) + " batchs, time_cost:" +
str(end_time - start_time))
start_time = time.time()
if len(eval_list) < 2 * args.eval_window:
eval_list.append(f1)
else:
eval_list.pop(0)
eval_list.append(f1)
last_avg_f1 = sum(
eval_list[0:args.eval_window]) / args.eval_window
cur_avg_f1 = sum(
eval_list[args.eval_window:2 *
args.eval_window]) / args.eval_window
if cur_avg_f1 <= last_avg_f1:
return
else:
print "keep training!"
iter += 1
if (iter == args.num_iterations):
return
return results
if __name__ == '__main__':
args = parse_args()
word2id_dict = reader.load_reverse_dict(args.word_dict_path)
label2id_dict = reader.load_reverse_dict(args.label_dict_path)
word_rep_dict = reader.load_dict(args.word_rep_dict_path)
word_dict_len = max(map(int, word2id_dict.values())) + 1
label_dict_len = max(map(int, label2id_dict.values())) + 1
pred = create_predictor(args)
test_data = paddle.batch(reader.file_reader(args.testdata_dir,
word2id_dict, label2id_dict,
word_rep_dict),
batch_size=1)
batch_id = 0
id2word = {v: k for k, v in word2id_dict.items()}
id2label = {v: k for k, v in label2id_dict.items()}
for data in test_data():
batch_id += 1
word_data, word_lod = to_lodtensor(list(map(lambda x: x[0], data)))
target_data, target_lod = to_lodtensor(list(map(lambda x: x[1], data)))
result_list = run(pred, [word_data, target_data],
[word_lod, target_lod])
number_infer = np.array(result_list[0])
number_label = np.array(result_list[1])
number_correct = np.array(result_list[2])
lac_result = ""
return results
if __name__ == '__main__':
args = parse_args()
word2id_dict = reader.load_reverse_dict(args.word_dict_path)
label2id_dict = reader.load_reverse_dict(args.label_dict_path)
word_rep_dict = reader.load_dict(args.word_rep_dict_path)
word_dict_len = max(map(int, word2id_dict.values())) + 1
label_dict_len = max(map(int, label2id_dict.values())) + 1
pred = create_predictor(args)
test_data = paddle.batch(
reader.file_reader(args.testdata_dir, word2id_dict, label2id_dict,
word_rep_dict),
batch_size=1)
batch_id = 0
id2word = {v: k for k, v in word2id_dict.items()}
id2label = {v: k for k, v in label2id_dict.items()}
for data in test_data():
batch_id += 1
word_data, word_lod = to_lodtensor(list(map(lambda x: x[0], data)))
target_data, target_lod = to_lodtensor(list(map(lambda x: x[1], data)))
result_list = run(pred, [word_data, target_data],
[word_lod, target_lod])
number_infer = np.array(result_list[0])
number_label = np.array(result_list[1])
number_correct = np.array(result_list[2])
lac_result = ""
for i in range(len(data[0][0])):