def do_save_inference_model(args):
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
if args.use_cuda:
dev_count = fluid.core.get_cuda_device_count()
place = fluid.CUDAPlace(0)
else:
dev_count = int(os.environ.get('CPU_NUM', 1))
place = fluid.CPUPlace()
exe = fluid.Executor(place)
test_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
infer_pyreader, ernie_inputs, labels = ernie_pyreader(
args, pyreader_name="infer_reader")
if args.use_paddle_hub:
embeddings = ernie_encoder_with_paddle_hub(
ernie_inputs, args.max_seq_len)
else:
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
probs = create_model(args,
embeddings,
labels=labels,
is_prediction=True)
test_prog = test_prog.clone(for_test=True)
exe.run(startup_prog)
assert (args.init_checkpoint)
if args.init_checkpoint:
utils.init_checkpoint(exe, args.init_checkpoint, test_prog)
fluid.io.save_inference_model(args.inference_model_dir,
feeded_var_names=[
ernie_inputs["src_ids"].name,
ernie_inputs["sent_ids"].name,
ernie_inputs["pos_ids"].name,
ernie_inputs["input_mask"].name,
ernie_inputs["seq_lens"].name
],
target_vars=[probs],
executor=exe,
main_program=test_prog,
model_filename="model.pdmodel",
params_filename="params.pdparams")
print("save inference model at %s" % (args.inference_model_dir))
def test_inference_model(args):
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
if args.use_cuda:
dev_count = fluid.core.get_cuda_device_count()
place = fluid.CUDAPlace(0)
else:
dev_count = int(os.environ.get('CPU_NUM', 1))
place = fluid.CPUPlace()
exe = fluid.Executor(place)
reader = task_reader.ClassifyReader(vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case,
random_seed=args.random_seed)
test_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
infer_pyreader, ernie_inputs, labels = ernie_pyreader(
args, pyreader_name="infer_pyreader")
embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config)
probs = create_model(args,
embeddings,
labels=labels,
is_prediction=True)
test_prog = test_prog.clone(for_test=True)
exe.run(startup_prog)
assert (args.inference_model_dir)
infer_data_generator = reader.data_generator(input_file=args.test_set,
batch_size=args.batch_size /
dev_count,
phase="infer",
epoch=1,
shuffle=False)
infer_program, feed_names, fetch_targets = fluid.io.load_inference_model(
dirname=args.inference_model_dir,
executor=exe,
model_filename="model.pdmodel",
params_filename="params.pdparams")
infer_pyreader.set_batch_generator(infer_data_generator)
inference(exe, test_prog, infer_pyreader, [probs.name], "infer")
def main(args):
"""
Main Function
"""
args = parser.parse_args()
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
if args.use_cuda:
place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
dev_count = fluid.core.get_cuda_device_count()
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
exe = fluid.Executor(place)
reader = task_reader.SequenceLabelReader(
vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case,
in_tokens=False,
random_seed=args.random_seed)
if not (args.do_train or args.do_test or args.do_infer):
raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
"least one of them must be True.")
startup_prog = fluid.Program()
if args.random_seed is not None:
startup_prog.random_seed = args.random_seed
if args.do_train:
num_train_examples = reader.get_num_examples(args.train_set)
max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
print("Device count: %d" % dev_count)
print("Num train examples: %d" % num_train_examples)
print("Max train steps: %d" % max_train_steps)
train_program = fluid.Program()
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
# create ernie_pyreader
train_pyreader, ernie_inputs, words, labels = ernie_pyreader(
args, pyreader_name='train_reader')
train_pyreader.decorate_tensor_provider(
reader.data_generator(args.train_set,
args.batch_size,
args.epoch,
shuffle=True,
phase="train"))
# get ernie_embeddings
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
# user defined model based on ernie embeddings
train_ret = create_model(args,
embeddings,
labels=labels,
is_prediction=False)
optimizer = fluid.optimizer.Adam(learning_rate=args.lr)
fluid.clip.set_gradient_clip(
clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0))
optimizer.minimize(train_ret["loss"])
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program, batch_size=args.batch_size)
print("Theoretical memory usage in training: %.3f - %.3f %s" %
(lower_mem, upper_mem, unit))
if args.do_test:
test_program = fluid.Program()
with fluid.program_guard(test_program, startup_prog):
with fluid.unique_name.guard():
# create ernie_pyreader
test_pyreader, ernie_inputs, words, labels = ernie_pyreader(
args, pyreader_name='test_reader')
test_pyreader.decorate_tensor_provider(
reader.data_generator(args.test_set,
args.batch_size,
phase='test',
epoch=1,
shuffle=False))
# get ernie_embeddings
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
# user defined model based on ernie embeddings
test_ret = create_model(args,
embeddings,
labels=labels,
is_prediction=False)
test_program = test_program.clone(for_test=True)
if args.do_infer:
infer_program = fluid.Program()
with fluid.program_guard(infer_program, startup_prog):
with fluid.unique_name.guard():
# create ernie_pyreader
infer_pyreader, ernie_inputs, words, labels = ernie_pyreader(
args, pyreader_name='infer_reader')
infer_pyreader.decorate_tensor_provider(
reader.data_generator(args.infer_set,
args.batch_size,
phase='infer',
epoch=1,
shuffle=False))
# get ernie_embeddings
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
# user defined model based on ernie embeddings
infer_ret = create_model(args,
embeddings,
labels=labels,
is_prediction=True)
infer_ret["words"] = words
infer_program = infer_program.clone(for_test=True)
exe.run(startup_prog)
# load checkpoints
if args.do_train:
if args.init_checkpoint and args.init_pretraining_params:
print(
"WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
"both are set! Only arg 'init_checkpoint' is made valid.")
if args.init_checkpoint:
utils.init_checkpoint(exe, args.init_checkpoint, startup_prog)
elif args.init_pretraining_params:
utils.init_pretraining_params(exe, args.init_pretraining_params,
startup_prog)
elif args.do_test or args.do_infer:
if not args.init_checkpoint:
raise ValueError(
"args 'init_checkpoint' should be set if only doing test or infer!"
)
utils.init_checkpoint(exe, args.init_checkpoint, startup_prog)
if args.do_train:
train_pyreader.start()
steps = 0
total_cost, total_acc, total_num_seqs = [], [], []
while True:
try:
steps += 1
if steps % args.skip_steps == 0:
fetch_list = [
train_ret["loss"],
train_ret["num_infer_chunks"],
train_ret["num_label_chunks"],
train_ret["num_correct_chunks"],
]
else:
fetch_list = []
start_time = time.time()
outputs = exe.run(program=train_program, fetch_list=fetch_list)
end_time = time.time()
if steps % args.skip_steps == 0:
loss, nums_infer, nums_label, nums_correct = outputs
train_ret["chunk_evaluator"].reset()
train_ret["chunk_evaluator"].update(
nums_infer, nums_label, nums_correct)
precision, recall, f1_score = train_ret[
"chunk_evaluator"].eval()
print(
"[train] batch_id = %d, loss = %.5f, P: %.5f, R: %.5f, F1: %.5f, elapsed time %.5f, "
"pyreader queue_size: %d " %
(steps, loss, precision, recall, f1_score,
end_time - start_time, train_pyreader.queue.size()))
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
print("\tsaving model as %s" % (save_path))
fluid.io.save_persistables(exe, save_path, train_program)
if steps % args.validation_steps == 0:
# evaluate test set
if args.do_test:
evaluate(exe, test_program, test_pyreader, test_ret)
except fluid.core.EOFException:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(exe, save_path, train_program)
train_pyreader.reset()
break
# final eval on test set
if args.do_test:
evaluate(exe, test_program, test_pyreader, test_ret)
if args.do_infer:
# create dict
id2word_dict = dict([(str(word_id), word)
for word, word_id in reader.vocab.items()])
id2label_dict = dict([(str(label_id), label)
for label, label_id in reader.label_map.items()])
Dataset = namedtuple("Dataset", ["id2word_dict", "id2label_dict"])
dataset = Dataset(id2word_dict, id2label_dict)
infer_pyreader.start()
while True:
try:
(words, crf_decode) = exe.run(
infer_program,
fetch_list=[infer_ret["words"], infer_ret["crf_decode"]],
return_numpy=False)
# User should notice that words had been clipped if long than args.max_seq_len
results = utils.parse_result(words, crf_decode, dataset)
for result in results:
print(result)
except fluid.core.EOFException:
infer_pyreader.reset()
break
def main(args):
"""
Main Function
"""
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
if args.use_cuda:
place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
dev_count = fluid.core.get_cuda_device_count()
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
exe = fluid.Executor(place)
reader = task_reader.ClassifyReader(vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case,
random_seed=args.random_seed)
if not (args.do_train or args.do_val or args.do_infer):
raise ValueError("For args `do_train`, `do_val` and `do_infer`, at "
"least one of them must be True.")
startup_prog = fluid.Program()
if args.random_seed is not None:
startup_prog.random_seed = args.random_seed
if args.do_train:
train_data_generator = reader.data_generator(
input_file=args.train_set,
batch_size=args.batch_size,
epoch=args.epoch,
shuffle=True,
phase="train")
num_train_examples = reader.get_num_examples(args.train_set)
max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
print("Device count: %d" % dev_count)
print("Num train examples: %d" % num_train_examples)
print("Max train steps: %d" % max_train_steps)
train_program = fluid.Program()
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
# create ernie_pyreader
train_pyreader, ernie_inputs, labels = ernie_pyreader(
args, pyreader_name='train_pyreader')
# get ernie_embeddings
if args.use_paddle_hub:
embeddings = ernie_encoder_with_paddle_hub(
ernie_inputs, args.max_seq_len)
else:
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
# user defined model based on ernie embeddings
loss, accuracy, num_seqs = create_model(args,
embeddings,
labels=labels,
is_prediction=False)
optimizer = fluid.optimizer.Adam(learning_rate=args.lr)
optimizer.minimize(loss)
if args.verbose:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program, batch_size=args.batch_size)
print("Theoretical memory usage in training: %.3f - %.3f %s" %
(lower_mem, upper_mem, unit))
if args.do_val:
test_data_generator = reader.data_generator(input_file=args.dev_set,
batch_size=args.batch_size,
phase='dev',
epoch=1,
shuffle=False)
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
# create ernie_pyreader
test_pyreader, ernie_inputs, labels = ernie_pyreader(
args, pyreader_name='eval_reader')
# get ernie_embeddings
if args.use_paddle_hub:
embeddings = ernie_encoder_with_paddle_hub(
ernie_inputs, args.max_seq_len)
else:
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
# user defined model based on ernie embeddings
loss, accuracy, num_seqs = create_model(args,
embeddings,
labels=labels,
is_prediction=False)
test_prog = test_prog.clone(for_test=True)
if args.do_infer:
infer_data_generator = reader.data_generator(
input_file=args.test_set,
batch_size=args.batch_size,
phase='infer',
epoch=1,
shuffle=False)
infer_prog = fluid.Program()
with fluid.program_guard(infer_prog, startup_prog):
with fluid.unique_name.guard():
infer_pyreader, ernie_inputs, labels = ernie_pyreader(
args, pyreader_name="infer_pyreader")
# get ernie_embeddings
if args.use_paddle_hub:
embeddings = ernie_encoder_with_paddle_hub(
ernie_inputs, args.max_seq_len)
else:
embeddings = ernie_encoder(ernie_inputs,
ernie_config=ernie_config)
probs = create_model(args,
embeddings,
labels=labels,
is_prediction=True)
infer_prog = infer_prog.clone(for_test=True)
exe.run(startup_prog)
if args.do_train:
if args.init_checkpoint:
init_checkpoint(exe,
args.init_checkpoint,
main_program=train_program)
elif args.do_val:
if not args.init_checkpoint:
raise ValueError("args 'init_checkpoint' should be set if"
"only doing validation or testing!")
init_checkpoint(exe, args.init_checkpoint, main_program=test_prog)
elif args.do_infer:
if not args.init_checkpoint:
raise ValueError("args 'init_checkpoint' should be set if"
"only doing validation or testing!")
init_checkpoint(exe, args.init_checkpoint, main_program=infer_prog)
if args.do_train:
train_exe = exe
train_pyreader.set_batch_generator(train_data_generator)
else:
train_exe = None
if args.do_val:
test_exe = exe
test_pyreader.set_batch_generator(test_data_generator)
if args.do_infer:
test_exe = exe
infer_pyreader.set_batch_generator(infer_data_generator)
if args.do_train:
train_pyreader.start()
steps = 0
total_cost, total_acc, total_num_seqs = [], [], []
time_begin = time.time()
while True:
try:
steps += 1
if steps % args.skip_steps == 0:
fetch_list = [loss.name, accuracy.name, num_seqs.name]
else:
fetch_list = []
outputs = train_exe.run(program=train_program,
fetch_list=fetch_list,
return_numpy=False)
if steps % args.skip_steps == 0:
np_loss, np_acc, np_num_seqs = outputs
np_loss = np.array(np_loss)
np_acc = np.array(np_acc)
np_num_seqs = np.array(np_num_seqs)
total_cost.extend(np_loss * np_num_seqs)
total_acc.extend(np_acc * np_num_seqs)
total_num_seqs.extend(np_num_seqs)
if args.verbose:
verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
)
print(verbose)
time_end = time.time()
used_time = time_end - time_begin
print("step: %d, ave loss: %f, "
"ave acc: %f, speed: %f steps/s" %
(steps, np.sum(total_cost) / np.sum(total_num_seqs),
np.sum(total_acc) / np.sum(total_num_seqs),
args.skip_steps / used_time))
total_cost, total_acc, total_num_seqs = [], [], []
time_begin = time.time()
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps),
"checkpoint")
fluid.save(train_program, save_path)
if steps % args.validation_steps == 0:
# evaluate dev set
if args.do_val:
evaluate(exe, test_prog, test_pyreader,
[loss.name, accuracy.name, num_seqs.name],
"dev")
except fluid.core.EOFException:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps), "checkpoint")
fluid.save(train_program, save_path)
train_pyreader.reset()
break
# final eval on dev set
if args.do_val:
print("Final validation result:")
evaluate(exe, test_prog, test_pyreader,
[loss.name, accuracy.name, num_seqs.name], "dev")
# final eval on test set
if args.do_infer:
print("Final test result:")
infer(exe, infer_prog, infer_pyreader, [probs.name], "infer")
def create_ernie_model(args, ernie_config):
"""
Create Model for LAC based on ERNIE encoder
"""
# ERNIE's input data
src_ids = fluid.data(name='src_ids',
shape=[-1, args.max_seq_len, 1],
dtype='int64')
sent_ids = fluid.data(name='sent_ids',
shape=[-1, args.max_seq_len, 1],
dtype='int64')
pos_ids = fluid.data(name='pos_ids',
shape=[-1, args.max_seq_len, 1],
dtype='int64')
input_mask = fluid.data(name='input_mask',
shape=[-1, args.max_seq_len, 1],
dtype='float32')
padded_labels = fluid.data(name='padded_labels',
shape=[-1, args.max_seq_len, 1],
dtype='int64')
seq_lens = fluid.data(name='seq_lens',
shape=[-1],
dtype='int64',
lod_level=0)
squeeze_labels = fluid.layers.squeeze(padded_labels, axes=[-1])
# ernie_pyreader
ernie_inputs = {
"src_ids": src_ids,
"sent_ids": sent_ids,
"pos_ids": pos_ids,
"input_mask": input_mask,
"seq_lens": seq_lens
}
embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config)
padded_token_embeddings = embeddings["padded_token_embeddings"]
emission = fluid.layers.fc(
size=args.num_labels,
input=padded_token_embeddings,
param_attr=fluid.ParamAttr(
initializer=fluid.initializer.Uniform(low=-args.init_bound,
high=args.init_bound),
regularizer=fluid.regularizer.L2DecayRegularizer(
regularization_coeff=1e-4)),
num_flatten_dims=2)
crf_cost = fluid.layers.linear_chain_crf(
input=emission,
label=padded_labels,
param_attr=fluid.ParamAttr(name='crfw',
learning_rate=args.crf_learning_rate),
length=seq_lens)
avg_cost = fluid.layers.mean(x=crf_cost)
crf_decode = fluid.layers.crf_decoding(
input=emission,
param_attr=fluid.ParamAttr(name='crfw'),
length=seq_lens)
(precision, recall, f1_score, num_infer_chunks, num_label_chunks,
num_correct_chunks) = fluid.layers.chunk_eval(
input=crf_decode,
label=squeeze_labels,
chunk_scheme="IOB",
num_chunk_types=int(math.ceil((args.num_labels - 1) / 2.0)),
seq_length=seq_lens)
chunk_evaluator = fluid.metrics.ChunkEvaluator()
chunk_evaluator.reset()
ret = {
"feed_list":
[src_ids, sent_ids, pos_ids, input_mask, padded_labels, seq_lens],
"words":
src_ids,
"pos_ids":
pos_ids,
"sent_ids":
sent_ids,
"input_mask":
input_mask,
"labels":
padded_labels,
"seq_lens":
seq_lens,
"avg_cost":
avg_cost,
"crf_decode":
crf_decode,
"precision":
precision,
"recall":
recall,
"f1_score":
f1_score,
"chunk_evaluator":
chunk_evaluator,
"num_infer_chunks":
num_infer_chunks,
"num_label_chunks":
num_label_chunks,
"num_correct_chunks":
num_correct_chunks,
"emission":
emission,
"alpha":
None
}
return ret