def build_data(args):
"""
build train and dev data
"""
task_name = args.task_name.lower()
processor = reader.MatchProcessor(data_dir=args.data_dir,
task_name=task_name,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case)
train_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='train',
epoch=args.epoch,
shuffle=True,
device=args.gpu)
num_train_examples = processor.get_num_examples(phase='train')
dev_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='dev',
epoch=1,
shuffle=False,
device=args.gpu)
num_dev_examples = processor.get_num_examples(phase='dev')
max_train_steps = args.epoch * num_train_examples // args.batch_size
warmup_steps = int(max_train_steps * args.warmup_proportion)
train_data = [train_data_generator, num_train_examples]
dev_data = [dev_data_generator, num_dev_examples]
return processor, [train_data, dev_data], warmup_steps
def main(args):
task_name = args.task_name.lower()
processor = reader.MatchProcessor(data_dir=args.data_dir,
task_name=task_name,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case)
num_labels = len(processor.get_labels())
infer_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='test',
epoch=1,
shuffle=False)
num_test_examples = processor.get_num_examples(phase='test')
main_program = fluid.default_main_program()
feed_order, loss, probs, accuracy, num_seqs = create_model(
args, num_labels=num_labels, is_prediction=True)
if args.use_cuda:
place = fluid.CUDAPlace(0)
dev_count = fluid.core.get_cuda_device_count()
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
exe = fluid.Executor(place)
exe.run(framework.default_startup_program())
if args.init_checkpoint:
init_pretraining_params(exe, args.init_checkpoint, main_program)
feed_list = [
main_program.global_block().var(var_name) for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
out_scores = open(args.output, 'w')
for batch_id, data in enumerate(infer_data_generator()):
results = exe.run(fetch_list=[probs],
feed=feeder.feed(data),
return_numpy=True)
for elem in results[0]:
out_scores.write(str(elem[1]) + '\n')
out_scores.close()
if args.save_inference_model_path:
model_path = args.save_inference_model_path
fluid.io.save_inference_model(model_path,
feed_order,
probs,
exe,
main_program=main_program)
def build_data(args):
"""
build test data
"""
task_name = args.task_name.lower()
processor = reader.MatchProcessor(data_dir=args.data_dir,
task_name=task_name,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case)
test_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='test',
epoch=1,
shuffle=False,
device=args.gpu)
num_test_examples = processor.get_num_examples(phase='test')
test_data = [test_data_generator, num_test_examples]
return processor, test_data
def main(args):
"""main"""
task_name = args.task_name.lower()
processor = reader.MatchProcessor(data_dir=args.data_dir,
task_name=task_name,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case)
args.voc_size = len(open(args.vocab_path, 'r').readlines())
num_labels = len(processor.get_labels())
train_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='train',
epoch=args.epoch,
shuffle=True)
num_train_examples = processor.get_num_examples(phase='train')
dev_data_generator = processor.data_generator(batch_size=args.batch_size,
phase='dev',
epoch=1,
shuffle=False)
num_dev_examples = processor.get_num_examples(phase='dev')
if args.use_cuda:
place = fluid.CUDAPlace(0)
dev_count = fluid.core.get_cuda_device_count()
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
max_train_steps = args.epoch * num_train_examples // args.batch_size
warmup_steps = int(max_train_steps * args.warmup_proportion)
train_program = fluid.Program()
train_startup = fluid.Program()
with fluid.program_guard(train_program, train_startup):
with fluid.unique_name.guard():
feed_order, loss, predict, accuracy, num_seqs = \
create_model(args, num_labels, \
is_prediction=False)
lr_decay = fluid.layers.learning_rate_scheduler.noam_decay(
256, warmup_steps)
with fluid.default_main_program()._lr_schedule_guard():
learning_rate = lr_decay * args.learning_rate
optimizer = fluid.optimizer.Adam(learning_rate=learning_rate)
optimizer.minimize(loss)
test_program = fluid.Program()
test_startup = fluid.Program()
with fluid.program_guard(test_program, test_startup):
with fluid.unique_name.guard():
feed_order, loss, predict, accuracy, num_seqs = \
create_model(args, num_labels, \
is_prediction=True)
test_program = test_program.clone(for_test=True)
exe = Executor(place)
exe.run(train_startup)
exe.run(test_startup)
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = dev_count
train_exe = fluid.ParallelExecutor(use_cuda=args.use_cuda,
loss_name=loss.name,
exec_strategy=exec_strategy,
main_program=train_program)
test_exe = fluid.ParallelExecutor(use_cuda=args.use_cuda,
main_program=test_program,
share_vars_from=train_exe)
feed_list = [
train_program.global_block().var(var_name) for var_name in feed_order
]
feeder = fluid.DataFeeder(feed_list, place)
time_begin = time.time()
total_cost, total_acc, total_num_seqs = [], [], []
for batch_id, data in enumerate(train_data_generator()):
fetch_outs = train_exe.run(
feed=feeder.feed(data),
fetch_list=[loss.name, accuracy.name, num_seqs.name])
avg_loss = fetch_outs[0]
avg_acc = fetch_outs[1]
cur_num_seqs = fetch_outs[2]
total_cost.extend(avg_loss * cur_num_seqs)
total_acc.extend(avg_acc * cur_num_seqs)
total_num_seqs.extend(cur_num_seqs)
if batch_id % args.skip_steps == 0:
time_end = time.time()
used_time = time_end - time_begin
current_example, current_epoch = processor.get_train_progress()
print("epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
"ave acc: %f, speed: %f steps/s" %
(current_epoch, current_example, num_train_examples,
batch_id, np.sum(total_cost) / np.sum(total_num_seqs),
np.sum(total_acc) / np.sum(total_num_seqs),
args.skip_steps / used_time))
time_begin = time.time()
total_cost, total_acc, total_num_seqs = [], [], []
if batch_id % args.validation_steps == 0:
total_dev_cost, total_dev_acc, total_dev_num_seqs = [], [], []
for dev_id, dev_data in enumerate(dev_data_generator()):
fetch_outs = test_exe.run(
feed=feeder.feed(dev_data),
fetch_list=[loss.name, accuracy.name, num_seqs.name])
avg_dev_loss = fetch_outs[0]
avg_dev_acc = fetch_outs[1]
cur_dev_num_seqs = fetch_outs[2]
total_dev_cost.extend(avg_dev_loss * cur_dev_num_seqs)
total_dev_acc.extend(avg_dev_acc * cur_dev_num_seqs)
total_dev_num_seqs.extend(cur_dev_num_seqs)
print("valid eval: ave loss: %f, ave acc: %f" %
(np.sum(total_dev_cost) / np.sum(total_dev_num_seqs),
np.sum(total_dev_acc) / np.sum(total_dev_num_seqs)))
total_dev_cost, total_dev_acc, total_dev_num_seqs = [], [], []
if batch_id % args.save_steps == 0:
model_path = os.path.join(args.checkpoints, str(batch_id))
if not os.path.isdir(model_path):
os.makedirs(model_path)
fluid.io.save_persistables(executor=exe,
dirname=model_path,
main_program=train_program)