def default_exe_params(is_distributed, use_cuda, thread_num):
"""
Set the default execute parameters.
"""
gpu_id = 0
trainer_num = 1
trainer_id = 0
dist_strategy = None
places = None
if is_distributed:
if use_cuda:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
gpu_id = int(os.getenv("FLAGS_selected_gpus"))
trainer_num = fleet.worker_num()
trainer_id = fleet.worker_index()
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = 4
exec_strategy.num_iteration_per_drop_scope = 1
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.nccl_comm_num = 2
dist_strategy.fuse_all_reduce_ops = True
dist_strategy.forward_recompute = True
dist_strategy.use_amp = True
dist_strategy.amp_loss_scaling = 12800.0
places = fluid.cuda_places()
else:
print('Only gpu is supported for distributed mode at present.')
exit(-1)
else:
if use_cuda:
places = fluid.cuda_places()
else:
places = fluid.cpu_places(thread_num)
os.environ['CPU_NUM'] = str(thread_num)
if use_cuda:
exe = fluid.Executor(fluid.CUDAPlace(gpu_id))
else:
exe = fluid.Executor(fluid.CPUPlace())
return {
'exe': exe,
'trainer_num': trainer_num,
'trainer_id': trainer_id,
'gpu_id': gpu_id,
'dist_strategy': dist_strategy,
'places': places
}
def distributed_optimize(optimizer):
'''
A part of configuration for distributed training
'''
strategy = DistributedStrategy()
strategy.fuse_all_reduce_ops = True
strategy.nccl_comm_num = 2
strategy.fuse_elewise_add_act_ops=True
strategy.fuse_bn_act_ops = True
return fleet.distributed_optimizer(optimizer, strategy=strategy)
def dist_optimizer(config, optimizer):
"""
Create a distributed optimizer based on a normal optimizer
Args:
config(dict):
optimizer(): a normal optimizer
Returns:
optimizer: a distributed optimizer
"""
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 3
exec_strategy.num_iteration_per_drop_scope = 10
dist_strategy = DistributedStrategy()
dist_strategy.nccl_comm_num = 1
dist_strategy.fuse_all_reduce_ops = True
dist_strategy.exec_strategy = exec_strategy
optimizer = fleet.distributed_optimizer(optimizer, strategy=dist_strategy)
return optimizer
def net(self, args=None):
"""
BERT net struct.
Args:
fleet:
args (ArgumentParser): run args to config dist fleet.
Returns:
tuple: the return value contains avg_cost, py_reader
"""
args = p_args()
bert_config = BertConfig(DATA_DIR +
"uncased_L-24_H-1024_A-16/bert_config.json")
bert_config.print_config()
place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
exe = fluid.Executor(place)
# init program
train_program = fluid.Program()
startup_prog = fluid.Program()
if args.random_seed != 0:
print("set program random seed as: ", args.random_seed)
startup_prog.random_seed = args.random_seed
train_program.random_seed = args.random_seed
task_name = args.task_name.lower()
processors = {
'xnli': reader.XnliProcessor,
'cola': reader.ColaProcessor,
'mrpc': reader.MrpcProcessor,
'mnli': reader.MnliProcessor,
}
processor = processors[task_name](data_dir=args.data_dir,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case,
in_tokens=args.in_tokens,
random_seed=args.random_seed)
num_labels = len(processor.get_labels())
dev_count = 1
self.train_data_generator = processor.data_generator(
batch_size=args.batch_size,
phase='train',
epoch=args.epoch,
dev_count=dev_count,
dev_idx=0,
shuffle=args.shuffle,
shuffle_seed=args.shuffle_seed)
num_train_examples = processor.get_num_examples(phase='train')
max_train_steps = 5
self.warmup_steps = 0.5
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.use_experimental_executor = args.use_fast_executor
exec_strategy.num_threads = dev_count
exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
dist_strategy = DistributedStrategy()
args.run_params = json.loads(args.run_params)
dist_strategy.enable_inplace = args.run_params['enable_inplace']
dist_strategy.fuse_all_reduce_ops = args.run_params[
'fuse_all_reduce_ops']
dist_strategy.nccl_comm_num = args.run_params['nccl_comm_num']
dist_strategy.use_local_sgd = args.run_params['use_local_sgd']
dist_strategy.mode = args.run_params["mode"]
dist_strategy.collective_mode = args.run_params["collective"]
dist_strategy.exec_strategy = exec_strategy
dist_strategy.use_hierarchical_allreduce = False
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
self.train_pyreader, self.loss, probs, accuracy, num_seqs, checkpoints = create_model(
args, bert_config=bert_config, num_labels=num_labels)
scheduled_lr = optimization(loss=self.loss,
warmup_steps=self.warmup_steps,
num_train_steps=max_train_steps,
learning_rate=args.learning_rate,
train_program=train_program,
startup_prog=startup_prog,
weight_decay=args.weight_decay,
scheduler=args.lr_scheduler,
use_fp16=False,
loss_scaling=args.loss_scaling,
dist_strategy=dist_strategy)
exe.run(startup_prog)
with open("__model__", "wb") as f:
f.write(fleet._origin_program.desc.serialize_to_string())
with open("debug_program", "w") as f:
f.write(str(fleet._origin_program))
return self.loss
def _build_programs(self):
"""
Build programs.
Build train_program, eval_program and inference_program. Only use in static graph mode.
"""
if self.run_infer:
self.startup_program = fluid.Program()
# build infer program
self.infer_program = fluid.Program()
with fluid.program_guard(self.infer_program, self.startup_program):
with fluid.unique_name.guard():
self.infer_feed_dict = inputs = self._get_feed_dict(
is_infer=True)
outputs = self.forward(inputs, is_infer=True)
predictions = self.infer(inputs, outputs)
self.infer_fetch_dict = predictions
self.infer_program = self.infer_program.clone(for_test=True)
self.program = self.infer_program
else:
if self.is_distributed:
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = 4
exec_strategy.num_iteration_per_drop_scope = 1
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.nccl_comm_num = 1
dist_strategy.fuse_all_reduce_ops = True
if self.use_recompute:
dist_strategy.forward_recompute = True
dist_strategy.enable_sequential_execution = True
if self.use_amp:
dist_strategy.use_amp = True
dist_strategy.amp_loss_scaling = self.amp_loss_scaling
self.dist_strategy = dist_strategy
self.startup_program = fluid.Program()
# build train program
self.train_program = fluid.Program()
with fluid.program_guard(self.train_program, self.startup_program):
with fluid.unique_name.guard():
self.feed_dict = inputs = self._get_feed_dict()
outputs = self.forward(inputs)
if self.is_distributed and self.use_recompute:
self.dist_strategy.recompute_checkpoints = outputs[
"checkpoints"]
metrics, statistics = self.get_metrics_and_statistics(
inputs, outputs)
# build eval program
self.eval_program = self.train_program.clone(for_test=True)
self.eval_fetch_dict = {**metrics, **statistics}
scheduled_lr = self.optimize(metrics)
metrics["scheduled_lr"] = scheduled_lr
self.train_fetch_dict = metrics
self.program = self.train_program
if self.is_distributed:
self.train_program = fleet.main_program
self.exe.run(self.startup_program)
if self.init_pretraining_params != "":
init_pretraining_params(self.exe, self.init_pretraining_params,
self.program)
elif self.init_checkpoint != "":
init_checkpoint(self.exe, self.init_checkpoint, self.program)
return
def main(args):
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
if args.use_cuda:
dev_list = fluid.cuda_places()
place = dev_list[0]
dev_count = len(dev_list)
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
exe = fluid.Executor(place)
reader = reader_ce.ClassifyReader(vocab_path=args.vocab_path,
label_map_config=args.label_map_config,
max_seq_len=args.max_seq_len,
total_num=args.train_data_size,
do_lower_case=args.do_lower_case,
in_tokens=args.in_tokens,
random_seed=args.random_seed,
tokenizer=args.tokenizer,
for_cn=args.for_cn,
task_id=args.task_id)
if not (args.do_train or args.do_val or args.do_test):
raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
"least one of them must be True.")
if args.do_test:
assert args.test_save is not None
startup_prog = fluid.Program()
if args.random_seed is not None:
startup_prog.random_seed = args.random_seed
if args.predict_batch_size == None:
args.predict_batch_size = args.batch_size
if args.do_train:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
dev_count = fleet.worker_num()
train_data_generator = reader.data_generator(
input_file=args.train_set,
batch_size=args.batch_size,
epoch=args.epoch,
dev_count=1,
trainer_id=fleet.worker_index(),
trainer_num=fleet.worker_num(),
shuffle=True,
phase="train")
num_train_examples = reader.get_num_examples(args.train_set)
if args.in_tokens:
max_train_steps = args.epoch * num_train_examples // (
args.batch_size // args.max_seq_len) // dev_count
else:
max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
warmup_steps = int(max_train_steps * args.warmup_proportion)
log.info("Device count: %d" % dev_count)
log.info("Num train examples: %d" % num_train_examples)
log.info("Max train steps: %d" % max_train_steps)
log.info("Num warmup steps: %d" % warmup_steps)
train_program = fluid.Program()
# use fleet api
exec_strategy = fluid.ExecutionStrategy()
if args.use_fast_executor:
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = dev_count
if args.is_distributed:
exec_strategy.num_threads = 3
exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.nccl_comm_num = 1
if args.is_distributed:
dist_strategy.nccl_comm_num = 2
dist_strategy.use_hierarchical_allreduce = True
if args.use_mix_precision:
dist_strategy.use_amp = True
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_pyreader, graph_vars = create_model(
args,
pyreader_name='train_reader',
ernie_config=ernie_config)
scheduled_lr = optimization(
loss=graph_vars["loss"],
warmup_steps=warmup_steps,
num_train_steps=max_train_steps,
learning_rate=args.learning_rate,
train_program=train_program,
startup_prog=startup_prog,
weight_decay=args.weight_decay,
scheduler=args.lr_scheduler,
use_dynamic_loss_scaling=args.use_dynamic_loss_scaling,
incr_every_n_steps=args.incr_every_n_steps,
decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
incr_ratio=args.incr_ratio,
decr_ratio=args.decr_ratio,
dist_strategy=dist_strategy)
if args.verbose:
if args.in_tokens:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program,
batch_size=args.batch_size // args.max_seq_len)
else:
lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
program=train_program, batch_size=args.batch_size)
log.info("Theoretical memory usage in training: %.3f - %.3f %s" %
(lower_mem, upper_mem, unit))
if args.do_val or args.do_test:
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_pyreader, graph_vars = create_model(
args,
pyreader_name='test_reader',
ernie_config=ernie_config,
is_prediction=True)
test_prog = test_prog.clone(for_test=True)
train_program = fleet.main_program
exe = fluid.Executor(place)
exe.run(startup_prog)
if args.do_train:
if args.init_checkpoint and args.init_pretraining_params:
log.warning(
"WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
"both are set! Only arg 'init_checkpoint' is made valid.")
if args.init_checkpoint:
init_checkpoint(exe,
args.init_checkpoint,
main_program=startup_prog)
elif args.init_pretraining_params:
init_pretraining_params(exe,
args.init_pretraining_params,
main_program=startup_prog)
elif args.do_val or args.do_test:
if not args.init_checkpoint:
raise ValueError("args 'init_checkpoint' should be set if"
"only doing validation or testing!")
init_checkpoint(exe, args.init_checkpoint, main_program=startup_prog)
if args.do_train:
train_exe = exe
train_pyreader.decorate_tensor_provider(train_data_generator)
else:
train_exe = None
test_exe = exe
# if args.do_val or args.do_test:
# if args.use_multi_gpu_test:
# test_exe = fluid.ParallelExecutor(
# use_cuda=args.use_cuda,
# main_program=test_prog,
# share_vars_from=train_exe)
current_epoch = 0
steps = 0
if args.do_train:
train_pyreader.start()
if warmup_steps > 0:
graph_vars["learning_rate"] = scheduled_lr
ce_info = []
time_begin = time.time()
last_epoch = 0
while True:
try:
steps += 1
# log.info("step: %d" % steps)
if fleet.worker_index() != 0:
train_exe.run(fetch_list=[], program=train_program)
continue
if steps % args.skip_steps != 0:
train_exe.run(fetch_list=[], program=train_program)
else:
outputs = evaluate(train_exe,
train_program,
train_pyreader,
graph_vars,
"train",
metric=args.metric)
if args.verbose:
verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
)
verbose += "learning rate: %f" % (
outputs["learning_rate"]
if warmup_steps > 0 else args.learning_rate)
log.info(verbose)
current_example, current_epoch = reader.get_train_progress(
)
time_end = time.time()
used_time = time_end - time_begin
log.info(
"epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
"ave acc: %f, speed: %f steps/s" %
(current_epoch, current_example * dev_count,
num_train_examples, steps, outputs["loss"],
outputs["accuracy"], args.skip_steps / used_time))
ce_info.append(
[outputs["loss"], outputs["accuracy"], used_time])
time_begin = time.time()
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(exe, save_path,
fleet._origin_program)
# if steps % args.validation_steps == 0 or last_epoch != current_epoch:
if steps % args.validation_steps == 0:
# evaluate dev set
if args.do_val:
evaluate_wrapper(args, reader, exe, test_prog,
test_pyreader, graph_vars,
current_epoch, steps)
if args.do_test:
predict_wrapper(args, reader, exe, test_prog,
test_pyreader, graph_vars,
current_epoch, steps)
if last_epoch != current_epoch:
last_epoch = current_epoch
except fluid.core.EOFException:
save_path = os.path.join(args.checkpoints,
"step_" + str(steps))
fluid.io.save_persistables(exe, save_path,
fleet._origin_program)
train_pyreader.reset()
break
# final eval on dev set
if args.do_val:
evaluate_wrapper(args, reader, exe, test_prog, test_pyreader,
graph_vars, current_epoch, steps)
# final eval on test set
if args.do_test:
predict_wrapper(args, reader, exe, test_prog, test_pyreader,
graph_vars, current_epoch, steps)
# final eval on dianostic, hack for glue-ax
if args.diagnostic:
test_pyreader.decorate_tensor_provider(
reader.data_generator(args.diagnostic,
batch_size=args.batch_size,
epoch=1,
dev_count=1,
shuffle=False))
log.info("Final diagnostic")
qids, preds, probs = predict(test_exe, test_prog, test_pyreader,
graph_vars)
assert len(qids) == len(preds), '{} v.s. {}'.format(
len(qids), len(preds))
with open(args.diagnostic_save, 'w') as f:
for id, s, p in zip(qids, preds, probs):
f.write('{}\t{}\t{}\n'.format(id, s, p))
log.info("Done final diagnostic, saving to {}".format(
args.diagnostic_save))
def train(args):
print("pretraining start")
ernie_config = ErnieConfig(args.ernie_config_path)
ernie_config.print_config()
with open(args.task_group_json) as f:
task_group = json.load(f)
exec_strategy = fluid.ExecutionStrategy()
if args.use_fast_executor:
exec_strategy.use_experimental_executor = True
exec_strategy.num_threads = 4 if args.use_amp else 2
exec_strategy.num_iteration_per_drop_scope = min(1, args.skip_steps)
node_nums = int(os.getenv("PADDLE_NODES_NUM"))
print("args.is_distributed:", args.is_distributed)
num_trainers = 1
trainer_id = 0
if args.is_distributed:
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
trainer_id = fleet.worker_index()
current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT")
worker_endpoints = fleet.worker_endpoints()
trainers_num = len(worker_endpoints)
print("worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"
.format(worker_endpoints, trainers_num, current_endpoint, trainer_id))
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.remove_unnecessary_lock = False # not useful
dist_strategy.fuse_all_reduce_ops = True if args.use_fuse else False
dist_strategy.nccl_comm_num = args.nccl_comm_num
if args.use_hierarchical_allreduce \
and trainers_num > args.hierarchical_allreduce_inter_nranks:
dist_strategy.use_hierarchical_allreduce = args.use_hierarchical_allreduce
dist_strategy.hierarchical_allreduce_inter_nranks = \
args.hierarchical_allreduce_inter_nranks
assert dist_strategy.use_hierarchical_allreduce > 1
assert trainers_num % dist_strategy.hierarchical_allreduce_inter_nranks == 0
dist_strategy.hierarchical_allreduce_exter_nranks = \
trainers_num / dist_strategy.hierarchical_allreduce_inter_nranks
if args.use_amp:
dist_strategy.use_amp = True
dist_strategy.amp_loss_scaling = args.init_loss_scaling
if args.use_recompute:
dist_strategy.forward_recompute = True
dist_strategy.enable_sequential_execution=True
trainer_id = fleet.worker_index()
current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT")
worker_endpoints = fleet.worker_endpoints()
trainers_num = len(worker_endpoints)
print("worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"
.format(worker_endpoints,trainers_num, current_endpoint, trainer_id))
else:
dist_strategy=None
gpu_id=0
gpus = fluid.core.get_cuda_device_count()
if args.is_distributed:
gpus = os.getenv("FLAGS_selected_gpus").split(",")
gpu_id = int(gpus[0])
if args.use_cuda:
place = fluid.CUDAPlace(gpu_id)
dev_count = len(gpus)
else:
place = fluid.CPUPlace()
dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
print("Device count %d, gpu_id:%d" % (dev_count, gpu_id))
train_program = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_pyreader, fetch_vars = create_model(
pyreader_name='train_reader', ernie_config=ernie_config, task_group=task_group)
graph_vars = fetch_vars["graph_vars"]
checkpoints = fetch_vars["checkpoints"]
total_loss = graph_vars[-1]
if args.use_recompute:
dist_strategy.recompute_checkpoints = checkpoints
scheduled_lr, loss_scaling = optimization(
loss=total_loss,
warmup_steps=args.warmup_steps,
num_train_steps=args.num_train_steps,
learning_rate=args.learning_rate,
train_program=train_program,
startup_prog=startup_prog,
weight_decay=args.weight_decay,
scheduler=args.lr_scheduler,
use_fp16=args.use_amp,
use_dynamic_loss_scaling=args.use_dynamic_loss_scaling,
init_loss_scaling=args.init_loss_scaling,
incr_every_n_steps=args.incr_every_n_steps,
decr_every_n_nan_or_inf=args.decr_every_n_nan_or_inf,
incr_ratio=args.incr_ratio,
decr_ratio=args.decr_ratio,
dist_strategy=dist_strategy)
origin_train_program = train_program
if args.is_distributed:
#raped by fleet, need to assign fleet's modified train_grogram back
train_program = fleet.main_program
origin_train_program = fleet._origin_program
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_pyreader, fetch_vars = create_model(
pyreader_name='test_reader', ernie_config=ernie_config, task_group=task_group)
graph_vars = fetch_vars["graph_vars"]
total_loss = graph_vars[-1]
test_prog = test_prog.clone(for_test=True)
exe = fluid.Executor(place)
exe.run(startup_prog)
if args.init_checkpoint and args.init_checkpoint != "":
#init_checkpoint(exe, args.init_checkpoint, origin_train_program, args.use_amp)
init_pretraining_params(exe, args.init_checkpoint, origin_train_program, args.use_amp)
data_reader = ErnieDataReader(
task_group,
False,
batch_size=args.batch_size,
vocab_path=args.vocab_path,
voc_size=ernie_config['vocab_size'],
epoch=args.epoch,
max_seq_len=args.max_seq_len,
generate_neg_sample=args.generate_neg_sample,
hack_old_trainset=args.hack_old_data)
#only fleet
train_exe = exe
predict = predict_wrapper(
args,
exe,
ernie_config,
task_group,
test_prog=test_prog,
pyreader=test_pyreader,
fetch_list=[var.name for var in graph_vars])
train_pyreader.set_batch_generator(data_reader.data_generator())
train_pyreader.start()
steps = 112000
time_begin = time.time()
node_nums = int(os.getenv("PADDLE_NODES_NUM"))
while True:#steps < args.num_train_steps:
try:
steps += 1#node_nums
skip_steps = args.skip_steps# * node_nums
fetch_list = []
if trainer_id == 0 and steps % skip_steps == 0:
fetch_list = [var.name for var in graph_vars] + [scheduled_lr.name]
if args.use_amp:
fetch_list.append(loss_scaling.name)
outputs = train_exe.run(fetch_list=fetch_list, program=train_program)
time_end = time.time()
used_time = time_end - time_begin
if outputs:
each_mask_lm_cost, lm_w = outputs[:2]
if args.use_amp:
each_total_constract_loss, each_total_cost, np_lr, l_scaling = outputs[-4:]
else:
each_total_constract_loss, each_total_cost, np_lr = outputs[-3:]
acc_list =[]
index = 2
for task in task_group:
each_task_acc = outputs[index]
task_w = outputs[index + 1]
acc = np.sum(each_task_acc * task_w) / np.sum(task_w)
acc_list.append("%s acc: %f" % (task["task_name"], acc))
index += 2
print("feed_queue size", train_pyreader.queue.size())
epoch, current_file_index, total_file, current_file, mask_type = data_reader.get_progress()
if args.use_amp:
print("current learning_rate:%f, loss scaling:%f" % (np_lr[0], l_scaling[0]))
else:
print("current learning_rate:%f" % np_lr[0])
print(
"epoch: %d, progress: %d/%d, step: %d, constract_loss: %f, loss: %f, "
"ppl: %f, %s, speed: %f steps/s, file: %s, mask_type: %s"
% (epoch, current_file_index, total_file, steps,
np.mean(each_total_constract_loss), np.mean(each_total_cost),
np.exp(np.sum(each_mask_lm_cost * lm_w) / np.sum(lm_w)),
", ".join(acc_list), skip_steps / used_time,
current_file, mask_type))
time_begin = time.time()
elif steps % skip_steps == 0:
epoch, current_file_index, total_file, current_file, mask_type = data_reader.get_progress(
)
print("feed_queue size", train_pyreader.queue.size())
print("epoch: %d, progress: %d/%d, step: %d, "
"speed: %f steps/s, file: %s, mask_type: %s"
% (epoch, current_file_index, total_file, steps,
skip_steps / used_time, current_file, mask_type))
time_begin = time.time()
if not trainer_id == 0:
continue
if steps % args.save_steps == 0:
save_path = os.path.join(args.checkpoints, "step_" + str(steps))
fluid.io.save_persistables(exe, save_path, origin_train_program)
if steps % args.validation_steps == 0:
valid_list = predict()
print("[validation_set] epoch: %d, step: %d, %s" % \
(epoch, steps, ", ".join(valid_list)))
except fluid.core.EOFException:
train_pyreader.reset()
break
def net(self):
args = self.p_args()
bert_config = BertConfig("uncased_L-24_H-1024_A-16/bert_config.json")
bert_config.print_config()
place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
dev_count = 1
if args.do_train:
my_dist_env = dist_env()
worker_endpoints_env = my_dist_env["trainer_endpoints"]
worker_endpoints = worker_endpoints_env.split(",")
current_endpoint = my_dist_env["current_endpoint"]
trainer_id = worker_endpoints.index(current_endpoint)
# new rolemaker here
print("current_id: ", trainer_id)
print("worker_endpoints: ", worker_endpoints)
role = role_maker.UserDefinedCollectiveRoleMaker(
current_id=trainer_id, worker_endpoints=worker_endpoints)
# Fleet get role of each worker
fleet.init(role)
exe = fluid.Executor(place)
# init program
train_program = fluid.Program()
startup_prog = fluid.Program()
if args.random_seed != 0:
print("set program random seed as: ", args.random_seed)
startup_prog.random_seed = args.random_seed
train_program.random_seed = args.random_seed
task_name = args.task_name.lower()
processors = {
'xnli': reader.XnliProcessor,
'cola': reader.ColaProcessor,
'mrpc': reader.MrpcProcessor,
'mnli': reader.MnliProcessor,
}
processor = processors[task_name](data_dir=args.data_dir,
vocab_path=args.vocab_path,
max_seq_len=args.max_seq_len,
do_lower_case=args.do_lower_case,
in_tokens=args.in_tokens,
random_seed=args.random_seed)
num_labels = len(processor.get_labels())
dev_count = len(worker_endpoints)
# we need to keep every trainer of fleet the same shuffle_seed
print("shuffle_seed: ", args.shuffle_seed)
self.train_data_generator = processor.data_generator(
batch_size=args.batch_size,
phase='train',
epoch=args.epoch,
dev_count=dev_count,
dev_idx=0,
shuffle=args.shuffle,
shuffle_seed=args.shuffle_seed)
num_train_examples = processor.get_num_examples(phase='train')
max_train_steps = 5
self.warmup_steps = int(5 * 0.1)
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.use_experimental_executor = args.use_fast_executor
exec_strategy.num_threads = dev_count
exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.nccl_comm_num = 3
dist_strategy.use_hierarchical_allreduce = True
#dist_strategy.mode = "collective"
#dist_strategy.collective_mode = "grad_allreduce"
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
self.train_pyreader, self.loss, probs, accuracy, num_seqs, checkpoints = create_model(
args, bert_config=bert_config, num_labels=num_labels)
scheduled_lr = optimization(loss=self.loss,
warmup_steps=self.warmup_steps,
num_train_steps=max_train_steps,
learning_rate=args.learning_rate,
train_program=train_program,
startup_prog=startup_prog,
weight_decay=args.weight_decay,
scheduler=args.lr_scheduler,
use_fp16=False,
loss_scaling=args.loss_scaling,
dist_strategy=dist_strategy)
exe.run(startup_prog)
with open("__model__", "wb") as f:
f.write(fleet._origin_program.desc.serialize_to_string())
with open("debug_program", "w") as f:
f.write(str(fleet._origin_program))
return self.loss
def main():
role = role_maker.PaddleCloudRoleMaker(is_collective=True) # new line 3
fleet.init(role) # new line 4
env = os.environ
num_trainers = int(env.get('PADDLE_TRAINERS_NUM', 0))
assert num_trainers != 0, "multi-machine training process must be started using distributed.launch..."
trainer_id = int(env.get("PADDLE_TRAINER_ID", 0))
# set different seeds for different trainers
random.seed(trainer_id)
np.random.seed(trainer_id)
if FLAGS.enable_ce:
random.seed(0)
np.random.seed(0)
cfg = load_config(FLAGS.config)
merge_config(FLAGS.opt)
check_config(cfg)
# check if set use_gpu=True in paddlepaddle cpu version
check_gpu(cfg.use_gpu)
# check if paddlepaddle version is satisfied
check_version()
save_only = getattr(cfg, 'save_prediction_only', False)
if save_only:
raise NotImplementedError('The config file only support prediction,'
' training stage is not implemented now')
main_arch = cfg.architecture
assert cfg.use_gpu == True, "GPU must be supported for multi-machine training..."
devices_num = fluid.core.get_cuda_device_count()
if 'FLAGS_selected_gpus' in env:
device_id = int(env['FLAGS_selected_gpus'])
else:
device_id = 0
place = fluid.CUDAPlace(device_id) if cfg.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
lr_builder = create('LearningRate')
optim_builder = create('OptimizerBuilder')
# build program
startup_prog = fluid.Program()
train_prog = fluid.Program()
if FLAGS.enable_ce:
startup_prog.random_seed = 1000
train_prog.random_seed = 1000
with fluid.program_guard(train_prog, startup_prog):
with fluid.unique_name.guard():
model = create(main_arch)
if FLAGS.fp16:
assert (getattr(model.backbone, 'norm_type', None)
!= 'affine_channel'), \
'--fp16 currently does not support affine channel, ' \
' please modify backbone settings to use batch norm'
with mixed_precision_context(FLAGS.loss_scale, FLAGS.fp16) as ctx:
inputs_def = cfg['TrainReader']['inputs_def']
feed_vars, train_loader = model.build_inputs(**inputs_def)
train_fetches = model.train(feed_vars)
loss = train_fetches['loss']
if FLAGS.fp16:
loss *= ctx.get_loss_scale_var()
lr = lr_builder()
optimizer = optim_builder(lr)
dist_strategy = DistributedStrategy()
sync_bn = getattr(model.backbone, 'norm_type',
None) == 'sync_bn'
dist_strategy.sync_batch_norm = sync_bn
dist_strategy.nccl_comm_num = 1
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 3
exec_strategy.num_iteration_per_drop_scope = 30
dist_strategy.exec_strategy = exec_strategy
dist_strategy.fuse_all_reduce_ops = True
optimizer = fleet.distributed_optimizer(
optimizer, strategy=dist_strategy) # new line 5
optimizer.minimize(loss)
if FLAGS.fp16:
loss /= ctx.get_loss_scale_var()
if 'use_ema' in cfg and cfg['use_ema']:
global_steps = _decay_step_counter()
ema = ExponentialMovingAverage(cfg['ema_decay'],
thres_steps=global_steps)
ema.update()
# parse train fetches
train_keys, train_values, _ = parse_fetches(train_fetches)
train_values.append(lr)
if FLAGS.eval:
eval_prog = fluid.Program()
with fluid.program_guard(eval_prog, startup_prog):
with fluid.unique_name.guard():
model = create(main_arch)
inputs_def = cfg['EvalReader']['inputs_def']
feed_vars, eval_loader = model.build_inputs(**inputs_def)
fetches = model.eval(feed_vars)
eval_prog = eval_prog.clone(True)
eval_reader = create_reader(cfg.EvalReader, devices_num=1)
# When iterable mode, set set_sample_list_generator(eval_reader, place)
eval_loader.set_sample_list_generator(eval_reader)
# parse eval fetches
extra_keys = []
if cfg.metric == 'COCO':
extra_keys = ['im_info', 'im_id', 'im_shape']
if cfg.metric == 'VOC':
extra_keys = ['gt_bbox', 'gt_class', 'is_difficult']
if cfg.metric == 'WIDERFACE':
extra_keys = ['im_id', 'im_shape', 'gt_bbox']
eval_keys, eval_values, eval_cls = parse_fetches(
fetches, eval_prog, extra_keys)
exe.run(startup_prog)
compiled_train_prog = fleet.main_program
if FLAGS.eval:
compiled_eval_prog = fluid.CompiledProgram(eval_prog)
fuse_bn = getattr(model.backbone, 'norm_type', None) == 'affine_channel'
ignore_params = cfg.finetune_exclude_pretrained_params \
if 'finetune_exclude_pretrained_params' in cfg else []
start_iter = 0
if FLAGS.resume_checkpoint:
checkpoint.load_checkpoint(exe, train_prog, FLAGS.resume_checkpoint)
start_iter = checkpoint.global_step()
elif cfg.pretrain_weights and fuse_bn and not ignore_params:
checkpoint.load_and_fusebn(exe, train_prog, cfg.pretrain_weights)
elif cfg.pretrain_weights:
checkpoint.load_params(exe,
train_prog,
cfg.pretrain_weights,
ignore_params=ignore_params)
train_reader = create_reader(cfg.TrainReader,
(cfg.max_iters - start_iter) * devices_num,
cfg,
devices_num=devices_num)
# When iterable mode, set set_sample_list_generator(train_reader, place)
train_loader.set_sample_list_generator(train_reader)
# whether output bbox is normalized in model output layer
is_bbox_normalized = False
if hasattr(model, 'is_bbox_normalized') and \
callable(model.is_bbox_normalized):
is_bbox_normalized = model.is_bbox_normalized()
# if map_type not set, use default 11point, only use in VOC eval
map_type = cfg.map_type if 'map_type' in cfg else '11point'
train_stats = TrainingStats(cfg.log_iter, train_keys)
train_loader.start()
start_time = time.time()
end_time = time.time()
cfg_name = os.path.basename(FLAGS.config).split('.')[0]
save_dir = os.path.join(cfg.save_dir, cfg_name)
time_stat = deque(maxlen=cfg.log_iter)
best_box_ap_list = [0.0, 0] #[map, iter]
# use VisualDL to log data
if FLAGS.use_vdl:
assert six.PY3, "VisualDL requires Python >= 3.5"
from visualdl import LogWriter
vdl_writer = LogWriter(FLAGS.vdl_log_dir)
vdl_loss_step = 0
vdl_mAP_step = 0
for it in range(start_iter, cfg.max_iters):
start_time = end_time
end_time = time.time()
time_stat.append(end_time - start_time)
time_cost = np.mean(time_stat)
eta_sec = (cfg.max_iters - it) * time_cost
eta = str(datetime.timedelta(seconds=int(eta_sec)))
outs = exe.run(compiled_train_prog, fetch_list=train_values)
stats = {k: np.array(v).mean() for k, v in zip(train_keys, outs[:-1])}
# use vdl-paddle to log loss
if FLAGS.use_vdl:
if it % cfg.log_iter == 0:
for loss_name, loss_value in stats.items():
vdl_writer.add_scalar(loss_name, loss_value, vdl_loss_step)
vdl_loss_step += 1
train_stats.update(stats)
logs = train_stats.log()
if it % cfg.log_iter == 0 and trainer_id == 0:
strs = 'iter: {}, lr: {:.6f}, {}, time: {:.3f}, eta: {}'.format(
it, np.mean(outs[-1]), logs, time_cost, eta)
logger.info(strs)
# NOTE : profiler tools, used for benchmark
if FLAGS.is_profiler and it == 5:
profiler.start_profiler("All")
elif FLAGS.is_profiler and it == 10:
profiler.stop_profiler("total", FLAGS.profiler_path)
return
if (it > 0 and it % cfg.snapshot_iter == 0 or it == cfg.max_iters - 1) \
and trainer_id == 0:
save_name = str(it) if it != cfg.max_iters - 1 else "model_final"
if 'use_ema' in cfg and cfg['use_ema']:
exe.run(ema.apply_program)
checkpoint.save(exe, train_prog, os.path.join(save_dir, save_name))
if FLAGS.eval:
# evaluation
resolution = None
if 'Mask' in cfg.architecture:
resolution = model.mask_head.resolution
results = eval_run(exe,
compiled_eval_prog,
eval_loader,
eval_keys,
eval_values,
eval_cls,
cfg,
resolution=resolution)
box_ap_stats = eval_results(results, cfg.metric,
cfg.num_classes, resolution,
is_bbox_normalized,
FLAGS.output_eval, map_type,
cfg['EvalReader']['dataset'])
# use vdl_paddle to log mAP
if FLAGS.use_vdl:
vdl_writer.add_scalar("mAP", box_ap_stats[0], vdl_mAP_step)
vdl_mAP_step += 1
if box_ap_stats[0] > best_box_ap_list[0]:
best_box_ap_list[0] = box_ap_stats[0]
best_box_ap_list[1] = it
checkpoint.save(exe, train_prog,
os.path.join(save_dir, "best_model"))
logger.info("Best test box ap: {}, in iter: {}".format(
best_box_ap_list[0], best_box_ap_list[1]))
if 'use_ema' in cfg and cfg['use_ema']:
exe.run(ema.restore_program)
train_loader.reset()
def do_training(self, fleet, args):
"""
begin training.
Args:
fleet (Collective): Collective inherited base class Fleet
args (ArgumentParser): run args to config dist fleet.
Returns:
tuple: the value is train losses
"""
args = parse_args()
logging.info(args)
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 4))
place = fluid.CUDAPlace(gpu_id)
dev_count = 1
exe = fluid.Executor(place)
train_program = fluid.Program()
startup_program = fluid.Program()
args.num_trainers = fleet.worker_num()
args.trainer_id = fleet.worker_index()
args.run_params = json.loads(args.run_params)
dist_strategy = DistributedStrategy()
dist_strategy.enable_inplace = args.run_params['enable_inplace']
dist_strategy.fuse_all_reduce_ops = args.run_params[
'fuse_all_reduce_ops']
dist_strategy.nccl_comm_num = args.run_params['nccl_comm_num']
dist_strategy.use_local_sgd = args.run_params['use_local_sgd']
dist_strategy.mode = args.run_params["mode"]
dist_strategy.collective_mode = args.run_params["collective"]
with fluid.program_guard(train_program, startup_program):
with fluid.unique_name.guard():
sum_cost, avg_cost, predict, token_num, pyreader = transformer(
ModelHyperParams.src_vocab_size,
ModelHyperParams.trg_vocab_size,
ModelHyperParams.max_length + 1,
ModelHyperParams.n_layer,
ModelHyperParams.n_head,
ModelHyperParams.d_key,
ModelHyperParams.d_value,
ModelHyperParams.d_model,
ModelHyperParams.d_inner_hid,
ModelHyperParams.prepostprocess_dropout,
ModelHyperParams.attention_dropout,
ModelHyperParams.relu_dropout,
ModelHyperParams.preprocess_cmd,
ModelHyperParams.postprocess_cmd,
ModelHyperParams.weight_sharing,
TrainTaskConfig.label_smooth_eps,
ModelHyperParams.bos_idx,
use_py_reader=args.use_py_reader,
is_test=False)
optimizer = fluid.optimizer.SGD(0.003)
if args.run_params["fp16"]:
optimizer = decorate(optimizer, init_loss_scaling=64.0)
optimizer = fleet.distributed_optimizer(optimizer,
strategy=dist_strategy)
optimizer.minimize(avg_cost, startup_program)
train_program = fleet.main_program
exe.run(startup_program)
train_data = prepare_data_generator(
args,
is_test=False,
count=dev_count,
pyreader=pyreader,
py_reader_provider_wrapper=py_reader_provider_wrapper)
loss_normalizer = -(
(1. - TrainTaskConfig.label_smooth_eps) * np.log(
(1. - TrainTaskConfig.label_smooth_eps)) +
TrainTaskConfig.label_smooth_eps *
np.log(TrainTaskConfig.label_smooth_eps /
(ModelHyperParams.trg_vocab_size - 1) + 1e-20))
step_idx = 0
init_flag = True
result_loss = []
result_ppl = []
train_info = []
for pass_id in six.moves.xrange(args.num_epochs):
pass_start_time = time.time()
if args.use_py_reader:
pyreader.start()
data_generator = None
else:
data_generator = train_data()
batch_id = 0
while True:
try:
feed_dict_list = prepare_feed_dict_list(
data_generator, init_flag, dev_count)
t1 = time.time()
outs = exe.run(program=train_program,
fetch_list=[sum_cost.name, token_num.name]
if step_idx % args.fetch_steps == 0 else [],
feed=feed_dict_list)
if step_idx % args.fetch_steps == 0:
sum_cost_val, token_num_val = np.array(
outs[0]), np.array(outs[1])
total_sum_cost = sum_cost_val.sum()
total_token_num = token_num_val.sum()
total_avg_cost = total_sum_cost / total_token_num
result_loss.append(total_avg_cost - loss_normalizer)
result_ppl.append(
np.exp([min(total_avg_cost, 100)]).item(0))
train_info.append(result_loss)
init_flag = False
batch_id += 1
step_idx += 1
if batch_id >= 5:
break
except (StopIteration, fluid.core.EOFException):
if args.use_py_reader:
pyreader.reset()
break
train_info = [round(i, 6) for i in train_info[0]]
return train_info
def train(args):
# parameters from arguments
model_name = args.model
checkpoint = args.checkpoint
pretrained_model = args.pretrained_model
model_save_dir = args.model_save_dir
use_mixup = args.use_mixup
use_ngraph = os.getenv('FLAGS_use_ngraph')
startup_prog = fluid.Program()
train_prog = fluid.Program()
test_prog = fluid.Program()
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = args.num_threads
exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.enable_inplace = args.with_inplace
if args.fuse:
dist_strategy.fuse_all_reduce_ops = 1
dist_strategy.nccl_comm_num = args.nccl_comm_num
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
b_out = build_program(
is_train=True,
main_prog=train_prog,
startup_prog=startup_prog,
args=args,
dist_strategy=dist_strategy)
if use_mixup:
train_py_reader, train_cost, global_lr = b_out[0], b_out[1], b_out[2]
train_fetch_vars = [train_cost, global_lr]
train_fetch_list = []
for var in train_fetch_vars:
var.persistable=True
train_fetch_list.append(var.name)
else:
train_py_reader, train_cost, train_acc1, train_acc5, global_lr = b_out[0],b_out[1],b_out[2],b_out[3],b_out[4]
train_fetch_vars = [train_cost, train_acc1, train_acc5, global_lr]
train_fetch_list = []
for var in train_fetch_vars:
var.persistable=True
train_fetch_list.append(var.name)
train_prog = fleet.main_program
b_out_test = build_program(
is_train=False,
main_prog=test_prog,
startup_prog=startup_prog,
args=args,
dist_strategy=dist_strategy)
test_py_reader, test_cost, test_acc1, test_acc5 = b_out_test[0],b_out_test[1],b_out_test[2],b_out_test[3]
test_prog = test_prog.clone(for_test=True)
test_prog = compiler.CompiledProgram(test_prog).with_data_parallel(loss_name=test_cost.name, exec_strategy=exec_strategy)
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
if checkpoint is not None:
fluid.io.load_persistables(exe, checkpoint, main_program=train_prog)
if pretrained_model:
def if_exist(var):
return os.path.exists(os.path.join(pretrained_model, var.name))
fluid.io.load_vars(
exe, pretrained_model, main_program=train_prog, predicate=if_exist)
if args.use_gpu:
device_num = get_device_num()
else:
device_num = 1
train_batch_size = args.batch_size
print("train_batch_size: %d device_num:%d" % (train_batch_size, device_num))
test_batch_size = 16
# NOTE: the order of batch data generated by batch_reader
# must be the same in the respective processes.
shuffle_seed = 1 if num_trainers > 1 else None
train_reader = reader.train(settings=args, data_dir=args.data_dir, pass_id_as_seed=shuffle_seed)
test_reader = reader.val(settings=args, data_dir=args.data_dir)
train_py_reader.decorate_paddle_reader(paddle.batch(train_reader,
batch_size=train_batch_size))
test_py_reader.decorate_paddle_reader(paddle.batch(test_reader,
batch_size=test_batch_size))
test_fetch_vars = [test_cost, test_acc1, test_acc5]
test_fetch_list = []
for var in test_fetch_vars:
var.persistable=True
test_fetch_list.append(var.name)
train_exe = exe
params = models.__dict__[args.model]().params
for pass_id in range(params["num_epochs"]):
train_py_reader.start()
train_info = [[], [], []]
test_info = [[], [], []]
train_time = []
train_begin=time.time()
batch_id = 0
time_record=[]
try:
while True:
t1 = time.time()
if use_mixup:
loss, lr = train_exe.run(train_prog, fetch_list=train_fetch_list)
else:
loss, acc1, acc5, lr = train_exe.run(train_prog, fetch_list=train_fetch_list)
acc1 = np.mean(np.array(acc1))
acc5 = np.mean(np.array(acc5))
train_info[1].append(acc1)
train_info[2].append(acc5)
t2 = time.time()
period = t2 - t1
time_record.append(period)
loss = np.mean(np.array(loss))
train_info[0].append(loss)
lr = np.mean(np.array(lr))
train_time.append(period)
if batch_id % 10 == 0:
period = np.mean(time_record)
speed = args.batch_size * 1.0 / period
time_record=[]
if use_mixup:
print("Pass {0}, trainbatch {1}, loss {2}, lr {3}, time {4}, speed {5}"
.format(pass_id, batch_id, "%.5f"%loss, "%.5f" %lr, "%2.2f sec" % period, "%.2f" % speed))
else:
print("Pass {0}, trainbatch {1}, loss {2}, \
acc1 {3}, acc5 {4}, lr {5}, time {6}, speed {7}"
.format(pass_id, batch_id, "%.5f"%loss, "%.5f"%acc1, "%.5f"%acc5, "%.5f" %
lr, "%2.2f sec" % period, "%.2f" % speed))
sys.stdout.flush()
batch_id += 1
except fluid.core.EOFException:
train_py_reader.reset()
train_loss = np.array(train_info[0]).mean()
if not use_mixup:
train_acc1 = np.array(train_info[1]).mean()
train_acc5 = np.array(train_info[2]).mean()
train_end=time.time()
train_speed = (batch_id * train_batch_size) / (train_end - train_begin)
# test only run in last epoch
if (pass_id + 1) == params["num_epochs"]:
test_py_reader.start()
test_batch_id = 0
try:
while True:
t1 = time.time()
loss, acc1, acc5 = exe.run(program=test_prog,
fetch_list=test_fetch_list)
t2 = time.time()
period = t2 - t1
loss = np.mean(loss)
acc1 = np.mean(acc1)
acc5 = np.mean(acc5)
test_info[0].append(loss)
test_info[1].append(acc1)
test_info[2].append(acc5)
if test_batch_id % 10 == 0:
test_speed = test_batch_size * 1.0 / period
print("Pass {0},testbatch {1},loss {2}, \
acc1 {3},acc5 {4},time {5},speed {6}"
.format(pass_id, test_batch_id, "%.5f"%loss,"%.5f"%acc1, "%.5f"%acc5,
"%2.2f sec" % period, "%.2f" % test_speed))
sys.stdout.flush()
test_batch_id += 1
except fluid.core.EOFException:
test_py_reader.reset()
test_loss = np.array(test_info[0]).mean()
test_acc1 = np.array(test_info[1]).mean()
test_acc5 = np.array(test_info[2]).mean()
if trainer_id == 0:
model_path = os.path.join(model_save_dir + '/' + model_name, str(pass_id))
if not os.path.isdir(model_path):
os.makedirs(model_path)
fluid.io.save_persistables(exe, model_path, main_program=fleet._origin_program)
if args.benchmark_test:
if not os.path.isdir("./benchmark_logs/"):
os.makedirs("./benchmark_logs/")
with open("./benchmark_logs/log_%d" % trainer_id, 'w') as f:
result = dict()
result['0'] = dict()
result['0']['acc1'] = str(test_acc1)
result['0']['acc5'] = str(test_acc5)
result['1'] = str(train_speed * num_trainers)
print(result)
f.writelines(json.dumps(result) + '\n')
if use_mixup:
print("End pass {0}, train_loss {1}, speed {2}".format(pass_id, "%.5f"%train_loss, "%.2f" % train_speed))
else:
print("End pass {0}, train_loss {1}, train_acc1 {2}, train_acc5 {3}, ""speed {4}".format(
pass_id, "%.5f"%train_loss, "%.5f"%train_acc1, "%.5f"%train_acc5, "%.2f" % train_speed))
sys.stdout.flush()
def net(self, args=None):
"""
resnet struct.
Args:
fleet:
args (ArgumentParser): run args to config dist fleet.
Returns:
tuple: the return value contains avg_cost, py_reader
"""
from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy
from thirdparty.image_classfication.models.resnet import ResNet50
from thirdparty.image_classfication.train import parser
from thirdparty.image_classfication.train import optimizer_setting
parser.add_argument('--update_method',
type=str,
required=True,
choices=['pserver', 'nccl'])
parser.add_argument('--role',
type=str,
required=True,
choices=['pserver', 'trainer'])
parser.add_argument('--endpoints',
type=str,
required=False,
default="")
parser.add_argument('--current_id',
type=int,
required=False,
default=0)
parser.add_argument('--trainers', type=int, required=False, default=1)
# parser.add_argument('--sync_mode', action='store_true')
parser.add_argument('--run_params',
type=str,
required=False,
default='{}')
args = parser.parse_args()
args.run_params = json.loads(args.run_params)
image_shape = [3, 224, 224]
scale_loss = 1.0
self.py_reader = fluid.layers.py_reader(capacity=16,
shapes=[[-1] + image_shape,
[-1, 1]],
lod_levels=[0, 0],
dtypes=["float32", "int64"],
use_double_buffer=True)
image, label = fluid.layers.read_file(self.py_reader)
run_model = ResNet50()
out = run_model.net(image, 4)
softmax_out = fluid.layers.softmax(out, use_cudnn=False)
cost, prob = fluid.layers.softmax_with_cross_entropy(
out, label, return_softmax=True)
self.avg_cost = fluid.layers.mean(cost)
params = run_model.params
params["total_images"] = args.total_images
params["lr"] = 1e-5
params["num_epochs"] = args.num_epochs
params["learning_strategy"]["batch_size"] = args.batch_size
params["learning_strategy"]["name"] = args.lr_strategy
params["l2_decay"] = args.l2_decay
params["momentum_rate"] = args.momentum_rate
optimizer = optimizer_setting(params)
global_lr = optimizer._global_learning_rate()
global_lr.persistable = True
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = 1
exec_strategy.num_iteration_per_drop_scope = 30
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.enable_inplace = args.run_params['enable_inplace']
dist_strategy.fuse_all_reduce_ops = args.run_params[
'fuse_all_reduce_ops']
dist_strategy.nccl_comm_num = args.run_params['nccl_comm_num']
dist_strategy.use_local_sgd = args.run_params['use_local_sgd']
dist_strategy.mode = args.run_params["mode"]
dist_strategy.collective_mode = args.run_params["collective"]
if args.run_params["fp16"]:
optimizer = fluid.contrib.mixed_precision.decorate(
optimizer,
init_loss_scaling=128.0,
use_dynamic_loss_scaling=True)
if "use_dgc" in args.run_params and args.run_params["use_dgc"]:
# use dgc must close fuse
dist_strategy.fuse_all_reduce_ops = False
optimizer = fluid.optimizer.DGCMomentumOptimizer(
learning_rate=0.001, momentum=0.9, rampup_begin_step=0)
dist_optimizer = fleet.distributed_optimizer(optimizer,
strategy=dist_strategy)
_, param_grads = dist_optimizer.minimize(self.avg_cost)
shuffle_seed = 1
train_reader = reader.train(settings=args,
data_dir=DATA_DIR,
pass_id_as_seed=shuffle_seed)
self.py_reader.decorate_paddle_reader(
paddle.batch(train_reader, batch_size=self.batch_size))
if scale_loss > 1:
avg_cost = fluid.layers.mean(x=cost) * scale_loss
return self.avg_cost, self.py_reader