def distributed_optimize(optimizer):
'''
A part of configuration for distributed training
'''
strategy = DistributedStrategy()
strategy.fuse_all_reduce_ops = True
strategy.nccl_comm_num = 2
strategy.fuse_elewise_add_act_ops=True
strategy.fuse_bn_act_ops = True
return fleet.distributed_optimizer(optimizer, strategy=strategy)
def train(args):
# parameters from arguments
model_name = args.model
checkpoint = args.checkpoint
pretrained_model = args.pretrained_model
model_save_dir = args.model_save_dir
use_mixup = args.use_mixup
use_ngraph = os.getenv('FLAGS_use_ngraph')
startup_prog = fluid.Program()
train_prog = fluid.Program()
test_prog = fluid.Program()
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = args.num_threads
exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
dist_strategy = DistributedStrategy()
dist_strategy.exec_strategy = exec_strategy
dist_strategy.enable_inplace = args.with_inplace
if not args.fuse:
dist_strategy.fuse_all_reduce_ops = False
dist_strategy.nccl_comm_num = args.nccl_comm_num
dist_strategy.fuse_elewise_add_act_ops=args.fuse_elewise_add_act_ops
role = role_maker.PaddleCloudRoleMaker(is_collective=True)
fleet.init(role)
b_out = build_program(
is_train=True,
main_prog=train_prog,
startup_prog=startup_prog,
args=args,
dist_strategy=dist_strategy,
data_layout=args.data_format)
if use_mixup:
train_data_loader, train_cost, global_lr = b_out[0], b_out[1], b_out[2]
train_fetch_vars = [train_cost, global_lr]
train_fetch_list = []
for var in train_fetch_vars:
var.persistable=True
train_fetch_list.append(var.name)
else:
train_data_loader, train_cost, train_acc1, train_acc5, global_lr = b_out[0],b_out[1],b_out[2],b_out[3],b_out[4]
train_fetch_vars = [train_cost, train_acc1, train_acc5, global_lr]
train_fetch_list = []
for var in train_fetch_vars:
var.persistable=True
train_fetch_list.append(var.name)
train_prog = fleet.main_program
b_out_test = build_program(
is_train=False,
main_prog=test_prog,
startup_prog=startup_prog,
args=args,
dist_strategy=dist_strategy,
data_layout=args.data_format)
test_data_loader, test_cost, test_acc1, test_acc5 = b_out_test[0],b_out_test[1],b_out_test[2],b_out_test[3]
test_prog = test_prog.clone(for_test=True)
test_prog = compiler.CompiledProgram(test_prog).with_data_parallel(loss_name=test_cost.name, exec_strategy=exec_strategy)
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_prog)
if checkpoint is not None:
fluid.io.load_persistables(exe, checkpoint, main_program=train_prog)
if pretrained_model:
def if_exist(var):
return os.path.exists(os.path.join(pretrained_model, var.name))
fluid.io.load_vars(
exe, pretrained_model, main_program=train_prog, predicate=if_exist)
if args.use_gpu:
device_num = get_device_num()
else:
device_num = 1
train_batch_size = args.batch_size
print("train_batch_size: %d device_num:%d" % (train_batch_size, device_num))
test_batch_size = args.batch_size
# NOTE: the order of batch data generated by batch_reader
# must be the same in the respective processes.
shuffle_seed = 1 if num_trainers > 1 else None
if args.use_dali:
import dali
train_iter = dali.train(settings=args, trainer_id=trainer_id, trainers_num=num_trainers,
gpu_id=gpu_id, data_layout=args.data_format)
else:
train_reader = reader.train(settings=args, data_dir=args.data_dir,
pass_id_as_seed=shuffle_seed, data_layout=args.data_format, threads=10)
train_batch_reader=paddle.batch(train_reader, batch_size=train_batch_size)
test_reader = reader.val(settings=args, data_dir=args.data_dir, data_layout=args.data_format, threads=10)
test_batch_reader=paddle.batch(test_reader, batch_size=test_batch_size)
places = place
if num_trainers <= 1 and args.use_gpu:
places = fluid.framework.cuda_places()
train_data_loader.set_sample_list_generator(train_batch_reader, places)
test_data_loader.set_sample_list_generator(test_batch_reader, place)
test_fetch_vars = [test_cost, test_acc1, test_acc5]
test_fetch_list = []
for var in test_fetch_vars:
var.persistable=True
test_fetch_list.append(var.name)
train_exe = exe
params = models.__dict__[args.model]().params
train_speed_list = []
acc1_logs = []
acc5_logs = []
for pass_id in range(params["num_epochs"]):
train_info = [[], [], []]
test_info = [[], [], []]
train_begin=time.time()
batch_id = 0
time_record=[]
if not args.use_dali:
train_iter = train_data_loader()
for data in train_iter:
t1 = time.time()
if batch_id % args.fetch_steps != 0:
train_exe.run(train_prog, feed=data)
else:
if use_mixup:
loss, lr = train_exe.run(train_prog, feed=data, fetch_list=train_fetch_list)
else:
loss, acc1, acc5, lr = train_exe.run(train_prog, feed=data, fetch_list=train_fetch_list)
acc1 = np.mean(np.array(acc1))
acc5 = np.mean(np.array(acc5))
train_info[1].append(acc1)
train_info[2].append(acc5)
t2 = time.time()
period = t2 - t1
time_record.append(period)
if args.profile and batch_id == 100:
print("begin profiler")
if trainer_id == 0:
profiler.start_profiler("All")
elif args.profile and batch_id == 105:
print("begin to end profiler")
if trainer_id == 0:
profiler.stop_profiler("total", "./profile_pass_%d" % (pass_id))
print("end profiler break!")
args.profile=False
if batch_id % args.fetch_steps == 0:
loss = np.mean(np.array(loss))
train_info[0].append(loss)
lr = np.mean(np.array(lr))
period = np.mean(time_record)
speed = args.batch_size * 1.0 / period
time_record=[]
if use_mixup:
print("Pass {0}, trainbatch {1}, loss {2}, lr {3}, time {4}, speed {5}"
.format(pass_id, batch_id, "%.5f"%loss, "%.5f" %lr, "%2.4f sec" % period, "%.2f" % speed))
else:
print("Pass {0}, trainbatch {1}, loss {2}, \
acc1 {3}, acc5 {4}, lr {5}, time {6}, speed {7}"
.format(pass_id, batch_id, "%.5f"%loss, "%.5f"%acc1, "%.5f"%acc5, "%.5f" %
lr, "%2.4f sec" % period, "%.2f" % speed))
sys.stdout.flush()
batch_id += 1
if args.use_dali:
train_iter.reset()
train_loss = np.array(train_info[0]).mean()
if not use_mixup:
train_acc1 = np.array(train_info[1]).mean()
train_acc5 = np.array(train_info[2]).mean()
train_end=time.time()
train_speed = (batch_id * train_batch_size) / (train_end - train_begin)
train_speed_list.append(train_speed)
if trainer_id == 0 and (args.do_test or (pass_id + 1) == params["num_epochs"]):
if args.use_dali:
test_iter = dali.val(settings=args, trainer_id=trainer_id, trainers_num=num_trainers,
gpu_id=gpu_id, data_layout=args.data_format)
else:
test_iter = test_data_loader()
test_batch_id = 0
for data in test_iter:
t1 = time.time()
loss, acc1, acc5 = exe.run(program=test_prog,
feed=data,
fetch_list=test_fetch_list)
t2 = time.time()
period = t2 - t1
loss = np.mean(loss)
acc1 = np.mean(acc1)
acc5 = np.mean(acc5)
test_info[0].append(loss)
test_info[1].append(acc1)
test_info[2].append(acc5)
if test_batch_id % 10 == 0:
test_speed = test_batch_size * 1.0 / period
print("Pass {0},testbatch {1},loss {2}, \
acc1 {3},acc5 {4},time {5},speed {6}"
.format(pass_id, test_batch_id, "%.5f"%loss,"%.5f"%acc1, "%.5f"%acc5,
"%2.2f sec" % period, "%.2f" % test_speed))
sys.stdout.flush()
test_batch_id += 1
if args.use_dali:
test_iter.reset()
del test_iter
test_loss = np.array(test_info[0]).mean()
test_acc1 = np.array(test_info[1]).mean()
test_acc5 = np.array(test_info[2]).mean()
acc1_logs.append(test_acc1)
acc5_logs.append(test_acc5)
if use_mixup:
print("End pass {0}, train_loss {1}, test_loss {2}, test_acc1 {3}, test_acc5 {4}, speed {5}".format(
pass_id, "%.5f"%train_loss, "%.5f"%test_loss, "%.5f"%test_acc1, "%.5f"%test_acc5,
"%.2f" % train_speed))
else:
print("End pass {0}, train_loss {1}, train_acc1 {2}, train_acc5 {3}, "
"test_loss {4}, test_acc1 {5}, test_acc5 {6}, speed {7}".format(
pass_id, "%.5f"%train_loss, "%.5f"%train_acc1, "%.5f"%train_acc5, "%.5f"%test_loss,
"%.5f"%test_acc1, "%.5f"%test_acc5, "%.2f" % train_speed))
else:
if use_mixup:
print("End pass {0}, train_loss {1}, speed {2}".format(pass_id, "%.5f"%train_loss, "%.2f" % train_speed))
else:
print("End pass {0}, train_loss {1}, train_acc1 {2}, train_acc5 {3}, ""speed {4}".format(
pass_id, "%.5f"%train_loss, "%.5f"%train_acc1, "%.5f"%train_acc5, "%.2f" % train_speed))
sys.stdout.flush()
# save in last epoch
if trainer_id == 0:
model_path = os.path.join(model_save_dir + '/' + model_name, str(pass_id))
if not os.path.isdir(model_path):
os.makedirs(model_path)
fluid.io.save_persistables(exe, model_path, main_program=fleet._origin_program)
if args.benchmark_test:
if not os.path.isdir("./benchmark_logs/"):
os.makedirs("./benchmark_logs/")
with open("./benchmark_logs/log_%d" % trainer_id, 'w') as f:
result = dict()
result['0'] = dict()
result['0']['acc1'] = test_acc1
result['0']['acc5'] = test_acc5
result['0']['result_log'] = dict()
result['0']['result_log']['acc1'] = acc1_logs
result['0']['result_log']['acc5'] = acc5_logs
# maximum speed of all epochs
result['1'] = max(train_speed_list) * num_trainers
result['14'] = args.batch_size
print(str(result))
f.writelines(str(result))