def distribute_train(args):
# 根据环境变量确定当前机器/进程在分布式训练中扮演的角色
# 然后使用 fleet api的 init()方法初始化这个节点
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
# 我们还可以进一步指定分布式的运行模式，通过 DistributeTranspilerConfig进行配置
# 如下，我们设置分布式运行模式为异步(async)，同时将参数进行切分，以分配到不同的节点
strategy = DistributeTranspilerConfig()
strategy.sync_mode = False
strategy.runtime_split_send_recv = True
ctr_model = CTR()
inputs = ctr_model.input_data(args)
avg_cost, auc_var = ctr_model.net(inputs, args)
# 配置分布式的optimizer，传入我们指定的strategy，构建program
optimizer = fluid.optimizer.Adam(args.learning_rate)
optimizer = fleet.distributed_optimizer(optimizer, strategy)
optimizer.minimize(avg_cost)
# 根据节点角色，分别运行不同的逻辑
if fleet.is_server():
# 初始化及运行参数服务器节点
fleet.init_server()
fleet.run_server()
elif fleet.is_worker():
# 初始化工作节点
fleet.init_worker()
exe = fluid.Executor(fluid.CPUPlace())
# 初始化含有分布式流程的fleet.startup_program
exe.run(fleet.startup_program)
dataset, file_list = get_dataset(inputs, args)
for epoch in range(args.epochs):
# 以文件为粒度进行shuffle
random.shuffle(file_list)
dataset.set_filelist(file_list)
# 训练节点运行的是经过分布式裁剪的fleet.mian_program
start_time = time.time()
exe.train_from_dataset(program=fleet.main_program,
dataset=dataset,
fetch_list=[auc_var],
fetch_info=["Epoch {} auc ".format(epoch)],
print_period=100,
debug=False)
end_time = time.time()
logger.info("epoch %d finished, use time=%d\n" %
((epoch), end_time - start_time))
# 默认使用0号节点保存模型
if args.save_model and fleet.is_first_worker():
model_path = os.path.join(str(args.model_path),
"epoch_" + str(epoch))
fleet.save_persistables(executor=exe, dirname=model_path)
fleet.stop_worker()
logger.info("Distribute Train Success!")
def test_transpile(self):
role = role_maker.UserDefinedRoleMaker(
current_id=0,
role=role_maker.Role.SERVER,
worker_num=2,
server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"])
# for test optimizer without init(role)
fleet.init(role)
batch_size = 128
is_sparse = True
is_distribute = False
strategy = DistributeTranspilerConfig()
strategy.sync_mode = False
strategy.runtime_split_send_recv = True
avg_cost, _, _ = train_network(batch_size, is_distribute, is_sparse)
self.set_program(avg_cost, strategy)
strategy.runtime_split_send_recv = False
self.set_program(avg_cost, strategy)
def test_half_async_strategy(self):
role = role_maker.UserDefinedRoleMaker(
current_id=0,
role=role_maker.Role.WORKER,
worker_num=2,
server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"])
fleet.init(role)
half_async_config = DistributeTranspilerConfig()
half_async_config.sync_mode = False
half_async_config.geo_sgd_mode = False
half_async_config.runtime_split_send_recv = False
optimizer = fluid.optimizer.SGD(0.0001)
optimizer = fleet.distributed_optimizer(optimizer, half_async_config)
def run_trainer(self, args):
"""run trainer"""
from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet
import paddle.fluid as fluid
from paddle.fluid.transpiler.ps_dispatcher import RoundRobin
from paddle.fluid.transpiler.ps_dispatcher import HashName
fluid.default_startup_program().random_seed = 1
fluid.default_main_program().random_seed = 1
if args.role.upper() != "TRAINER":
raise ValueError("args role must be TRAINER")
role = role_maker.UserDefinedRoleMaker(
current_id=args.current_id,
role=role_maker.Role.WORKER,
worker_num=args.trainers,
server_endpoints=args.endpoints.split(","))
fleet.init(role)
strategy = DistributeTranspilerConfig()
strategy.sync_mode = args.run_params["sync_mode"]
strategy.async_mode = args.run_params["async_mode"]
strategy.mode = "pserver"
strategy.slice_var_up = args.run_params['slice_var_up']
strategy.enable_dc_asgd = args.run_params['enable_dc_asgd']
if args.run_params['split_method']:
strategy.split_method = HashName
strategy.split_method = RoundRobin
strategy.wait_port = args.run_params['wait_port']
strategy.runtime_split_send_recv = args.run_params['runtime_split_send_recv']
strategy.use_hierarchical_allreduce = args.run_params['use_hierarchical_allreduce']
# strategy.hierarchical_allreduce_exter_nranks = args.run_params['hierarchical_allreduce_exter_nranks']
# strategy.hierarchical_allreduce_inter_nranks = args.run_params['hierarchical_allreduce_inter_nranks']
strategy.geo_sgd_mode = args.run_params['geo_sgd']
strategy.geo_sgd_need_push_nums = args.run_params['push_nums']
avg_cost = self.net()
optimizer = fluid.optimizer.SGD(LEARNING_RATE)
optimizer = fleet.distributed_optimizer(optimizer, strategy)
optimizer.minimize(avg_cost)
losses = self.do_training(fleet, args)
losses = "" if not losses else losses
print(losses)
def test_communicator_async(self):
role = role_maker.UserDefinedRoleMaker(
current_id=0,
role=role_maker.Role.WORKER,
worker_num=2,
server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"])
fleet.init(role)
avg_cost = self.net()
optimizer = fluid.optimizer.SGD(0.01)
strategy = DistributeTranspilerConfig()
strategy.sync_mode = False
strategy.runtime_split_send_recv = True
strategy.wait_port = False
optimizer = fleet.distributed_optimizer(optimizer, strategy)
optimizer.minimize(avg_cost)
fleet.init_worker()
time.sleep(10)
fleet.stop_worker()
def train(use_cuda, save_dirname, is_local, is_increment):
"""
train
"""
# predict, avg_cost, feed_order, auc_var, auc_batch, auc_states = model()
old_model = None
model_args = model()
predict = model_args['predict']
avg_cost = model_args['avg_cost']
feed_order = model_args['feed_order']
loader = model_args['loader']
auc_batch = model_args['auc'][1]
# 加入 fleet distributed_optimizer 加入分布式策略配置及多机优化
sgd_optimizer = AdamOptimizer(learning_rate=2e-4)
# sgd_optimizer = fluid.optimizer.Adam(learning_rate=2e-5)
if is_local:
sgd_optimizer.minimize(avg_cost)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
readers = []
for i in range(16):
readers.append(data_reader(cluster_train_dir))
multi_readers = paddle.reader.multiprocess_reader(readers)
loader.set_sample_generator(
multi_readers, batch_size=BATCH_SIZE, places=fluid.cpu_places(CPU_NUM))
# data_reader(cluster_train_dir), batch_size=BATCH_SIZE, places=fluid.cpu_places(CPU_NUM))
# feeder = fluid.DataFeeder(feed_order, place)
# train_reader = feeder.decorate_reader(
# paddle.batch(paddle.reader.shuffle(
# data_reader(cluster_train_dir), buf_size=8192), batch_size=BATCH_SIZE),
# multi_devices=False, drop_last=True)
start_program = fluid.default_startup_program()
exe.run(start_program)
main_prog = fluid.default_main_program()
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = CPU_NUM * 2
build_strategy = fluid.BuildStrategy()
build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce # cpu reduce faster
build_strategy.fuse_broadcast_ops = True
# build_strategy.async_mode = True
main_program = fluid.CompiledProgram(main_prog).with_data_parallel(
loss_name=avg_cost.name, exec_strategy=exec_strategy, build_strategy=build_strategy)
#loss_name=avg_cost.name, exec_strategy=exec_strategy, build_strategy=build_strategy, places=fluid.cpu_places(CPU_NUM))
if is_increment: # load model to fine-tune
fluid.io.load_params(exe, old_model, main_program)
for auc_state in model_args['auc'][2]:
set_zero(place, fluid.global_scope(), auc_state.name)
# 并行训练，速度更快
# train_pe = fluid.ParallelExecutor(use_cuda=use_cuda,
# main_program=main_program, loss_name=avg_cost.name,
# exec_strategy=exec_strategy, build_strategy=build_strategy)
cost_list = []
auc_list = []
import time
pass_s_time = time.time()
for pass_id in range(PASS_NUM):
s_time = time.time()
for batch_id, data in enumerate(loader()):
r_time = time.time() - s_time
st_time = time.time()
cost_value, auc_value = exe.run(
program=main_program,
feed=data,
fetch_list=[avg_cost.name, auc_batch.name])
t_time = time.time() - st_time
cost_list.append(np.array(cost_value))
auc_list.append(np.array(auc_value))
if batch_id % 10 == 0 and batch_id != 0:
print "Pass %d, batch %d, cost %s auc %s readtime %f triantime %f" % \
(pass_id, batch_id, np.array(cost_list).mean(),
np.array(auc_list).mean(), r_time, t_time)
cost_list = []
auc_list = []
if batch_id % 1000 == 0:
if save_dirname is not None:
fluid.io.save_inference_model(
save_dirname,
feed_order,
[predict, avg_cost, auc_batch], exe
)
fluid.io.save_persistables(exe, save_dirname)
infer(cluster_test_dir, save_dirname, feed_order)
s_time = time.time()
pass_time = time.time() - pass_s_time
print("Pass train time: %f" % pass_time)
else:
role = role_maker.PaddleCloudRoleMaker()
# 全异步训练
config = DistributeTranspilerConfig()
config.sync_mode = False
config.runtime_split_send_recv = True
# 加入 fleet init 初始化环境
fleet.init(role)
optimizer = fleet.distributed_optimizer(sgd_optimizer, config)
optimizer.minimize(avg_cost)
if fleet.is_server():
fleet.init_server()
fleet.run_server()
# 启动worker
if fleet.is_worker():
# 初始化worker配置
fleet.init_worker()
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
feeder = fluid.DataFeeder(feed_order, place)
train_reader = feeder.decorate_reader(
paddle.batch(paddle.reader.shuffle(
data_reader(cluster_train_dir), buf_size=8192), batch_size=BATCH_SIZE),
multi_devices=False, drop_last=True)
exe.run(fleet.startup_program)
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = CPU_NUM
build_strategy = fluid.BuildStrategy()
build_strategy.async_mode = True
if CPU_NUM > 1:
build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
compiled_prog = fluid.compiler.CompiledProgram(
fleet.main_program).with_data_parallel(
loss_name=avg_cost.name, build_strategy=build_strategy, exec_strategy=exec_strategy)
for pass_id in range(PASS_NUM):
cost_list = []
auc_list = []
import time
s_time = time.time()
for batch_id, data in enumerate(train_reader()):
r_time = time.time() - s_time
cost_value, auc_value = exe.run(
program=compiled_prog, feed=data,
fetch_list=[avg_cost.name, auc_batch.name])
t_time = time.time() - r_time
cost_list.append(np.array(cost_value))
auc_list.append(np.array(auc_value))
if batch_id % 10 == 0 and batch_id != 0:
print "Pass %d, batch %d, cost %s auc %s readtime %f traintime %f" % \
(pass_id, batch_id, np.array(cost_list).mean(),
np.array(auc_list).mean(), r_time, t_time)
cost_list = []
auc_list = []
if batch_id % 1000 == 0 and fleet.is_first_worker():
if save_dirname is not None:
fleet.save_inference_model(
exe,
save_dirname,
feed_order,
[predict, avg_cost, auc_batch]
)
fleet.save_persistables(exe, save_dirname)
infer(cluster_test_dir, save_dirname, feed_order)
s_time = time.time()
fleet.stop_worker()
def train(use_cuda, train_sample_dir, test_sample_dir, old_model, output_model,
is_local, is_increment):
"""
train
"""
# predict, avg_cost, feed_order, auc_var, auc_batch, auc_states = model()
model_args = model()
navi_predict = model_args['predict'][0]
voice_navi_predict = model_args['predict'][1]
speed_navi_predict = model_args['predict'][2]
avg_cost = model_args['avg_cost']
feed_order = model_args['feed_order']
role = role_maker.PaddleCloudRoleMaker()
# 全异步训练
config = DistributeTranspilerConfig()
config.sync_mode = False
config.runtime_split_send_recv = True
sgd_optimizer = AdamOptimizer(learning_rate=2e-4)
if is_local:
sgd_optimizer.minimize(avg_cost)
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
# train_reader = paddle.batch(
# paddle.reader.shuffle(
# streaming_data_reader(), buf_size=8192), batch_size=BATCH_SIZE)
feeder = fluid.DataFeeder(feed_order, place)
train_reader = feeder.decorate_reader(paddle.batch(
paddle.reader.shuffle(streaming_data_reader(), buf_size=8192),
batch_size=BATCH_SIZE),
multi_devices=False,
drop_last=True)
start_program = fluid.default_startup_program()
exe.run(start_program)
main_program = fluid.default_main_program()
if is_increment: # load model to fine-tune
fluid.io.load_params(exe, old_model, main_program)
# for auc_state in model_args['auc'][2]:
# set_zero(place, fluid.global_scope(), auc_state.name)
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = CPU_NUM
main_program.num_threads = CPU_NUM
build_strategy = fluid.BuildStrategy()
build_strategy.async_mode = True
# 并行训练，速度更快
train_pe = fluid.ParallelExecutor(use_cuda=use_cuda,
main_program=main_program,
loss_name=avg_cost.name)
cost_list = []
for pass_id in range(PASS_NUM):
for batch_id, data in enumerate(train_reader()):
cost_value = train_pe.run(feed=data,
fetch_list=[avg_cost.name])
cost_list.append(np.array(cost_value))
if batch_id % 100 == 0 and batch_id != 0:
print "Pass %d, batch %d, cost %s" % \
(pass_id, batch_id, np.array(cost_list).mean())
cost_list = []
if batch_id % 2000 == 0:
if output_model is not None:
fluid.io.save_inference_model(
output_model, feed_order, [
navi_predict, voice_navi_predict,
speed_navi_predict, avg_cost
], exe)
fluid.io.save_persistables(exe, output_model)
infer(test_sample_dir, output_model, feed_order)
else:
# 加入 fleet init 初始化环境
fleet.init(role)
# 加入 fleet distributed_optimizer 加入分布式策略配置及多机优化
optimizer = fleet.distributed_optimizer(sgd_optimizer, config)
optimizer.minimize(avg_cost)
if fleet.is_server():
if is_increment:
fleet.init_server(old_model)
else:
fleet.init_server()
fleet.run_server()
# 启动worker
if fleet.is_worker():
# 初始化worker配置
fleet.init_worker()
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
exe = Executor(place)
# train_reader = paddle.batch(
# paddle.reader.shuffle(
# data_reader(train_sample_dir), buf_size=8192), batch_size=BATCH_SIZE)
feeder = fluid.DataFeeder(feed_order, place)
train_reader = feeder.decorate_reader(paddle.batch(
paddle.reader.shuffle(data_reader(train_sample_dir),
buf_size=8192),
batch_size=BATCH_SIZE),
multi_devices=False,
drop_last=True)
exe.run(fleet.startup_program)
exec_strategy = fluid.ExecutionStrategy()
exec_strategy.num_threads = CPU_NUM
build_strategy = fluid.BuildStrategy()
build_strategy.async_mode = True
if CPU_NUM > 1:
build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
compiled_prog = fluid.compiler.CompiledProgram(
fleet.main_program).with_data_parallel(
loss_name=avg_cost.name,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
cost_list = []
for pass_id in range(PASS_NUM):
for batch_id, data in enumerate(train_reader()):
cost_value = exe.run(program=compiled_prog,
feed=data,
fetch_list=[avg_cost.name])
cost_list.append(np.array(cost_value))
if batch_id % 100 == 0 and batch_id != 0:
print "Pass %d, batch %d, cost %s" % \
(pass_id, batch_id, np.array(cost_list).mean())
cost_list = []
if batch_id % 1000 == 0 and fleet.is_first_worker():
if output_model is not None:
fleet.save_inference_model(
exe, output_model, feed_order, [
navi_predict, voice_navi_predict,
speed_navi_predict, avg_cost
])
fleet.save_persistables(exe, output_model)
infer(test_sample_dir, output_model, feed_order)
fleet.stop_worker()