def dist_transpile(trainer_id):
if trainer_id < 0:
return None, None
# the port of all pservers, needed by both trainer and pserver
port = os.getenv("PADDLE_PSERVER_PORT", "6174")
# comma separated ips of all pservers, needed by trainer and
# pserver
pserver_ips = os.getenv("PADDLE_PSERVER_IPS", "")
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist)
# total number of workers/trainers in the job, needed by
# trainer and pserver
trainers = int(os.getenv("PADDLE_TRAINERS"))
# the IP of the local machine, needed by pserver only
current_endpoint = os.getenv("PADDLE_CURRENT_IP", "") + ":" + port
# the role, should be either PSERVER or TRAINER
training_role = os.getenv("PADDLE_TRAINING_ROLE")
t = distribute_transpiler.DistributeTranspiler()
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
if training_role == "PSERVER":
pserver_program = t.get_pserver_program(current_endpoint)
pserver_startup_program = t.get_startup_program(
current_endpoint, pserver_program)
return pserver_program, pserver_startup_program
elif training_role == "TRAINER":
train_program = t.get_trainer_program()
return train_program, fluid.default_startup_program()
else:
raise ValueError(
'TRAINING_ROLE environment variable must be either TRAINER or PSERVER'
)
def _transpile(self, startup_program, main_program):
"""
Transpile the programs to distributed programs. And add the variables.
"""
worker_endpoints = fleet.worker_endpoints()
trainer_id = fleet.worker_index()
current_endpoint = fleet.worker_endpoints()[trainer_id]
worker_endpoints_env = ','.join(worker_endpoints)
trainers_num = fleet.worker_num()
if self.print_config:
print("worker_endpoints:{} trainers_num:{} current_endpoint:{} \
trainer_id:{}".format(worker_endpoints, trainers_num,
current_endpoint, trainer_id))
# call transpiler
config = dist_transpiler.DistributeTranspilerConfig()
config.mode = self._strategy.mode
config.collective_mode = self._strategy.collective_mode
config.nccl_comm_num = self._strategy.nccl_comm_num
config.use_hierarchical_allreduce = self._strategy.use_hierarchical_allreduce
config.hierarchical_allreduce_inter_nranks = self._strategy.hierarchical_allreduce_inter_nranks
t = dist_transpiler.DistributeTranspiler(config=config)
t.transpile(trainer_id=trainer_id,
trainers=worker_endpoints_env,
startup_program=startup_program,
program=main_program,
current_endpoint=current_endpoint)
def dist_transpile(trainer_id, args, train_prog, startup_prog):
if trainer_id < 0:
return None, None
# the port of all pservers, needed by both trainer and pserver
port = os.getenv("PADDLE_PSERVER_PORT", "6174")
# comma separated ips of all pservers, needed by trainer and
# pserver
pserver_ips = os.getenv("PADDLE_PSERVER_IPS", "")
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist)
# total number of workers/trainers in the job, needed by
# trainer and pserver
trainers = int(os.getenv("PADDLE_TRAINERS"))
# the IP of the local machine, needed by pserver only
current_endpoint = os.getenv("PADDLE_CURRENT_IP", "") + ":" + port
# the role, should be either PSERVER or TRAINER
training_role = os.getenv("PADDLE_TRAINING_ROLE")
config = fluid.DistributeTranspilerConfig()
config.slice_var_up = not args.no_split_var
config.min_block_size = 1048576
t = distribute_transpiler.DistributeTranspiler(config=config)
t.transpile(
trainer_id,
# NOTE: *MUST* use train_prog, for we are using with guard to
# generate different program for train and test.
program=train_prog,
pservers=pserver_endpoints,
trainers=trainers,
sync_mode=not args.async_mode,
startup_program=startup_prog)
if training_role == "PSERVER":
pserver_program = t.get_pserver_program(current_endpoint)
pserver_startup_program = t.get_startup_program(
current_endpoint, pserver_program, startup_program=startup_prog)
return pserver_program, pserver_startup_program
elif training_role == "TRAINER":
train_program = t.get_trainer_program()
return train_program, startup_prog
else:
raise ValueError(
'PADDLE_TRAINING_ROLE environment variable must be either TRAINER or PSERVER'
)
def minimize(self,
loss,
startup_program=None,
parameter_list=None,
no_grad_set=None):
"""
minimize a program through loss
Args:
loss (Variable|Variable List): loss variable or loss variable list to run optimization.
startup_program (Program): startup_program for initializing parameters
in `parameter_list`.
parameter_list (list): list of Variables to update.
no_grad_set (set|None): set of Variables should be ignored.
Returns:
tuple: (optimize_ops, params_grads) which are, list of operators appended;
and list of (param, grad) Variables pair for optimization.
Note that in parameter server mode, a worker will not get anything about optimize_os
Because optmizer algorithms run on pserver side. We will make this usable in pserver
process, but currently the optimization part is written into Fleet(). A user does not
need to care about how to startup a pserver node.
"""
optimize_ops, param_grads = self._optimizer.minimize(
loss, startup_program, parameter_list, no_grad_set)
worker_endpoints = fleet.worker_endpoints()
trainer_id = fleet.worker_index()
current_endpoint = fleet.worker_endpoints()[trainer_id]
startup_program = startup_program if startup_program else \
fluid.framework.default_startup_program
# call transpiler
config = dist_transpiler.DistributeTranspilerConfig()
config.mode = "nccl2"
t = dist_transpiler.DistributeTranspiler(config=config)
t.transpile(
trainer_id,
trainers=','.join(worker_endpoints),
startup_program=startup_program,
current_endpoint=current_endpoint)
return optimize_ops, param_grads
def test(self):
self._check()
trainer_id = self.trainer_id
num_trainers = self.num_trainers
# if the test program is not built, which means that is the first time
# to call the test method, we will first build the test program and
# add ops to broadcast bn-related parameters from trainer 0 to other
# trainers for distributed tests.
if not self.test_initialized:
emb, loss, _, _, _ = self.build_program(False,
self.num_trainers > 1)
emb_name = emb.name
assert self._get_info(emb_name) is None
self._set_info('emb_name', emb.name)
if num_trainers > 1 and self.has_run_train:
self._append_broadcast_ops(self.test_program)
if num_trainers > 1 and not self.has_run_train:
worker_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS")
current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT")
config = dist_transpiler.DistributeTranspilerConfig()
config.mode = "collective"
config.collective_mode = "grad_allreduce"
t = dist_transpiler.DistributeTranspiler(config=config)
t.transpile(trainer_id=trainer_id,
trainers=worker_endpoints,
startup_program=self.startup_program,
program=self.test_program,
current_endpoint=current_endpoint)
else:
emb_name = self._get_info('emb_name')
gpu_id = int(os.getenv("FLAGS_selected_gpus", 0))
place = fluid.CUDAPlace(gpu_id)
exe = fluid.Executor(place)
if not self.has_run_train:
exe.run(self.startup_program)
if not self.test_reader:
test_reader = reader.test
else:
test_reader = self.test_reader
if not self.test_initialized:
test_list, test_name_list = test_reader(self.dataset_dir,
self.val_targets)
assert self._get_info('test_list') is None
assert self._get_info('test_name_list') is None
self._set_info('test_list', test_list)
self._set_info('test_name_list', test_name_list)
else:
test_list = self._get_info('test_list')
test_name_list = self._get_info('test_name_list')
test_program = self.test_program
if not self.has_run_train:
assert self.checkpoint_dir, "No checkpoint found for test."
self.load_checkpoint(executor=exe,
main_program=test_program,
load_for_train=False)
feeder = fluid.DataFeeder(place=place,
feed_list=['image', 'label'],
program=test_program)
fetch_list = [emb_name]
self.test_initialized = True
test_start = time.time()
self._run_test(exe, test_list, test_name_list, feeder, fetch_list)
test_end = time.time()
logger.info("test time: {:.4f}".format(test_end - test_start))
def train(args, data_reader=ctc_reader):
"""OCR CTC training"""
num_classes = None
train_images = None
train_list = None
test_images = None
test_list = None
num_classes = data_reader.num_classes(
) if num_classes is None else num_classes
data_shape = data_reader.data_shape()
# define network
images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32')
label = fluid.layers.data(name='label',
shape=[1],
dtype='int32',
lod_level=1)
sum_cost, error_evaluator, inference_program, model_average = ctc_train_net(
images, label, args, num_classes)
# data reader
train_reader = data_reader.train(args.batch_size,
train_images_dir=train_images,
train_list_file=train_list)
test_reader = data_reader.test(args.batch_size,
test_images_dir=test_images,
test_list_file=test_list)
# prepare environment
place = fluid.CPUPlace()
if args.use_gpu:
place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
# load init model
if args.init_model is not None:
model_dir = args.init_model
model_file_name = None
if not os.path.isdir(args.init_model):
model_dir = os.path.dirname(args.init_model)
model_file_name = os.path.basename(args.init_model)
fluid.io.load_params(exe, dirname=model_dir, filename=model_file_name)
print "Init model from: %s." % args.init_model
fetch_vars = [sum_cost]
fetch_vars.extend([e for e in error_evaluator])
def test_parallel(exe, pass_id, batch_id):
distance_evaluator = fluid.metrics.EditDistance(None)
test_fetch = [v.name for v in error_evaluator]
distance_evaluator.reset()
for idx, data in enumerate(test_reader()):
test_ret = exe.run(test_fetch, feed=get_feeder_data(data, place))
distance_evaluator.update(distances=test_ret[0],
seq_num=np.mean(test_ret[1]))
return distance_evaluator.eval()
def test(exe, pass_id):
distance_evaluator = fluid.metrics.EditDistance(None)
test_fetch = [v.name for v in error_evaluator]
distance_evaluator.reset()
for idx, data in enumerate(test_reader()):
test_ret = exe.run(inference_program,
feed=get_feeder_data(data, place),
fetch_list=test_fetch)
distance_evaluator.update(distances=test_ret[0],
seq_num=np.mean(test_ret[1]))
return distance_evaluator.eval()
def train_parallel(train_exe):
var_names = [var.name for var in fetch_vars]
#test_exe = fluid.ParallelExecutor(
# use_cuda=True, main_program=inference_program, share_vars_from=train_exe)
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
test_exe = fluid.Executor(place)
for pass_id in range(args.pass_num):
batch_id = 1
total_loss = 0.0
total_seq_error = 0.0
# train a pass
num_samples, start_time = 0, time.time()
for idx, data in enumerate(train_reader()):
batch_start_time = time.time()
results = train_exe.run(var_names,
feed=get_feeder_data(data, place))
results = [np.array(result).sum() for result in results]
total_loss += results[0]
total_seq_error += results[1]
# training log
if batch_id % args.log_period == 0:
print(
"Pass[%d]-batch[%d]; Avg Warp-CTC loss: %s; Avg seq err: %s; Speed: %.5f samples/sec"
% (pass_id, batch_id, total_loss /
(batch_id * args.batch_size), total_seq_error /
(batch_id * args.batch_size), len(data) /
(time.time() - batch_start_time)))
batch_id += 1
num_samples += len(data)
print_train_time(start_time, time.time(), num_samples)
# run test
if model_average:
with model_average.apply(test_exe):
#test_ret = test_parallel(test_exe, pass_id, batch_id)
test_ret = test(test_exe, pass_id)
else:
#test_ret = test_parallel(test_exe, pass_id, batch_id)
test_ret = test(test_exe, pass_id)
print("Pass[%d]; Test avg seq error: %s\n" %
(pass_id, test_ret[1]))
if args.local:
place = core.CPUPlace() if args.use_gpu else core.CUDAPlace(0)
startup_exe = fluid.Executor(place)
startup_exe.run(fluid.default_startup_program())
exec_strategy = ExecutionStrategy()
exec_strategy.use_cuda = args.use_gpu
train_exe = fluid.ParallelExecutor(
use_cuda=args.use_gpu,
main_program=fluid.default_main_program(),
loss_name=sum_cost.name,
exec_strategy=exec_strategy)
train_parallel(train_exe)
else:
port = os.getenv("PADDLE_PSERVER_PORT", "6174")
trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0"))
pserver_ips = os.getenv("PADDLE_PSERVER_IPS")
trainers = int(os.getenv("PADDLE_TRAINERS"))
eplist = []
for ip in pserver_ips.split(","):
eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist)
# the IP of the local machine, needed by pserver only
current_endpoint = os.getenv("PADDLE_CURRENT_IP", "") + ":" + port
# the role, should be either PSERVER or TRAINER
training_role = os.getenv("PADDLE_TRAINING_ROLE")
t = distribute_transpiler.DistributeTranspiler()
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
place = core.CUDAPlace(0) if args.use_gpu else core.CPUPlace()
if training_role == "PSERVER":
pserver_program = t.get_pserver_program(current_endpoint)
pserver_startup_program = t.get_startup_program(
current_endpoint, pserver_program)
exe = fluid.Executor(core.CPUPlace())
exe.run(pserver_startup_program)
exe.run(pserver_program)
elif training_role == "TRAINER":
exe.run(fluid.default_startup_program())
trainer_program = t.get_trainer_program()
exec_strategy = ExecutionStrategy()
exec_strategy.use_cuda = args.use_gpu
exec_strategy.num_threads = 1
train_exe = fluid.ParallelExecutor(use_cuda=args.use_gpu,
main_program=trainer_program,
loss_name=sum_cost.name,
exec_strategy=exec_strategy)
train_parallel(train_exe)
else:
raise ValueError(
"env PADDLE_TRAINING_ROLE should be in [PSERVER, TRIANER]")