def test_single_run_ps_minimize(self):
input_x = paddle.static.data(name="x", shape=[-1, 32], dtype='float32')
input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64')
fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh')
prediction = fluid.layers.fc(input=fc_1, size=2, act='softmax')
cost = fluid.layers.cross_entropy(input=prediction, label=input_y)
avg_cost = paddle.mean(x=cost)
fleet.init()
strategy = paddle.distributed.fleet.DistributedStrategy()
optimizer = fluid.optimizer.SGD(learning_rate=0.01)
optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
optimizer.minimize(avg_cost)
if fleet.is_server():
fleet.init_server()
fleet.run_server()
elif fleet.is_worker():
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(paddle.static.default_startup_program())
step = 100
for i in range(step):
cost_val = exe.run(program=fluid.default_main_program(),
feed=self.gen_data(),
fetch_list=[avg_cost.name])
print("worker_index: %d, step%d cost = %f" %
(fleet.worker_index(), i, cost_val[0]))
fleet.save_persistables(exe, "fleet_single_model/")
print("save fleet models done.")
def do_dataset_training(self, fleet):
train_file_list = ctr_dataset_reader.prepare_fake_data()
exe = self.get_executor()
exe.run(fluid.default_startup_program())
fleet.init_worker()
thread_num = 2
batch_size = 128
filelist = train_file_list
# config dataset
dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset")
dataset.set_use_var(self.feeds)
dataset.set_batch_size(128)
dataset.set_thread(2)
dataset.set_filelist(filelist)
dataset.set_pipe_command('python ctr_dataset_reader.py')
dataset.load_into_memory()
dataset.global_shuffle(fleet, 12) ##TODO: thread configure
shuffle_data_size = dataset.get_shuffle_data_size(fleet)
local_data_size = dataset.get_shuffle_data_size()
data_size_list = fleet.util.all_gather(local_data_size)
print('after global_shuffle data_size_list: ', data_size_list)
print('after global_shuffle data_size: ', shuffle_data_size)
for epoch_id in range(1):
pass_start = time.time()
exe.train_from_dataset(program=fluid.default_main_program(),
dataset=dataset,
fetch_list=[self.avg_cost],
fetch_info=["cost"],
print_period=2,
debug=int(os.getenv("Debug", "0")))
pass_time = time.time() - pass_start
dataset.release_memory()
if os.getenv("SAVE_MODEL") == "1":
model_dir = tempfile.mkdtemp()
fleet.save_inference_model(exe, model_dir,
[feed.name for feed in self.feeds],
self.avg_cost)
self.check_model_right(model_dir)
shutil.rmtree(model_dir)
dirname = os.getenv("SAVE_DIRNAME", None)
if dirname:
fleet.save_persistables(exe, dirname=dirname)
cache_dirname = os.getenv("SAVE_CACHE_DIRNAME", None)
if cache_dirname:
fleet.save_cache_model(cache_dirname)
def save_batch_model(exe, output_path, day):
"""
save batch model
Args:
output_path(str): output path
day(str|int): training day
"""
day = str(day)
suffix_name = "/%s/0/" % day
model_path = output_path + suffix_name
logger.info("going to save_batch_model %s" % model_path)
fleet.save_persistables(exe, model_path, mode=3)
def main(args):
paddle.set_device("cpu")
paddle.enable_static()
fleet.init()
fake_num_nodes = 1
py_reader, loss = StaticSkipGramModel(
fake_num_nodes,
args.neg_num,
args.embed_size,
sparse_embedding=True,
shared_embedding=args.shared_embedding)
optimizer = F.optimizer.Adam(args.learning_rate, lazy_mode=True)
dist_strategy = fleet.DistributedStrategy()
dist_strategy.a_sync = True
optimizer = fleet.distributed_optimizer(optimizer, dist_strategy)
optimizer.minimize(loss)
# init and run server or worker
if fleet.is_server():
fleet.init_server()
fleet.run_server()
if fleet.is_worker():
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
fleet.init_worker()
graph = build_graph(args)
# bind gen
train_ds = ShardedDataset(graph.nodes, args.epoch)
collate_fn = BatchRandWalk(graph, args.walk_len, args.win_size,
args.neg_num, args.neg_sample_type)
data_loader = Dataloader(train_ds,
batch_size=args.cpu_batch_size,
shuffle=True,
num_workers=args.sample_workers,
collate_fn=collate_fn)
py_reader.set_batch_generator(lambda: data_loader)
train_loss = train(exe, paddle.static.default_main_program(),
py_reader, loss)
fleet.stop_worker()
if fleet.is_first_worker():
fleet.save_persistables(exe, "./model",
paddle.static.default_main_program())
def save_model(exe, output_path, day, pass_id, mode=0):
"""
Args:
output_path(str): output path
day(str|int): training day
pass_id(str|int): training pass id
"""
day = str(day)
pass_id = str(pass_id)
suffix_name = "/%s/%s/" % (day, pass_id)
model_path = output_path + suffix_name
logger.info("going to save_model %s" % model_path)
fleet.save_persistables(exe, model_path, None, mode=mode)
def main(args):
paddle.enable_static()
paddle.set_device('gpu:%d' % paddle.distributed.ParallelEnv().dev_id)
fleet.init(is_collective=True)
graph = load(args.dataset)
loss = StaticSkipGramModel(graph.num_nodes,
args.neg_num,
args.embed_size,
num_emb_part=args.num_emb_part,
shared_embedding=args.shared_embedding)
optimizer = F.optimizer.Adam(args.learning_rate)
dist_strategy = fleet.DistributedStrategy()
dist_strategy.sharding = True
dist_strategy.sharding_configs = {
"segment_anchors": None,
"sharding_segment_strategy": "segment_broadcast_MB",
"segment_broadcast_MB": 32,
"sharding_degree": int(paddle.distributed.get_world_size()),
}
optimizer = fleet.distributed_optimizer(optimizer, dist_strategy)
optimizer.minimize(loss)
place = paddle.CUDAPlace(paddle.distributed.ParallelEnv().dev_id)
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
# bind gen
train_ds = ShardedDataset(graph.nodes)
collate_fn = BatchRandWalk(graph, args.walk_len, args.win_size,
args.neg_num, args.neg_sample_type)
data_loader = Dataloader(train_ds,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.sample_workers,
collate_fn=collate_fn)
for epoch in range(args.epoch):
train_loss = train(exe, paddle.static.default_main_program(),
data_loader, loss)
log.info("Runing epoch:%s\t train_loss:%.6f", epoch, train_loss)
fleet.stop_worker()
if fleet.is_first_worker():
fleet.save_persistables(exe, "./model",
paddle.static.default_main_program())
def do_pyreader_training(self, fleet):
"""
do training using dataset, using fetch handler to catch variable
Args:
fleet(Fleet api): the fleet object of Parameter Server, define distribute training role
"""
exe = self.get_executor()
exe.run(fluid.default_startup_program())
fleet.init_worker()
batch_size = 4
train_reader = paddle.batch(fake_ctr_reader(), batch_size=batch_size)
self.reader.decorate_sample_list_generator(train_reader)
for epoch_id in range(1):
self.reader.start()
try:
pass_start = time.time()
while True:
loss_val = exe.run(program=fluid.default_main_program(),
fetch_list=[self.avg_cost.name])
loss_val = np.mean(loss_val)
# TODO(randomly fail)
# reduce_output = fleet.util.all_reduce(
# np.array(loss_val), mode="sum")
# loss_all_trainer = fleet.util.all_gather(float(loss_val))
# loss_val = float(reduce_output) / len(loss_all_trainer)
message = "TRAIN ---> pass: {} loss: {}\n".format(
epoch_id, loss_val)
fleet.util.print_on_rank(message, 0)
pass_time = time.time() - pass_start
except fluid.core.EOFException:
self.reader.reset()
dirname = os.getenv("SAVE_DIRNAME", None)
if dirname:
fleet.save_persistables(exe, dirname=dirname)
model_dir = tempfile.mkdtemp()
fleet.save_inference_model(exe, model_dir,
[feed.name for feed in self.feeds],
self.avg_cost)
self.check_model_right(model_dir)
shutil.rmtree(model_dir)
def do_dataset_training_queuedataset(self, fleet):
train_file_list = ctr_dataset_reader.prepare_fake_data()
exe = self.get_executor()
exe.run(fluid.default_startup_program())
fleet.init_worker()
thread_num = 2
batch_size = 128
filelist = train_file_list
# config dataset
dataset = paddle.distributed.QueueDataset()
pipe_command = 'python ctr_dataset_reader.py'
dataset.init(batch_size=batch_size,
use_var=self.feeds,
pipe_command=pipe_command,
thread_num=thread_num)
dataset.set_filelist(filelist)
for epoch_id in range(1):
pass_start = time.time()
dataset.set_filelist(filelist)
exe.train_from_dataset(program=fluid.default_main_program(),
dataset=dataset,
fetch_list=[self.avg_cost],
fetch_info=["cost"],
print_period=2,
debug=int(os.getenv("Debug", "0")))
pass_time = time.time() - pass_start
if os.getenv("SAVE_MODEL") == "1":
model_dir = tempfile.mkdtemp()
fleet.save_inference_model(exe, model_dir,
[feed.name for feed in self.feeds],
self.avg_cost)
self.check_model_right(model_dir)
shutil.rmtree(model_dir)
dirname = os.getenv("SAVE_DIRNAME", None)
if dirname:
fleet.save_persistables(exe, dirname=dirname)
def train_prog(exe, program, loss, node2vec_pyreader, args, train_steps):
step = 0
node2vec_pyreader.start()
while True:
try:
begin_time = time.time()
loss_val, = exe.run(program, fetch_list=[loss])
log.info("step %s: loss %.5f speed: %.5f s/step" %
(step, np.mean(loss_val), time.time() - begin_time))
step += 1
except F.core.EOFException:
node2vec_pyreader.reset()
if step % args.steps_per_save == 0 or step == train_steps:
if fleet.is_first_worker():
model_save_dir = args.save_path
model_path = os.path.join(model_save_dir, str(step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
fleet.save_persistables(exe, model_path)
if step == train_steps:
break
def main(args):
paddle.set_device("cpu")
paddle.enable_static()
fleet.init()
if args.num_nodes is None:
num_nodes = load(args.dataset).num_nodes
else:
num_nodes = args.num_nodes
loss = StaticSkipGramModel(num_nodes,
args.neg_num,
args.embed_size,
sparse=True)
optimizer = F.optimizer.Adam(args.learning_rate, lazy_mode=True)
dist_strategy = fleet.DistributedStrategy()
dist_strategy.a_sync = True
optimizer = fleet.distributed_optimizer(optimizer, dist_strategy)
optimizer.minimize(loss)
# init and run server or worker
if fleet.is_server():
fleet.init_server()
fleet.run_server()
if fleet.is_worker():
place = paddle.CPUPlace()
exe = paddle.static.Executor(place)
exe.run(paddle.static.default_startup_program())
fleet.init_worker()
graph = load(args.dataset)
# bind gen
train_ds = ShardedDataset(graph.nodes)
collate_fn = BatchRandWalk(graph, args.walk_len, args.win_size,
args.neg_num, args.neg_sample_type)
data_loader = Dataloader(train_ds,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.sample_workers,
collate_fn=collate_fn)
cpu_num = int(os.environ.get('CPU_NUM', 1))
if int(cpu_num) > 1:
parallel_places = [paddle.CPUPlace()] * cpu_num
exec_strategy = paddle.static.ExecutionStrategy()
exec_strategy.num_threads = int(cpu_num)
build_strategy = paddle.static.BuildStrategy()
build_strategy.reduce_strategy = paddle.static.BuildStrategy.ReduceStrategy.Reduce
compiled_prog = paddle.static.CompiledProgram(
paddle.static.default_main_program()).with_data_parallel(
loss_name=loss.name,
places=parallel_places,
build_strategy=build_strategy,
exec_strategy=exec_strategy)
else:
compiled_prog = paddle.static.default_main_program()
for epoch in range(args.epoch):
train_loss = train(exe, compiled_prog, data_loader, loss)
log.info("Runing epoch:%s\t train_loss:%.6f", epoch, train_loss)
fleet.stop_worker()
if fleet.is_first_worker():
fleet.save_persistables(exe, "./model",
paddle.static.default_main_program())