def build_optimizer(self, avg_cost, strategy):
use_grad_clip = int(os.getenv('GRAD_CLIP', 0))
if use_grad_clip:
# 1: clip_by_value; 2: clip_by_norm; 3:clip_by_global_norm
if use_grad_clip == 1:
fluid.clip.set_gradient_clip(
clip=fluid.clip.GradientClipByValue(2.0))
elif use_grad_clip == 2:
fluid.clip.set_gradient_clip(
clip=fluid.clip.GradientClipByNorm(2.0))
elif use_grad_clip == 3:
fluid.clip.set_gradient_clip(
clip=fluid.clip.GradientClipByGlobalNorm(2.0))
use_decay = int(os.getenv("DECAY", "0"))
if use_decay:
optimizer = fluid.optimizer.SGD(
learning_rate=fluid.layers.exponential_decay(
learning_rate=LEARNING_RATE,
decay_steps=500,
decay_rate=0.969,
staircase=True))
else:
optimizer = fluid.optimizer.SGD(LEARNING_RATE)
optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
optimizer.minimize(avg_cost)
def test_server_init(self):
scope, train_program, startup_program, loss = self.net()
with fluid.scope_guard(scope):
with fluid.program_guard(train_program, startup_program):
optimizer = fluid.optimizer.Ftrl(1e-3)
optimizer = fleet.distributed_optimizer(
optimizer, self.strategy)
optimizer.minimize(loss)
fleet.init_server()
def test_2ps_0_load(self):
# init No.0 server env
env = {}
env["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002"
env["PADDLE_TRAINERS_NUM"] = str(2)
env["TRAINING_ROLE"] = "PSERVER"
env["PADDLE_PORT"] = "4001"
env["POD_IP"] = "127.0.0.1"
for k, v in env.items():
os.environ[k] = str(v)
"""
array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],
[0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1],
[0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2],
[0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3],
[0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4],
[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6],
[0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7],
[0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8],
[0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9]])
"""
emb_array = np.arange(0, 1, 0.1).repeat(10).reshape(10, 10)
fc_array = np.arange(0, 1, 0.1).repeat(10).reshape(10, 10)
model_path = self.save_origin_model(emb_array, fc_array)
role = role_maker.PaddleCloudRoleMaker()
fleet.init(role)
loss = self.net(emb_array, fc_array)
strategy = paddle.distributed.fleet.DistributedStrategy()
strategy.a_sync = True
optimizer = fluid.optimizer.Adam(1e-3)
optimizer = fleet.distributed_optimizer(optimizer, strategy)
optimizer.minimize(loss)
fleet.init_server(model_path)
fc_w = np.array(fluid.global_scope().find_var("fc").get_tensor())
emb = np.array(
fluid.global_scope().find_var("embedding.block0").get_tensor())
assert fc_w.all() == fc_array.all()
assert emb.all() == emb_array[::2].all()
shutil.rmtree(model_path)
def build_optimizer(self, avg_cost, strategy):
optimizer = fluid.optimizer.SGD(LEARNING_RATE)
optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
optimizer.minimize(avg_cost)