def build_optimizer(self, avg_cost, strategy): use_grad_clip = int(os.getenv('GRAD_CLIP', 0)) if use_grad_clip: # 1: clip_by_value; 2: clip_by_norm; 3:clip_by_global_norm if use_grad_clip == 1: fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByValue(2.0)) elif use_grad_clip == 2: fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByNorm(2.0)) elif use_grad_clip == 3: fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByGlobalNorm(2.0)) use_decay = int(os.getenv("DECAY", "0")) if use_decay: optimizer = fluid.optimizer.SGD( learning_rate=fluid.layers.exponential_decay( learning_rate=LEARNING_RATE, decay_steps=500, decay_rate=0.969, staircase=True)) else: optimizer = fluid.optimizer.SGD(LEARNING_RATE) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost)
def test_server_init(self): scope, train_program, startup_program, loss = self.net() with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.Ftrl(1e-3) optimizer = fleet.distributed_optimizer( optimizer, self.strategy) optimizer.minimize(loss) fleet.init_server()
def test_2ps_0_load(self): # init No.0 server env env = {} env["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" env["PADDLE_TRAINERS_NUM"] = str(2) env["TRAINING_ROLE"] = "PSERVER" env["PADDLE_PORT"] = "4001" env["POD_IP"] = "127.0.0.1" for k, v in env.items(): os.environ[k] = str(v) """ array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ], [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], [0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2], [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3], [0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4], [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5], [0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6], [0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7], [0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], [0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9]]) """ emb_array = np.arange(0, 1, 0.1).repeat(10).reshape(10, 10) fc_array = np.arange(0, 1, 0.1).repeat(10).reshape(10, 10) model_path = self.save_origin_model(emb_array, fc_array) role = role_maker.PaddleCloudRoleMaker() fleet.init(role) loss = self.net(emb_array, fc_array) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.a_sync = True optimizer = fluid.optimizer.Adam(1e-3) optimizer = fleet.distributed_optimizer(optimizer, strategy) optimizer.minimize(loss) fleet.init_server(model_path) fc_w = np.array(fluid.global_scope().find_var("fc").get_tensor()) emb = np.array( fluid.global_scope().find_var("embedding.block0").get_tensor()) assert fc_w.all() == fc_array.all() assert emb.all() == emb_array[::2].all() shutil.rmtree(model_path)
def build_optimizer(self, avg_cost, strategy): optimizer = fluid.optimizer.SGD(LEARNING_RATE) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost)