def startup(self, context): """R """ if fleet.is_server(): fleet.run_server() context['status'] = 'wait' return stdout_str = "" self._train_pass = util.TimeTrainPass(self.global_config) if not self.global_config['cold_start']: cost_printer = util.CostPrinter( util.print_cost, { 'master': True, 'log_format': 'load model cost %s sec', 'stdout': stdout_str }) self.print_log( "going to load model %s" % self._train_pass._checkpoint_model_path, {'master': True}) # if config.need_reqi_changeslot and config.reqi_dnn_plugin_day >= self._train_pass.date() # and config.reqi_dnn_plugin_pass >= self._pass_id: # fleet.load_one_table(0, self._train_pass._checkpoint_model_path) # else: fleet.init_server(self._train_pass._checkpoint_model_path, mode=0) cost_printer.done() if self.global_config['save_first_base']: self.print_log("save_first_base=True", {'master': True}) self.print_log("going to save xbox base model", { 'master': True, 'stdout': stdout_str }) self._train_pass._base_key = int(time.time()) stdout_str += self.save_xbox_model(self._train_pass.date(), 0, self._train_pass._base_key, "") context['status'] = 'begin_day'
def processor_register(self): role = MPISymetricRoleMaker() fleet.init(role) if fleet.is_server(): self.regist_context_processor('uninit', self.instance) self.regist_context_processor('init_pass', self.init) self.regist_context_processor('server_pass', self.server) else: self.regist_context_processor('uninit', self.instance) self.regist_context_processor('init_pass', self.init) self.regist_context_processor('train_pass', self.train) self.regist_context_processor('terminal_pass', self.terminal)
def init(self, context): """R """ self.model.train_net() optimizer = self.model.optimizer() optimizer = fleet.distributed_optimizer(optimizer, strategy={"use_cvm": False}) optimizer.minimize(self.model.get_avg_cost()) if fleet.is_server(): context['status'] = 'server_pass' else: self.fetch_vars = [] self.fetch_alias = [] self.fetch_period = self.model.get_fetch_period() metrics = self.model.get_metrics() if metrics: self.fetch_vars = metrics.values() self.fetch_alias = metrics.keys() context['status'] = 'train_pass'
"join_5.w_0", "join_5.b_0", "join_6.w_0", "join_6.b_0", "join_7.w_0", "join_7.b_0" ] common_save_params = [ "common.batch_size", "common.batch_sum", "common.batch_square_sum", "common_0.w_0", "common_0.b_0", "common_1.w_0", "common_1.b_0", "common_2.w_0", "common_2.b_0", "common_3.w_0", "common_3.b_0", "common_4.w_0", "common_4.b_0", "common_5.w_0", "common_5.b_0", "common_6.w_0", "common_6.b_0", "common_7.w_0", "common_7.b_0" ] update_save_params = [ "fc_0.w_0", "fc_0.b_0", "fc_1.w_0", "fc_1.b_0", "fc_2.w_0", "fc_2.b_0", "fc_3.w_0", "fc_3.b_0", "fc_4.w_0", "fc_4.b_0", "fc_5.w_0", "fc_5.b_0" ] if fleet.is_server(): fleet.run_server() elif fleet.is_worker(): with fluid.scope_guard(scope3): exe.run(update_model._startup_program) with fluid.scope_guard(scope2): exe.run(join_common_model._startup_program) configs = { "fs.default.name": config.fs_name, "hadoop.job.ugi": config.fs_ugi } hdfs_client = HDFSClient("$HADOOP_HOME", configs) save_first_base = config.save_first_base path = config.train_data_path
def init(self, context): """R """ role_maker = None if self.global_config.get('process_mode', 'mpi') == 'brilliant_cpu': afs_config = self.global_config['io']['afs'] role_maker = GeneralRoleMaker( hdfs_name=afs_config['fs_name'], hdfs_ugi=afs_config['fs_ugi'], path=self.global_config['output_path'] + "/gloo", init_timeout_seconds=1200, run_timeout_seconds=1200) fleet.init(role_maker) data_var_list = [] data_var_name_dict = {} runnnable_scope = [] runnnable_cost_op = [] context['status'] = 'startup' for executor in self.global_config['executor']: scope = fluid.Scope() self._exector_context[executor['name']] = {} self._exector_context[executor['name']]['scope'] = scope self._exector_context[ executor['name']]['model'] = model_basic.create(executor) model = self._exector_context[executor['name']]['model'] self._metrics.update(model.get_metrics()) runnnable_scope.append(scope) runnnable_cost_op.append(model.get_cost_op()) for var in model._data_var: if var.name in data_var_name_dict: continue data_var_list.append(var) data_var_name_dict[var.name] = var optimizer = model_basic.YamlModel.build_optimizer({ 'metrics': self._metrics, 'optimizer_conf': self.global_config['optimizer'] }) optimizer.minimize(runnnable_cost_op, runnnable_scope) for executor in self.global_config['executor']: scope = self._exector_context[executor['name']]['scope'] model = self._exector_context[executor['name']]['model'] program = model._build_param['model']['train_program'] if not executor['is_update_sparse']: program._fleet_opt["program_configs"][str( id(model.get_cost_op().block.program) )]["push_sparse"] = [] if 'train_thread_num' not in executor: executor['train_thread_num'] = self.global_config[ 'train_thread_num'] with fluid.scope_guard(scope): self._exe.run(model._build_param['model']['startup_program']) model.dump_model_program('./') # server init done if fleet.is_server(): return 0 self._dataset = {} for dataset_item in self.global_config['dataset']['data_list']: dataset_item['data_vars'] = data_var_list dataset_item.update(self.global_config['io']['afs']) dataset_item["batch_size"] = self.global_config['batch_size'] self._dataset[dataset_item[ 'name']] = dataset.FluidTimeSplitDataset(dataset_item) # if config.need_reqi_changeslot and config.reqi_dnn_plugin_day >= last_day and config.reqi_dnn_plugin_pass >= last_pass: # util.reqi_changeslot(config.hdfs_dnn_plugin_path, join_save_params, common_save_params, update_save_params, scope2, scope3) fleet.init_worker() pass
def do_training(self, args=None): """do training""" avg_cost = self.net() place = fluid.CPUPlace() exe = fluid.Executor(place) fleet.init() # optimizer optimizer = fluid.optimizer.Adam(learning_rate=0.001) # 加入 fleet distributed_optimizer 加入分布式策略配置及多机优化 optimizer = fleet.distributed_optimizer(optimizer, strategy={"fleet_desc_file": "./thirdparty/pslib/fleet_desc.prototxt"}) optimizer.minimize(avg_cost) train_info = [] # 启动server if fleet.is_server(): fleet.init_server() fleet.run_server() # 启动worker if fleet.is_worker(): train_data_path = 'thirdparty/data/dist_data/pslib/train_data' train_data_files = [] for filename in os.listdir(train_data_path): train_data_files.append(os.path.join(train_data_path, filename)) # fleet dataset label = fluid.layers.data(name="click", shape=[-1, 1], dtype="int64", lod_level=1, append_batch_size=False) data = fluid.layers.data(name="1", shape=[1], dtype="int64", lod_level=1) dataset = fluid.DatasetFactory().create_dataset("InMemoryDataset") dataset.set_use_var([label, data]) dataset.set_pipe_command("./python/bin/python ./thirdparty/pslib/dataset_generator.py") dataset.set_batch_size(32) dataset.set_thread(3) dataset.set_filelist(train_data_files) # 把数据读到内存 dataset.load_into_memory() # 本地shuffle dataset.local_shuffle() # 初始化worker配置 fleet.init_worker() exe.run(fluid.default_startup_program()) PASS_NUM = 1 for pass_id in range(PASS_NUM): var_dict = {"loss": avg_cost} global var_dict class FetchVars(fluid.executor.FetchHandler): def __init__(self, var_dict=None, period_secs=2): super(FetchVars, self).__init__(var_dict, period_secs=2) def handler(self, res_dict): train_info.extend(res_dict["loss"]) print(train_info) exe.train_from_dataset( program=fluid.default_main_program(), dataset=dataset, fetch_handler=FetchVars(var_dict)) dataset.release_memory() fleet.shrink_sparse_table() fleet.shrink_dense_table(0.01, 11) fleet.print_table_stat(0) fleet.clear_one_table(0) fleet.clear_model() fleet.stop_worker() return train_info