def _init_hyper_parameters(self): # tree meta hyper parameters self.max_layers = envs.get_global_env("hyper_parameters.max_layers", 4) self.node_nums = envs.get_global_env("hyper_parameters.node_nums", 26) self.leaf_node_nums = envs.get_global_env( "hyper_parameters.leaf_node_nums", 13) self.output_positive = envs.get_global_env( "hyper_parameters.output_positive", True) self.layer_node_num_list = envs.get_global_env( "hyper_parameters.layer_node_num_list", [2, 4, 7, 12]) self.child_nums = envs.get_global_env("hyper_parameters.child_nums", 2) self.tree_layer_path = envs.get_global_env( "hyper_parameters.tree.tree_layer_path", None) # model training hyper parameter self.node_emb_size = envs.get_global_env( "hyper_parameters.node_emb_size", 64) self.input_emb_size = envs.get_global_env( "hyper_parameters.input_emb_size", 768) self.act = envs.get_global_env("hyper_parameters.act", "tanh") self.neg_sampling_list = envs.get_global_env( "hyper_parameters.neg_sampling_list", [1, 2, 3, 4]) # model infer hyper parameter self.topK = envs.get_global_env( "hyper_parameters.topK", 1, ) self.batch_size = envs.get_global_env( "dataset.dataset_infer.batch_size", 1)
def _init_hyper_parameters(self): self.vocab_size = envs.get_global_env("hyper_parameters.vocab_size") self.embed_size = envs.get_global_env("hyper_parameters.embed_size")
def _init_hyper_parameters(self): self.query_encoder = envs.get_global_env( "hyper_parameters.query_encoder") self.title_encoder = envs.get_global_env( "hyper_parameters.title_encoder") self.query_encode_dim = envs.get_global_env( "hyper_parameters.query_encode_dim") self.title_encode_dim = envs.get_global_env( "hyper_parameters.title_encode_dim") self.emb_size = envs.get_global_env( "hyper_parameters.sparse_feature_dim") self.emb_dim = envs.get_global_env("hyper_parameters.embedding_dim") self.emb_shape = [self.emb_size, self.emb_dim] self.hidden_size = envs.get_global_env("hyper_parameters.hidden_size") self.margin = envs.get_global_env("hyper_parameters.margin") self.query_len = envs.get_global_env("hyper_parameters.query_len") self.pos_len = envs.get_global_env("hyper_parameters.pos_len") self.neg_len = envs.get_global_env("hyper_parameters.neg_len")
def _init_hyper_parameters(self): self.emb_path = envs.get_global_env("hyper_parameters.emb_path") self.sentence_left_size = envs.get_global_env( "hyper_parameters.sentence_left_size") self.sentence_right_size = envs.get_global_env( "hyper_parameters.sentence_right_size") self.vocab_size = envs.get_global_env("hyper_parameters.vocab_size") self.emb_size = envs.get_global_env("hyper_parameters.emb_size") self.kernel_num = envs.get_global_env("hyper_parameters.kernel_num") self.hidden_size = envs.get_global_env("hyper_parameters.hidden_size") self.hidden_act = envs.get_global_env("hyper_parameters.hidden_act") self.out_size = envs.get_global_env("hyper_parameters.out_size") self.channels = envs.get_global_env("hyper_parameters.channels") self.conv_filter = envs.get_global_env("hyper_parameters.conv_filter") self.conv_act = envs.get_global_env("hyper_parameters.conv_act") self.pool_size = envs.get_global_env("hyper_parameters.pool_size") self.pool_stride = envs.get_global_env("hyper_parameters.pool_stride") self.pool_type = envs.get_global_env("hyper_parameters.pool_type") self.pool_padding = envs.get_global_env( "hyper_parameters.pool_padding")
def infer_net(self): sparse_feature_dim = envs.get_global_env( "hyper_parameters.sparse_feature_dim", None, self._namespace) sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None, self._namespace) def embedding_layer(input, table_name, initializer_instance=None): emb = fluid.embedding( input=input, size=[sparse_feature_number, sparse_feature_dim], param_attr=table_name) return emb self.analogy_input() all_label = np.arange(sparse_feature_number).reshape( sparse_feature_number).astype('int32') self.all_label = fluid.layers.cast(x=fluid.layers.assign(all_label), dtype='int64') emb_all_label = embedding_layer(self.all_label, "emb") emb_a = embedding_layer(self.analogy_a, "emb") emb_b = embedding_layer(self.analogy_b, "emb") emb_c = embedding_layer(self.analogy_c, "emb") target = fluid.layers.elementwise_add( fluid.layers.elementwise_sub(emb_b, emb_a), emb_c) emb_all_label_l2 = fluid.layers.l2_normalize(x=emb_all_label, axis=1) dist = fluid.layers.matmul(x=target, y=emb_all_label_l2, transpose_y=True) values, pred_idx = fluid.layers.topk(input=dist, k=4) label = fluid.layers.expand(fluid.layers.unsqueeze(self.analogy_d, axes=[1]), expand_times=[1, 4]) label_ones = fluid.layers.fill_constant_batch_size_like( label, shape=[-1, 1], value=1.0, dtype='float32') right_cnt = fluid.layers.reduce_sum(input=fluid.layers.cast( fluid.layers.equal(pred_idx, label), dtype='float32')) total_cnt = fluid.layers.reduce_sum(label_ones) global_right_cnt = fluid.layers.create_global_var( name="global_right_cnt", persistable=True, dtype='float32', shape=[1], value=0) global_total_cnt = fluid.layers.create_global_var( name="global_total_cnt", persistable=True, dtype='float32', shape=[1], value=0) global_right_cnt.stop_gradient = True global_total_cnt.stop_gradient = True tmp1 = fluid.layers.elementwise_add(right_cnt, global_right_cnt) fluid.layers.assign(tmp1, global_right_cnt) tmp2 = fluid.layers.elementwise_add(total_cnt, global_total_cnt) fluid.layers.assign(tmp2, global_total_cnt) acc = fluid.layers.elementwise_div(global_right_cnt, global_total_cnt, name="total_acc") self._infer_results['acc'] = acc
def init(self): self.batch_size = envs.get_global_env( "dataset.dataset_infer.batch_size") self.input = [] self.length = None
def _init_hyper_parameters(self): self.vocab_size = envs.get_global_env("hyper_parameters.vocab_size") self.emb_dim = envs.get_global_env("hyper_parameters.emb_dim") self.hidden_size = envs.get_global_env("hyper_parameters.hidden_size")
def net(self, inputs, is_infer=False): num_users = envs.get_global_env("hyper_parameters.num_users", None, self._namespace) num_items = envs.get_global_env("hyper_parameters.num_items", None, self._namespace) latent_dim = envs.get_global_env("hyper_parameters.latent_dim", None, self._namespace) layers = envs.get_global_env("hyper_parameters.layers", None, self._namespace) num_layer = len(layers) #Number of layers in the MLP MF_Embedding_User = fluid.embedding( input=inputs[0], size=[num_users, latent_dim], param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), is_sparse=True) MF_Embedding_Item = fluid.embedding( input=inputs[1], size=[num_items, latent_dim], param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), is_sparse=True) MLP_Embedding_User = fluid.embedding( input=inputs[0], size=[num_users, int(layers[0] / 2)], param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), is_sparse=True) MLP_Embedding_Item = fluid.embedding( input=inputs[1], size=[num_items, int(layers[0] / 2)], param_attr=fluid.initializer.Normal(loc=0.0, scale=0.01), is_sparse=True) # MF part mf_user_latent = fluid.layers.flatten(x=MF_Embedding_User, axis=1) mf_item_latent = fluid.layers.flatten(x=MF_Embedding_Item, axis=1) mf_vector = fluid.layers.elementwise_mul(mf_user_latent, mf_item_latent) # MLP part # The 0-th layer is the concatenation of embedding layers mlp_user_latent = fluid.layers.flatten(x=MLP_Embedding_User, axis=1) mlp_item_latent = fluid.layers.flatten(x=MLP_Embedding_Item, axis=1) mlp_vector = fluid.layers.concat( input=[mlp_user_latent, mlp_item_latent], axis=-1) for i in range(1, num_layer): mlp_vector = fluid.layers.fc( input=mlp_vector, size=layers[i], act='relu', param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormal( loc=0.0, scale=1.0 / math.sqrt(mlp_vector.shape[1])), regularizer=fluid.regularizer.L2DecayRegularizer( regularization_coeff=1e-4)), name='layer_' + str(i)) # Concatenate MF and MLP parts predict_vector = fluid.layers.concat(input=[mf_vector, mlp_vector], axis=-1) # Final prediction layer prediction = fluid.layers.fc( input=predict_vector, size=1, act='sigmoid', param_attr=fluid.initializer.MSRAInitializer(uniform=True), name='prediction') if is_infer: self._infer_results["prediction"] = prediction return cost = fluid.layers.log_loss(input=prediction, label=fluid.layers.cast(x=inputs[2], dtype='float32')) avg_cost = fluid.layers.mean(cost) self._cost = avg_cost self._metrics["cost"] = avg_cost
def dataloader_by_name(readerclass, dataset_name, yaml_file, context, reader_class_name="Reader"): reader_class = lazy_instance_by_fliename(readerclass, reader_class_name) name = "dataset." + dataset_name + "." data_path = get_global_env(name + "data_path") if data_path.startswith("paddlerec::"): package_base = get_runtime_environ("PACKAGE_BASE") assert package_base is not None data_path = os.path.join(package_base, data_path.split("::")[1]) hidden_file_list, files = check_filelist(hidden_file_list=[], data_file_list=[], train_data_path=data_path) if (hidden_file_list is not None): print( "Warning:please make sure there are no hidden files in the dataset folder and check these hidden files:{}" .format(hidden_file_list)) files.sort() # for local cluster: discard some files if files cannot be divided equally between GPUs if (context["device"] == "GPU") and "PADDLEREC_GPU_NUMS" in os.environ: selected_gpu_nums = int(os.getenv("PADDLEREC_GPU_NUMS")) discard_file_nums = len(files) % selected_gpu_nums if (discard_file_nums != 0): warnings.warn( "Because files cannot be divided equally between GPUs,discard these files:{}" .format(files[-discard_file_nums:])) files = files[:len(files) - discard_file_nums] need_split_files = False if context["engine"] == EngineMode.LOCAL_CLUSTER: # for local cluster: split files for multi process need_split_files = True elif context["engine"] == EngineMode.CLUSTER and context[ "cluster_type"] == "K8S": # for k8s mount mode, split files for every node need_split_files = True print("need_split_files: {}".format(need_split_files)) if need_split_files: files = split_files(files, context["fleet"].worker_index(), context["fleet"].worker_num()) context["file_list"] = files reader = reader_class(yaml_file) reader.init() def gen_reader(): for file in files: with open(file, 'r') as f: for line in f: line = line.rstrip('\n') iter = reader.generate_sample(line) for parsed_line in iter(): if parsed_line is None: continue else: values = [] for pased in parsed_line: values.append(pased[1]) yield values def gen_batch_reader(): return reader.generate_batch_from_trainfiles(files) if hasattr(reader, 'generate_batch_from_trainfiles'): return gen_batch_reader() if hasattr(reader, "batch_tensor_creator"): return reader.batch_tensor_creator(gen_reader) return gen_reader
def _init_hyper_parameters(self): self.num_users = envs.get_global_env("hyper_parameters.num_users") self.num_items = envs.get_global_env("hyper_parameters.num_items") self.latent_dim = envs.get_global_env("hyper_parameters.latent_dim") self.layers = envs.get_global_env("hyper_parameters.fc_layers")
def _executor_dataloader_train(self, model_dict): reader_name = model_dict["dataset_name"] model_name = model_dict["name"] model_class = self._model[model_name][3] program = self._model[model_name][0].clone() _build_strategy = fluid.BuildStrategy() _exe_strategy = fluid.ExecutionStrategy() # 0: kCoeffNumDevice; 1: One; 2: Customized _gradient_scale_strategy = model_dict.get("gradient_scale_strategy", 0) if _gradient_scale_strategy == 0: gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.CoeffNumDevice elif _gradient_scale_strategy == 1: gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.One elif _gradient_scale_strategy == 2: gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized else: raise ValueError( "Unsurpported config. gradient_scale_strategy must be one of [0, 1, 2]." ) _build_strategy.gradient_scale_strategy = gradient_scale_strategy if "thread_num" in model_dict and model_dict["thread_num"] > 1: _build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce _exe_strategy.num_threads = model_dict["thread_num"] os.environ['CPU_NUM'] = str(_exe_strategy.num_threads) program = fluid.compiler.CompiledProgram(program).with_data_parallel( loss_name=model_class.get_avg_cost().name, build_strategy=_build_strategy, exec_strategy=_exe_strategy) fetch_vars = [] fetch_alias = [] fetch_period = int( envs.get_global_env( "runner." + self._runner_name + ".print_interval", 20)) metrics = model_class.get_metrics() if metrics: fetch_vars = metrics.values() fetch_alias = metrics.keys() metrics_varnames = [] metrics_format = [] metrics_format.append("{}: {{}}".format("batch")) for name, var in metrics.items(): metrics_varnames.append(var.name) metrics_format.append("{}: {{}}".format(name)) metrics_format = ", ".join(metrics_format) reader = self._model[model_name][3]._data_loader reader.start() batch_id = 0 scope = self._model[model_name][2] with fluid.scope_guard(scope): try: while True: metrics_rets = self._exe.run(program=program, fetch_list=metrics_varnames) metrics = [batch_id] metrics.extend(metrics_rets) if batch_id % fetch_period == 0 and batch_id != 0: print(metrics_format.format(*metrics)) batch_id += 1 except fluid.core.EOFException: reader.reset()
def _init_hyper_parameters(self): self.sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None) self.num_field = envs.get_global_env("hyper_parameters.num_field", None) self.reg = envs.get_global_env("hyper_parameters.reg", 1e-4)
def MMOE(self, is_infer=False): feature_size = envs.get_global_env("hyper_parameters.feature_size", None, self._namespace) expert_num = envs.get_global_env("hyper_parameters.expert_num", None, self._namespace) gate_num = envs.get_global_env("hyper_parameters.gate_num", None, self._namespace) expert_size = envs.get_global_env("hyper_parameters.expert_size", None, self._namespace) tower_size = envs.get_global_env("hyper_parameters.tower_size", None, self._namespace) input_data = fluid.data(name="input", shape=[-1, feature_size], dtype="float32") label_income = fluid.data(name="label_income", shape=[-1, 2], dtype="float32", lod_level=0) label_marital = fluid.data(name="label_marital", shape=[-1, 2], dtype="float32", lod_level=0) if is_infer: self._infer_data_var = [input_data, label_income, label_marital] self._infer_data_loader = fluid.io.DataLoader.from_generator( feed_list=self._infer_data_var, capacity=64, use_double_buffer=False, iterable=False) self._data_var.extend([input_data, label_income, label_marital]) # f_{i}(x) = activation(W_{i} * x + b), where activation is ReLU according to the paper expert_outputs = [] for i in range(0, expert_num): expert_output = fluid.layers.fc( input=input_data, size=expert_size, act='relu', bias_attr=fluid.ParamAttr(learning_rate=1.0), name='expert_' + str(i)) expert_outputs.append(expert_output) expert_concat = fluid.layers.concat(expert_outputs, axis=1) expert_concat = fluid.layers.reshape(expert_concat, [-1, expert_num, expert_size]) # g^{k}(x) = activation(W_{gk} * x + b), where activation is softmax according to the paper output_layers = [] for i in range(0, gate_num): cur_gate = fluid.layers.fc( input=input_data, size=expert_num, act='softmax', bias_attr=fluid.ParamAttr(learning_rate=1.0), name='gate_' + str(i)) # f^{k}(x) = sum_{i=1}^{n}(g^{k}(x)_{i} * f_{i}(x)) cur_gate_expert = fluid.layers.elementwise_mul(expert_concat, cur_gate, axis=0) cur_gate_expert = fluid.layers.reduce_sum(cur_gate_expert, dim=1) # Build tower layer cur_tower = fluid.layers.fc(input=cur_gate_expert, size=tower_size, act='relu', name='task_layer_' + str(i)) out = fluid.layers.fc(input=cur_tower, size=2, act='softmax', name='out_' + str(i)) output_layers.append(out) pred_income = fluid.layers.clip(output_layers[0], min=1e-15, max=1.0 - 1e-15) pred_marital = fluid.layers.clip(output_layers[1], min=1e-15, max=1.0 - 1e-15) label_income_1 = fluid.layers.slice(label_income, axes=[1], starts=[1], ends=[2]) label_marital_1 = fluid.layers.slice(label_marital, axes=[1], starts=[1], ends=[2]) auc_income, batch_auc_1, auc_states_1 = fluid.layers.auc( input=pred_income, label=fluid.layers.cast(x=label_income_1, dtype='int64')) auc_marital, batch_auc_2, auc_states_2 = fluid.layers.auc( input=pred_marital, label=fluid.layers.cast(x=label_marital_1, dtype='int64')) if is_infer: self._infer_results["AUC_income"] = auc_income self._infer_results["AUC_marital"] = auc_marital return cost_income = fluid.layers.cross_entropy(input=pred_income, label=label_income, soft_label=True) cost_marital = fluid.layers.cross_entropy(input=pred_marital, label=label_marital, soft_label=True) avg_cost_income = fluid.layers.mean(x=cost_income) avg_cost_marital = fluid.layers.mean(x=cost_marital) cost = avg_cost_income + avg_cost_marital self._cost = cost self._metrics["AUC_income"] = auc_income self._metrics["BATCH_AUC_income"] = batch_auc_1 self._metrics["AUC_marital"] = auc_marital self._metrics["BATCH_AUC_marital"] = batch_auc_2
def optimizer(self): learning_rate = envs.get_global_env("hyper_parameters.base_lr", None, self._namespace) sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=learning_rate) return sgd_optimizer
def infer(self, context): infer_program = fluid.Program() startup_program = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(infer_program, startup_program): self.model.infer_net() if self.model._infer_data_loader is None: context['status'] = 'terminal_pass' return reader = self._get_dataloader("Evaluate") metrics_varnames = [] metrics_format = [] metrics_format.append("{}: {{}}".format("epoch")) metrics_format.append("{}: {{}}".format("batch")) for name, var in self.model.get_infer_results().items(): metrics_varnames.append(var.name) metrics_format.append("{}: {{}}".format(name)) metrics_format = ", ".join(metrics_format) self._exe.run(startup_program) model_list = self.increment_models evaluate_only = envs.get_global_env('evaluate_only', False, namespace='evaluate') if evaluate_only: model_list = [(0, envs.get_global_env('evaluate_model_path', "", namespace='evaluate'))] is_return_numpy = envs.get_global_env('is_return_numpy', True, namespace='evaluate') for (epoch, model_dir) in model_list: print("Begin to infer No.{} model, model_dir: {}".format( epoch, model_dir)) program = infer_program.clone() fluid.io.load_persistables(self._exe, model_dir, program) reader.start() batch_id = 0 try: while True: metrics_rets = self._exe.run(program=program, fetch_list=metrics_varnames, return_numpy=is_return_numpy) metrics = [epoch, batch_id] metrics.extend(metrics_rets) if batch_id % 2 == 0 and batch_id != 0: print(metrics_format.format(*metrics)) batch_id += 1 except fluid.core.EOFException: reader.reset() context['status'] = 'terminal_pass'
def _init_hyper_parameters(self): self.item_len = envs.get_global_env("hyper_parameters.self.item_len") self.hidden_size = envs.get_global_env("hyper_parameters.hidden_size") self.user_vocab = envs.get_global_env("hyper_parameters.user_vocab") self.item_vocab = envs.get_global_env("hyper_parameters.item_vocab") self.embed_size = envs.get_global_env("hyper_parameters.embed_size")
def _get_dataset(self, state="TRAIN"): if state == "TRAIN": inputs = self.model.get_inputs() namespace = "train.reader" train_data_path = envs.get_global_env("train_data_path", None, namespace) else: inputs = self.model.get_infer_inputs() namespace = "evaluate.reader" train_data_path = envs.get_global_env("test_data_path", None, namespace) sparse_slots = envs.get_global_env("sparse_slots", None, namespace) dense_slots = envs.get_global_env("dense_slots", None, namespace) threads = int(envs.get_runtime_environ("train.trainer.threads")) batch_size = envs.get_global_env("batch_size", None, namespace) reader_class = envs.get_global_env("class", None, namespace) abs_dir = os.path.dirname(os.path.abspath(__file__)) reader = os.path.join(abs_dir, '../utils', 'dataset_instance.py') if sparse_slots is None and dense_slots is None: pipe_cmd = "python {} {} {} {}".format(reader, reader_class, state, self._config_yaml) else: if sparse_slots is None: sparse_slots = "#" if dense_slots is None: dense_slots = "#" padding = envs.get_global_env("padding", 0, namespace) pipe_cmd = "python {} {} {} {} {} {} {} {}".format( reader, "slot", "slot", self._config_yaml, namespace, \ sparse_slots.replace(" ", "#"), dense_slots.replace(" ", "#"), str(padding)) if train_data_path.startswith("paddlerec::"): package_base = envs.get_runtime_environ("PACKAGE_BASE") assert package_base is not None train_data_path = os.path.join(package_base, train_data_path.split("::")[1]) dataset = fluid.DatasetFactory().create_dataset() dataset.set_use_var(inputs) dataset.set_pipe_command(pipe_cmd) dataset.set_batch_size(batch_size) dataset.set_thread(threads) file_list = [ os.path.join(train_data_path, x) for x in os.listdir(train_data_path) ] self.files = file_list dataset.set_filelist(self.files) debug_mode = envs.get_global_env("reader_debug_mode", False, namespace) if debug_mode: print("--- Dataset Debug Mode Begin , show pre 10 data of {}---". format(file_list[0])) os.system("cat {} | {} | head -10".format(file_list[0], pipe_cmd)) print("--- Dataset Debug Mode End , show pre 10 data of {}---". format(file_list[0])) exit(0) return dataset
def net(self, inputs, is_infer=False): vocab_size = envs.get_global_env("hyper_parameters.vocab_size", None, self._namespace) embed_size = envs.get_global_env("hyper_parameters.embed_size", None, self._namespace) emb = [] for data in inputs[0:-2]: feat_emb = fluid.embedding( input=data, size=[vocab_size, embed_size], param_attr=fluid.ParamAttr( name='dis_emb', learning_rate=5, initializer=fluid.initializer.Xavier(fan_in=embed_size, fan_out=embed_size)), is_sparse=True) field_emb = fluid.layers.sequence_pool(input=feat_emb, pool_type='sum') emb.append(field_emb) concat_emb = fluid.layers.concat(emb, axis=1) # ctr active = 'relu' ctr_fc1 = self.fc('ctr_fc1', concat_emb, 200, active) ctr_fc2 = self.fc('ctr_fc2', ctr_fc1, 80, active) ctr_out = self.fc('ctr_out', ctr_fc2, 2, 'softmax') # cvr cvr_fc1 = self.fc('cvr_fc1', concat_emb, 200, active) cvr_fc2 = self.fc('cvr_fc2', cvr_fc1, 80, active) cvr_out = self.fc('cvr_out', cvr_fc2, 2, 'softmax') ctr_clk = inputs[-2] ctcvr_buy = inputs[-1] ctr_prop_one = fluid.layers.slice(ctr_out, axes=[1], starts=[1], ends=[2]) cvr_prop_one = fluid.layers.slice(cvr_out, axes=[1], starts=[1], ends=[2]) ctcvr_prop_one = fluid.layers.elementwise_mul(ctr_prop_one, cvr_prop_one) ctcvr_prop = fluid.layers.concat( input=[1 - ctcvr_prop_one, ctcvr_prop_one], axis=1) auc_ctr, batch_auc_ctr, auc_states_ctr = fluid.layers.auc( input=ctr_out, label=ctr_clk) auc_ctcvr, batch_auc_ctcvr, auc_states_ctcvr = fluid.layers.auc( input=ctcvr_prop, label=ctcvr_buy) if is_infer: self._infer_results["AUC_ctr"] = auc_ctr self._infer_results["AUC_ctcvr"] = auc_ctcvr return loss_ctr = fluid.layers.cross_entropy(input=ctr_out, label=ctr_clk) loss_ctcvr = fluid.layers.cross_entropy(input=ctcvr_prop, label=ctcvr_buy) cost = loss_ctr + loss_ctcvr avg_cost = fluid.layers.mean(cost) self._cost = avg_cost self._metrics["AUC_ctr"] = auc_ctr self._metrics["BATCH_AUC_ctr"] = batch_auc_ctr self._metrics["AUC_ctcvr"] = auc_ctcvr self._metrics["BATCH_AUC_ctcvr"] = batch_auc_ctcvr
def deepfm_net(self): init_value_ = 0.1 is_distributed = True if envs.get_trainer() == "CtrTrainer" else False sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None, self._namespace) sparse_feature_dim = envs.get_global_env( "hyper_parameters.sparse_feature_dim", None, self._namespace) # ------------------------- network input -------------------------- num_field = envs.get_global_env("hyper_parameters.num_field", None, self._namespace) raw_feat_idx = self._sparse_data_var[1] raw_feat_value = self._dense_data_var[0] self.label = self._sparse_data_var[0] feat_idx = raw_feat_idx feat_value = fluid.layers.reshape( raw_feat_value, [-1, num_field, 1]) # None * num_field * 1 reg = envs.get_global_env("hyper_parameters.reg", 1e-4, self._namespace) first_weights_re = fluid.embedding( input=feat_idx, is_sparse=True, is_distributed=is_distributed, dtype='float32', size=[sparse_feature_number + 1, 1], padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_), regularizer=fluid.regularizer.L1DecayRegularizer(reg))) first_weights = fluid.layers.reshape(first_weights_re, shape=[-1, num_field, 1]) # None * num_field * 1 y_first_order = fluid.layers.reduce_sum((first_weights * feat_value), 1) # ------------------------- second order term -------------------------- feat_embeddings_re = fluid.embedding( input=feat_idx, is_sparse=True, is_distributed=is_distributed, dtype='float32', size=[sparse_feature_number + 1, sparse_feature_dim], padding_idx=0, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_ / math.sqrt(float(sparse_feature_dim))))) feat_embeddings = fluid.layers.reshape( feat_embeddings_re, shape=[-1, num_field, sparse_feature_dim]) # None * num_field * embedding_size feat_embeddings = feat_embeddings * feat_value # None * num_field * embedding_size # sum_square part summed_features_emb = fluid.layers.reduce_sum( feat_embeddings, 1) # None * embedding_size summed_features_emb_square = fluid.layers.square( summed_features_emb) # None * embedding_size # square_sum part squared_features_emb = fluid.layers.square( feat_embeddings) # None * num_field * embedding_size squared_sum_features_emb = fluid.layers.reduce_sum( squared_features_emb, 1) # None * embedding_size y_second_order = 0.5 * fluid.layers.reduce_sum( summed_features_emb_square - squared_sum_features_emb, 1, keep_dim=True) # None * 1 # ------------------------- DNN -------------------------- layer_sizes = envs.get_global_env("hyper_parameters.fc_sizes", None, self._namespace) act = envs.get_global_env("hyper_parameters.act", None, self._namespace) y_dnn = fluid.layers.reshape(feat_embeddings, [-1, num_field * sparse_feature_dim]) for s in layer_sizes: y_dnn = fluid.layers.fc( input=y_dnn, size=s, act=act, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_ / math.sqrt(float(10)))), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_))) y_dnn = fluid.layers.fc( input=y_dnn, size=1, act=None, param_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_)), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_))) # ------------------------- DeepFM -------------------------- self.predict = fluid.layers.sigmoid(y_first_order + y_second_order + y_dnn)
def build_network(self, context): context["model"] = {} for model_dict in context["phases"]: context["model"][model_dict["name"]] = {} train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() dataset_name = model_dict["dataset_name"] with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): with fluid.scope_guard(scope): model_path = envs.os_path_adapter( envs.workspace_adapter(model_dict["model"])) model = envs.lazy_instance_by_fliename( model_path, "Model")(context["env"]) model._data_var = model.input_data( dataset_name=model_dict["dataset_name"]) if envs.get_global_env("dataset." + dataset_name + ".type") == "DataLoader": model._init_dataloader( is_infer=context["is_infer"]) data_loader = DataLoader(context) data_loader.get_dataloader(context, dataset_name, model._data_loader) model.net(model._data_var, context["is_infer"]) finetuning_varnames = envs.get_global_env( "runner." + context["runner_name"] + ".finetuning_aspect_varnames", default_value=[]) if len(finetuning_varnames) == 0: raise ValueError( "nothing need to be fine tuning, you may use other traning mode" ) if len(finetuning_varnames) != 1: raise ValueError( "fine tuning mode can only accept one varname now" ) varname = finetuning_varnames[0] finetuning_vars = train_program.global_block( ).vars[varname] finetuning_vars.stop_gradient = True optimizer = model.optimizer() optimizer.minimize(model._cost) context["model"][ model_dict["name"]]["main_program"] = train_program context["model"][ model_dict["name"]]["startup_program"] = startup_program context["model"][model_dict["name"]]["scope"] = scope context["model"][model_dict["name"]]["model"] = model context["model"][model_dict["name"]][ "default_main_program"] = train_program.clone() context["model"][model_dict["name"]]["compiled_program"] = None context["dataset"] = {} for dataset in context["env"]["dataset"]: type = envs.get_global_env("dataset." + dataset["name"] + ".type") if type == "QueueDataset": dataset_class = QueueDataset(context) context["dataset"][ dataset["name"]] = dataset_class.create_dataset( dataset["name"], context) context["status"] = "startup_pass"
def _executor_dataloader_train(self, model_dict, context): model_name = model_dict["name"] model_class = context["model"][model_dict["name"]]["model"] program = self._get_dataloader_program(model_dict, context) fetch_period = int( envs.get_global_env( "runner." + context["runner_name"] + ".print_interval", 20)) save_step_interval = int( envs.get_global_env( "runner." + context["runner_name"] + ".save_step_interval", -1)) if context["is_infer"]: metrics = model_class.get_infer_results() else: metrics = model_class.get_metrics() metrics_varnames = [] metrics_format = [] if context["is_infer"]: metrics_format.append("\t[Infer] {}: {{}}".format("batch")) else: metrics_format.append("\t[Train]") if "current_epoch" in context: metrics_format.append(" epoch: {}".format( context["current_epoch"])) metrics_format.append(" {}: {{}}".format("batch")) metrics_format.append("{}: {{:.2f}}s".format("time_each_interval")) metrics_names = ["total_batch"] metrics_indexes = dict() for name, var in metrics.items(): metrics_names.append(name) metrics_varnames.append(var.name) metrics_indexes[var.name] = len(metrics_varnames) - 1 metrics_format.append("{}: {{}}".format(name)) metrics_format = ", ".join(metrics_format) reader = context["model"][model_dict["name"]]["model"]._data_loader reader.start() batch_id = 0 begin_time = time.time() scope = context["model"][model_name]["scope"] runner_results = [] result = None with fluid.scope_guard(scope): try: while True: metrics_tensors = context["exe"].run( program=program, fetch_list=metrics_varnames, return_numpy=False) metrics = [batch_id] metrics_rets = [ as_numpy(metrics_tensor) for metrics_tensor in metrics_tensors ] metrics.extend(metrics_rets) batch_runner_result = {} for k, v in metrics_indexes.items(): batch_runner_result[k] = np.array( metrics_rets[v]).tolist() runner_results.append(batch_runner_result) if batch_id % fetch_period == 0: end_time = time.time() seconds = end_time - begin_time metrics_logging = metrics[:] metrics_logging.insert(1, seconds) begin_time = end_time logging.info(metrics_format.format(*metrics_logging)) if save_step_interval >= 1 and batch_id % save_step_interval == 0 and context[ "is_infer"] == False: if context["is_fleet"]: if context["fleet_mode"].upper() == "PS": train_prog = context["model"][ model_dict["name"]]["main_program"] else: train_prog = context["model"][ model_dict["name"]]["default_main_program"] else: train_prog = context["model"][ model_dict["name"]]["default_main_program"] startup_prog = context["model"][ model_dict["name"]]["startup_program"] with fluid.program_guard(train_prog, startup_prog): self.save(context, is_fleet=context["is_fleet"], epoch_id=None, batch_id=batch_id) batch_id += 1 except fluid.core.EOFException: reader.reset() runner_result_save_path = envs.get_global_env( "runner." + context["runner_name"] + ".runner_result_dump_path", None) if runner_result_save_path: if "current_epoch" in context: runner_result_save_path = runner_result_save_path + "_epoch_{}".format( context["current_epoch"]) logging.info( "Dump runner result in {}".format(runner_result_save_path)) with open(runner_result_save_path, 'w+') as fout: json.dump(runner_results, fout) if batch_id > 0: result = dict(zip(metrics_names, metrics)) return result
def build_network(self, context): context["model"] = {} if len(context["env"]["phase"]) > 1: warnings.warn("Cluster Train Only Support One Phase.", category=UserWarning, stacklevel=2) model_dict = context["env"]["phase"][0] train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() dataset_name = model_dict["dataset_name"] with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): with fluid.scope_guard(scope): context["model"][model_dict["name"]] = {} model_path = envs.os_path_adapter( envs.workspace_adapter(model_dict["model"])) model = envs.lazy_instance_by_fliename( model_path, "Model")(context["env"]) model._data_var = model.input_data( dataset_name=model_dict["dataset_name"]) if envs.get_global_env("dataset." + dataset_name + ".type") == "DataLoader": model._init_dataloader(is_infer=False) model.net(model._data_var, False) optimizer = model.optimizer() optimizer = context["fleet"].distributed_optimizer( optimizer) optimizer.minimize([model._cost], [fluid.global_scope()]) context["model"][ model_dict["name"]]["main_program"] = train_program context["model"][model_dict["name"]][ "startup_program"] = startup_program context["model"][model_dict["name"]]["scope"] = scope context["model"][model_dict["name"]]["model"] = model context["model"][model_dict["name"]][ "default_main_program"] = train_program.clone() context["model"][ model_dict["name"]]["compile_program"] = None if context["fleet"].is_server(): self._server(context) else: context["dataset"] = {} for phase in context["env"]["phase"]: type = envs.get_global_env("dataset." + dataset["name"] + ".type") if type == "DataLoader": data_loader = DataLoader(context) data_loader.get_dataloader( context, dataset_name, context["model"][ model_dict["name"]]["model"]._data_loader) elif type == "QueueDataset": dataset_class = QueueDataset(context) context["dataset"][ dataset["name"]] = dataset_class.create_dataset( dataset["name"], context) context["status"] = "startup_pass"
def optimizer(self): learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) optimizer = fluid.optimizer.Adam(learning_rate, lazy_mode=True) return optimizer
def build_network(self, context): context["model"] = {} for model_dict in context["phases"]: context["model"][model_dict["name"]] = {} train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() dataset_name = model_dict["dataset_name"] with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): with fluid.scope_guard(scope): model_path = envs.os_path_adapter( envs.workspace_adapter(model_dict["model"])) model = envs.lazy_instance_by_fliename( model_path, "Model")(context["env"]) if context["is_infer"]: model._infer_data_var = model.input_data( is_infer=context["is_infer"], dataset_name=model_dict["dataset_name"]) else: model._data_var = model.input_data( dataset_name=model_dict["dataset_name"]) if envs.get_global_env("dataset." + dataset_name + ".type") == "DataLoader": model._init_dataloader( is_infer=context["is_infer"]) data_loader = DataLoader(context) data_loader.get_dataloader(context, dataset_name, model._data_loader) if context["is_infer"]: model.net(model._infer_data_var, context["is_infer"]) else: model.net(model._data_var, context["is_infer"]) optimizer = model.optimizer() optimizer.minimize(model._cost) context["model"][ model_dict["name"]]["main_program"] = train_program context["model"][ model_dict["name"]]["startup_program"] = startup_program context["model"][model_dict["name"]]["scope"] = scope context["model"][model_dict["name"]]["model"] = model context["model"][model_dict["name"]][ "default_main_program"] = train_program.clone() context["model"][model_dict["name"]]["compiled_program"] = None context["dataset"] = {} for dataset in context["env"]["dataset"]: type = envs.get_global_env("dataset." + dataset["name"] + ".type") if type == "QueueDataset": dataset_class = QueueDataset(context) context["dataset"][ dataset["name"]] = dataset_class.create_dataset( dataset["name"], context) context["status"] = "startup_pass"
def net(self): is_distributed = True if envs.get_trainer() == "CtrTrainer" else False neg_num = int( envs.get_global_env("hyper_parameters.neg_num", None, self._namespace)) sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None, self._namespace) sparse_feature_dim = envs.get_global_env( "hyper_parameters.sparse_feature_dim", None, self._namespace) with_shuffle_batch = bool( int( envs.get_global_env("hyper_parameters.with_shuffle_batch", None, self._namespace))) def embedding_layer(input, table_name, emb_dim, initializer_instance=None, squeeze=False): emb = fluid.embedding( input=input, is_sparse=True, is_distributed=is_distributed, size=[sparse_feature_number, emb_dim], param_attr=fluid.ParamAttr(name=table_name, initializer=initializer_instance), ) if squeeze: return fluid.layers.squeeze(input=emb, axes=[1]) else: return emb init_width = 0.5 / sparse_feature_dim emb_initializer = fluid.initializer.Uniform(-init_width, init_width) emb_w_initializer = fluid.initializer.Constant(value=0.0) input_emb = embedding_layer(self.input_word, "emb", sparse_feature_dim, emb_initializer, True) true_emb_w = embedding_layer(self.true_word, "emb_w", sparse_feature_dim, emb_w_initializer, True) true_emb_b = embedding_layer(self.true_word, "emb_b", 1, emb_w_initializer, True) if with_shuffle_batch: neg_emb_w_list = [] for i in range(neg_num): neg_emb_w_list.append( fluid.contrib.layers.shuffle_batch( true_emb_w)) # shuffle true_word neg_emb_w_concat = fluid.layers.concat(neg_emb_w_list, axis=0) neg_emb_w = fluid.layers.reshape( neg_emb_w_concat, shape=[-1, neg_num, sparse_feature_dim]) neg_emb_b_list = [] for i in range(neg_num): neg_emb_b_list.append( fluid.contrib.layers.shuffle_batch( true_emb_b)) # shuffle true_word neg_emb_b = fluid.layers.concat(neg_emb_b_list, axis=0) neg_emb_b_vec = fluid.layers.reshape(neg_emb_b, shape=[-1, neg_num]) else: neg_emb_w = embedding_layer(self.neg_word, "emb_w", sparse_feature_dim, emb_w_initializer) neg_emb_b = embedding_layer(self.neg_word, "emb_b", 1, emb_w_initializer) neg_emb_b_vec = fluid.layers.reshape(neg_emb_b, shape=[-1, neg_num]) true_logits = fluid.layers.elementwise_add( fluid.layers.reduce_sum(fluid.layers.elementwise_mul( input_emb, true_emb_w), dim=1, keep_dim=True), true_emb_b) input_emb_re = fluid.layers.reshape(input_emb, shape=[-1, 1, sparse_feature_dim]) neg_matmul = fluid.layers.matmul(input_emb_re, neg_emb_w, transpose_y=True) neg_logits = fluid.layers.elementwise_add( fluid.layers.reshape(neg_matmul, shape=[-1, neg_num]), neg_emb_b_vec) label_ones = fluid.layers.fill_constant_batch_size_like( true_logits, shape=[-1, 1], value=1.0, dtype='float32') label_zeros = fluid.layers.fill_constant_batch_size_like( true_logits, shape=[-1, neg_num], value=0.0, dtype='float32') true_xent = fluid.layers.sigmoid_cross_entropy_with_logits( true_logits, label_ones) neg_xent = fluid.layers.sigmoid_cross_entropy_with_logits( neg_logits, label_zeros) cost = fluid.layers.elementwise_add( fluid.layers.reduce_sum(true_xent, dim=1), fluid.layers.reduce_sum(neg_xent, dim=1)) self.avg_cost = fluid.layers.reduce_mean(cost) global_right_cnt = fluid.layers.create_global_var( name="global_right_cnt", persistable=True, dtype='float32', shape=[1], value=0) global_total_cnt = fluid.layers.create_global_var( name="global_total_cnt", persistable=True, dtype='float32', shape=[1], value=0) global_right_cnt.stop_gradient = True global_total_cnt.stop_gradient = True
def _init_hyper_parameters(self): self.is_distributed = True if envs.get_fleet_mode().upper( ) == "PSLIB" else False self.sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None) self.sparse_feature_dim = envs.get_global_env( "hyper_parameters.sparse_feature_dim", None) self.is_sparse = envs.get_global_env("hyper_parameters.is_sparse", False) self.use_batchnorm = envs.get_global_env( "hyper_parameters.use_batchnorm", False) self.use_dropout = envs.get_global_env("hyper_parameters.use_dropout", False) self.dropout_prob = envs.get_global_env( "hyper_parameters.dropout_prob", None) self.layer_sizes = envs.get_global_env("hyper_parameters.fc_sizes", None) self.loss_type = envs.get_global_env("hyper_parameters.loss_type", 'logloss') self.reg = envs.get_global_env("hyper_parameters.reg", 1e-4) self.num_field = envs.get_global_env("hyper_parameters.num_field", None) self.act = envs.get_global_env("hyper_parameters.act", None)
def __init__(self, config): ModelBase.__init__(self, config) # tree meta hyper parameters self.max_layers = envs.get_global_env("tree_parameters.max_layers", 4, self._namespace) self.node_nums = envs.get_global_env("tree_parameters.node_nums", 26, self._namespace) self.leaf_node_nums = envs.get_global_env( "tree_parameters.leaf_node_nums", 13, self._namespace) self.output_positive = envs.get_global_env( "tree_parameters.output_positive", True, self._namespace) self.layer_node_num_list = envs.get_global_env( "tree_parameters.layer_node_num_list", [2, 4, 7, 12], self._namespace) self.child_nums = envs.get_global_env("tree_parameters.child_nums", 2, self._namespace) self.tree_layer_path = envs.get_global_env("tree.tree_layer_path", None, "train.startup") # model training hyper parameter self.node_emb_size = envs.get_global_env( "hyper_parameters.node_emb_size", 64, self._namespace) self.input_emb_size = envs.get_global_env( "hyper_parameters.input_emb_size", 768, self._namespace) self.act = envs.get_global_env("hyper_parameters.act", "tanh", self._namespace) self.neg_sampling_list = envs.get_global_env( "hyper_parameters.neg_sampling_list", [1, 2, 3, 4], self._namespace) # model infer hyper parameter self.topK = envs.get_global_env("hyper_parameters.node_nums", 1, self._namespace) self.batch_size = envs.get_global_env("batch_size", 1, "evaluate.reader")
def instance(self, context): models = envs.get_global_env("train.model.models") model_class = envs.lazy_instance_by_fliename(models, "Model") self.model = model_class(None) context['status'] = 'init_pass'
def optimizer(self): learning_rate = envs.get_global_env("hyper_parameters.learning_rate", None, self._namespace) optimizer = fluid.optimizer.SGD(learning_rate) return optimizer
def xdeepfm_net(self): init_value_ = 0.1 initer = fluid.initializer.TruncatedNormalInitializer( loc=0.0, scale=init_value_) is_distributed = True if envs.get_trainer() == "CtrTrainer" else False sparse_feature_number = envs.get_global_env( "hyper_parameters.sparse_feature_number", None, self._namespace) sparse_feature_dim = envs.get_global_env( "hyper_parameters.sparse_feature_dim", None, self._namespace) # ------------------------- network input -------------------------- num_field = envs.get_global_env("hyper_parameters.num_field", None, self._namespace) raw_feat_idx = self._sparse_data_var[1] raw_feat_value = self._dense_data_var[0] self.label = self._sparse_data_var[0] feat_idx = raw_feat_idx feat_value = fluid.layers.reshape( raw_feat_value, [-1, num_field, 1]) # None * num_field * 1 feat_embeddings = fluid.embedding( input=feat_idx, is_sparse=True, dtype='float32', size=[sparse_feature_number + 1, sparse_feature_dim], padding_idx=0, param_attr=fluid.ParamAttr(initializer=initer)) feat_embeddings = fluid.layers.reshape(feat_embeddings, [ -1, num_field, sparse_feature_dim ]) # None * num_field * embedding_size feat_embeddings = feat_embeddings * feat_value # None * num_field * embedding_size # -------------------- linear -------------------- weights_linear = fluid.embedding( input=feat_idx, is_sparse=True, dtype='float32', size=[sparse_feature_number + 1, 1], padding_idx=0, param_attr=fluid.ParamAttr(initializer=initer)) weights_linear = fluid.layers.reshape( weights_linear, [-1, num_field, 1]) # None * num_field * 1 b_linear = fluid.layers.create_parameter( shape=[1], dtype='float32', default_initializer=fluid.initializer.ConstantInitializer(value=0)) y_linear = fluid.layers.reduce_sum( (weights_linear * feat_value), 1) + b_linear # -------------------- CIN -------------------- layer_sizes_cin = envs.get_global_env( "hyper_parameters.layer_sizes_cin", None, self._namespace) Xs = [feat_embeddings] last_s = num_field for s in layer_sizes_cin: # calculate Z^(k+1) with X^k and X^0 X_0 = fluid.layers.reshape( fluid.layers.transpose(Xs[0], [0, 2, 1]), [-1, sparse_feature_dim, num_field, 1]) # None, embedding_size, num_field, 1 X_k = fluid.layers.reshape( fluid.layers.transpose(Xs[-1], [0, 2, 1]), [-1, sparse_feature_dim, 1, last_s]) # None, embedding_size, 1, last_s Z_k_1 = fluid.layers.matmul( X_0, X_k) # None, embedding_size, num_field, last_s # compresses Z^(k+1) to X^(k+1) Z_k_1 = fluid.layers.reshape(Z_k_1, [ -1, sparse_feature_dim, last_s * num_field ]) # None, embedding_size, last_s*num_field Z_k_1 = fluid.layers.transpose( Z_k_1, [0, 2, 1]) # None, s*num_field, embedding_size Z_k_1 = fluid.layers.reshape( Z_k_1, [-1, last_s * num_field, 1, sparse_feature_dim] ) # None, last_s*num_field, 1, embedding_size (None, channal_in, h, w) X_k_1 = fluid.layers.conv2d( Z_k_1, num_filters=s, filter_size=(1, 1), act=None, bias_attr=False, param_attr=fluid.ParamAttr( initializer=initer)) # None, s, 1, embedding_size X_k_1 = fluid.layers.reshape( X_k_1, [-1, s, sparse_feature_dim]) # None, s, embedding_size Xs.append(X_k_1) last_s = s # sum pooling y_cin = fluid.layers.concat(Xs[1:], 1) # None, (num_field++), embedding_size y_cin = fluid.layers.reduce_sum(y_cin, -1) # None, (num_field++) y_cin = fluid.layers.fc(input=y_cin, size=1, act=None, param_attr=fluid.ParamAttr(initializer=initer), bias_attr=None) y_cin = fluid.layers.reduce_sum(y_cin, dim=-1, keep_dim=True) # -------------------- DNN -------------------- layer_sizes_dnn = envs.get_global_env( "hyper_parameters.layer_sizes_dnn", None, self._namespace) act = envs.get_global_env("hyper_parameters.act", None, self._namespace) y_dnn = fluid.layers.reshape(feat_embeddings, [-1, num_field * sparse_feature_dim]) for s in layer_sizes_dnn: y_dnn = fluid.layers.fc( input=y_dnn, size=s, act=act, param_attr=fluid.ParamAttr(initializer=initer), bias_attr=None) y_dnn = fluid.layers.fc(input=y_dnn, size=1, act=None, param_attr=fluid.ParamAttr(initializer=initer), bias_attr=None) # ------------------- xDeepFM ------------------ self.predict = fluid.layers.sigmoid(y_linear + y_cin + y_dnn)