def main(_): data_dir = FLAGS.train_path data_files = [] for i in range(FLAGS.train_parts): data_files.append(data_dir + 'part-r-{:0>5}'.format(i)) train_files = data_files[:-FLAGS.eval_parts] eval_files = data_files[-FLAGS.eval_parts:] linear_feature_columns, embedding_feature_columns = build_feature_columns( FLAGS.embedding_size) distribute_strategy = None if FLAGS.mirror: distribute_strategy = tf.distribute.MirroredStrategy() config = estimator.RunConfig( save_checkpoints_steps=FLAGS.save_checkpoints_steps, keep_checkpoint_max=5, log_step_count_steps=FLAGS.log_steps, save_summary_steps=200, train_distribute=distribute_strategy, eval_distribute=distribute_strategy) model_params = { 'linear_feature_columns': linear_feature_columns, 'embedding_feature_columns': embedding_feature_columns, 'embedding_size': FLAGS.embedding_size, "learning_rate": FLAGS.learning_rate, "dropout": FLAGS.dropout, "deep_layers": FLAGS.deep_layers } dcn = estimator.Estimator(model_fn=model_fn, model_dir='./model/', params=model_params, config=config) if FLAGS.task_type == 'train': train_spec = estimator.TrainSpec( input_fn=lambda: input_fn(train_files, num_epochs=FLAGS.num_epochs, batch_size=FLAGS.batch_size, need_shuffle=True)) eval_spec = estimator.EvalSpec( input_fn=lambda: input_fn(eval_files, num_epochs=-1, batch_size=FLAGS.batch_size, need_shuffle=True), steps=200, start_delay_secs=1, throttle_secs=5) estimator.train_and_evaluate(dcn, train_spec, eval_spec) elif FLAGS.task_type == 'eval': dcn.evaluate(input_fn=lambda: input_fn( eval_files, num_epochs=1, batch_size=FLAGS.batch_size), steps=200)
def main(_): print( "==================== 1.Check Args and Initialized Distributed Env...") if FLAGS.model_dir == "": # 算法模型checkpoint文件 FLAGS.model_dir = (date.today() + timedelta(-1) ).strftime("%Y%m") + "_ckt_" + FLAGS.algorithm if FLAGS.serve_dir == "": # 算法模型输出pb文件 FLAGS.serve_dir = (date.today() + timedelta(-1) ).strftime("%Y%m") + "_exp_" + FLAGS.algorithm if FLAGS.input_dir == "": # windows环境测试 FLAGS.input_dir = os.getcwd() + "\\feat1\\" + "\\CV\\" train_files = glob.glob("%s/train.set" % FLAGS.input_dir) # 获取指定目录下train文件 valid_files = glob.glob("%s/valid.set" % FLAGS.input_dir) # 获取指定目录下valid文件 tests_files = glob.glob("%s/tests.set" % FLAGS.input_dir) # 获取指定目录下tests文件 infer_files = glob.glob("%s/infer.set" % FLAGS.input_dir) # 获取指定目录下infer文件 random.shuffle(train_files) # 打散train文件 if FLAGS.clear_mod == "True" and FLAGS.task_mode == "train": # 删除已存在的模型文件 try: shutil.rmtree(FLAGS.model_dir) # 递归删除目录下的目录及文件 except Exception as e: print(e, "At clear_existed_model") else: print("Existed model cleared at %s folder" % FLAGS.model_dir) print("==================== 2.Set model params and Build CTR model...") model_params = { "algorithm": FLAGS.algorithm, "feature_size": FLAGS.feature_size, "field_size": FLAGS.field_size, "embed_size": FLAGS.embed_size, "loss_mode": FLAGS.loss_mode, "optimizer": FLAGS.optimizer, "learning_rate": FLAGS.learning_rate, "l2_reg_lambda": FLAGS.l2_reg_lambda, "deep_layers": FLAGS.deep_layers, "cross_layers": FLAGS.cross_layers, "dropout": FLAGS.dropout } if FLAGS.algorithm == "LR": model_fn = lr elif FLAGS.algorithm == "FM": model_fn = fm elif FLAGS.algorithm == "DC": model_fn = deepcrossing elif FLAGS.algorithm == "FNN" or FLAGS.algorithm == "IPNN" or FLAGS.algorithm == "OPNN": model_fn = fpnn elif FLAGS.algorithm == "WD": model_fn = wd elif FLAGS.algorithm == "DeepFM": model_fn = deepfm elif FLAGS.algorithm == "DCN": model_fn = dcn elif FLAGS.algorithm == "NFM": model_fn = nfm else: model_fn = None print("Invalid algorithm, not supported!") epoch_step = int(FLAGS.train_size / FLAGS.batch_size) # one epoch = num of steps train_step = epoch_step * FLAGS.num_epochs # data_num * num_epochs / batch_size session_config = tf.ConfigProto(device_count={ "GPU": 1, "CPU": FLAGS.num_thread }) config = estimator.RunConfig(session_config=session_config, save_checkpoints_steps=epoch_step * 2, save_summary_steps=FLAGS.log_steps, log_step_count_steps=FLAGS.log_steps) ctr = estimator.Estimator(model_fn=model_fn, model_dir=FLAGS.model_dir, params=model_params, config=config) print("==================== 3.Apply CTR model to diff tasks...") if FLAGS.task_mode == "train": train_spec = estimator.TrainSpec(input_fn=lambda: input_fn( train_files, FLAGS.batch_size, FLAGS.num_epochs, True), max_steps=train_step) eval_spec = estimator.EvalSpec( input_fn=lambda: input_fn(valid_files, FLAGS.batch_size, 1, False), steps=None, start_delay_secs=50, throttle_secs=15) estimator.train_and_evaluate(ctr, train_spec, eval_spec) elif FLAGS.task_mode == "eval": ctr.evaluate( input_fn=lambda: input_fn(train_files, FLAGS.batch_size, 1, False)) elif FLAGS.task_mode == "infer": preds = ctr.predict( input_fn=lambda: input_fn(infer_files, FLAGS.batch_size, 1, False), predict_keys="prob") with open(FLAGS.input_dir + "/infer.csv", "w") as fo: for prob in preds: fo.write("%f\n" % (prob['prob'])) elif FLAGS.task_mode == "export": feature_spec = { "feat_idx": tf.placeholder(dtype=tf.int64, shape=[None, FLAGS.field_size], name="feat_idx"), "feat_val": tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.field_size], name="feat_val") } serving_input_receiver_fn = estimator.export.build_raw_serving_input_receiver_fn( feature_spec) ctr.export_savedmodel(FLAGS.serve_dir, serving_input_receiver_fn)
show_dataflow=True) feature_columns = [ tf.feature_column.numeric_column(key=key) for key in train_x.keys() ] test = tf.feature_column.numeric_column(train_x.keys()[0], default_value=0.0) test = tf.feature_column.bucketized_column(test, [0.1, 1, 100]) test_emb = tf.feature_column.embedding_column(test, 10) feature_columns.append(test_emb) session_config = tf.ConfigProto() mirrored_strategy = tf.distribute.MirroredStrategy() config = estimator.RunConfig( train_distribute=mirrored_strategy, eval_distribute=mirrored_strategy, ) classifier = estimator.DNNClassifier(feature_columns=feature_columns, hidden_units=[10, 10], n_classes=3, config=config) classifier.train( input_fn=lambda: train_input_fn(train_x, train_y, batch_size=BATCH_SIZE), hooks=[hook], steps=STEPS) # evaluate eval_result = classifier.evaluate( input_fn=lambda: eval_input_fn(test_x, test_y, batch_size=BATCH_SIZE))
def main(_): # tf.enable_eager_execution() # dataset = input_fn(['train2'], 1) # for raw_record in dataset.take(1): # print(raw_record[1]) data_dir = FLAGS.train_path data_files = [] for i in range(FLAGS.train_parts): data_files.append(data_dir + 'part-r-{:0>5}'.format(i)) train_files = data_files[:-FLAGS.eval_parts] eval_files = data_files[-FLAGS.eval_parts:] train_files = ['train2'] eval_files = ['valid2'] test_files = [] for i in range(FLAGS.test_parts): test_files.append(FLAGS.test_path + 'part-r-{:0>5}'.format(i)) distribute_strategy = None if FLAGS.mirror: distribute_strategy = tf.distribute.MirroredStrategy() config = estimator.RunConfig( save_checkpoints_steps=FLAGS.save_checkpoints_steps, keep_checkpoint_max=5, log_step_count_steps=FLAGS.log_steps, save_summary_steps=200, train_distribute=distribute_strategy, eval_distribute=distribute_strategy) model_params = { 'pkg_count': FLAGS.pkg_count, 'pkgc_count': FLAGS.pkgc_count, 'ssid_count': FLAGS.ssid_count, 'oper_count': FLAGS.oper_count, 'embedding_size': FLAGS.embedding_size, "learning_rate": FLAGS.learning_rate, "dropout": FLAGS.dropout, "deep_layers": FLAGS.deep_layers, } din = estimator.Estimator(model_fn=model_fn, model_dir='./models/', params=model_params, config=config) if FLAGS.task_type == 'train': train_spec = estimator.TrainSpec( input_fn=lambda: input_fn(train_files, num_epochs=FLAGS.num_epochs, batch_size=FLAGS.batch_size, need_shuffle=True)) eval_spec = estimator.EvalSpec(input_fn=lambda: input_fn( eval_files, num_epochs=-1, batch_size=FLAGS.batch_size), steps=FLAGS.eval_steps, start_delay_secs=1, throttle_secs=5) start = time() estimator.train_and_evaluate(din, train_spec, eval_spec) elapsed = (time() - start) tf.logging.info("Training time used: {0}ms".format( round(elapsed * 1000, 2))) elif FLAGS.task_type == 'eval': din.evaluate(input_fn=lambda: input_fn( eval_files, num_epochs=1, batch_size=FLAGS.batch_size), steps=FLAGS.eval_steps * 10) elif FLAGS.task_type == 'predict': p = din.predict(input_fn=lambda: input_fn( eval_files, num_epochs=1, batch_size=FLAGS.batch_size)) tf.logging.info('done predit')
def main(_): linear_feature_columns, embedding_feature_columns = build_model_columns(FLAGS.embedding_size) # session_config = tf.ConfigProto(log_device_placement=True) # session_config.gpu_options.allow_growth = True distribute_strategy = None if FLAGS.mirror: distribute_strategy = tf.distribute.MirroredStrategy() config = estimator.RunConfig( save_checkpoints_steps=FLAGS.save_checkpoints_steps, keep_checkpoint_max=5, log_step_count_steps=FLAGS.log_steps, save_summary_steps=200, train_distribute=distribute_strategy, eval_distribute=distribute_strategy ) model_params = { 'linear_feature_columns': linear_feature_columns, 'embedding_feature_columns': embedding_feature_columns, 'embedding_size': FLAGS.embedding_size, "learning_rate": FLAGS.learning_rate, "dropout": FLAGS.dropout, "deep_layers": FLAGS.deep_layers } deepfm = estimator.Estimator( model_fn=model_fn, model_dir='./models/deepfm', params=model_params, config=config ) data_dir = FLAGS.dataset_path data_files = [] for i in range(FLAGS.dataset_parts): data_files.append(data_dir + 'part-r-{:0>5}'.format(i)) train_files = data_files[:-FLAGS.dataset_eval] eval_files = data_files[-FLAGS.dataset_eval:] # hook = estimator.ProfilerHook(save_steps=10, output_dir='./time/', show_memory=True, show_dataflow=True) if FLAGS.task_type == 'train': train_spec = estimator.TrainSpec(input_fn=lambda: input_fn( train_files, num_epochs=FLAGS.num_epochs, batch_size=FLAGS.batch_size, need_shuffle=True)) eval_spec = estimator.EvalSpec(input_fn=lambda: input_fn( eval_files, num_epochs=1, batch_size=FLAGS.batch_size), steps=None, start_delay_secs=1, throttle_secs=5) start = time() estimator.train_and_evaluate(deepfm, train_spec, eval_spec) elapsed = (time() - start) print("Training time used: {0}ms".format(round(elapsed * 1000, 2))) elif FLAGS.task_type == 'eval': deepfm.evaluate(input_fn=lambda: input_fn(eval_files, num_epochs=1, batch_size=FLAGS.batch_size)) elif FLAGS.task_type == 'predict': p = deepfm.predict(input_fn=lambda: input_fn(eval_files, num_epochs=1, batch_size=FLAGS.batch_size)) print('done predit') # tf.data.Dataset.from_tensor_slices() # for i in p: # print(i["prob"]) feature_description.pop('label') serving_fn = estimator.export.build_parsing_serving_input_receiver_fn(feature_description) # features = { # "u_id": tf.placeholder(dtype=tf.int32, shape=1, name='u_id'), # "i_id": tf.placeholder(dtype=tf.int32, shape=1, name='i_id'), # } # serving_fn = estimator.export.build_raw_serving_input_receiver_fn(features) deepfm.export_savedmodel( export_dir_base=FLAGS.export_path, serving_input_receiver_fn=serving_fn, as_text=True, ) tf.logging.info('Model exported.')