def get_adanet_model():
# Estimator configuration.
runConfig = tf.estimator.RunConfig(save_checkpoints_steps=100,
save_summary_steps=100,
tf_random_seed=RANDOM_SEED)
estimator = adanet.Estimator(
model_dir=OUTPUT_DIR,
# adanet_loss_decay=0.99,
head=tf.contrib.estimator.binary_classification_head(),
subnetwork_generator=simple_dnn.Generator(
learn_mixture_weights=True,
dropout=CONFIG["DROPOUT"],
feature_columns=bidding_data.
get_feature_columns_for_imp_prediction(),
optimizer=tf.train.RMSPropOptimizer(
learning_rate=ADANET_LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=NUM_EPOCHS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(
input_fn=lambda: bidding_data.validation_input_fn_for_predict_imp(
batch_size=BATCH_SIZE, num_epochs=NUM_EPOCHS),
steps=EVAL_STEPS),
config=runConfig)
return estimator
def get_adanet_model():
LEARNING_RATE = 0.003 #@param {type:"number"}
TRAIN_STEPS = NUM_EPOCHS #@param {type:"integer"}
# BATCH_SIZE = 64 #@param {type:"integer"}
ADANET_ITERATIONS = 8 #@param {type:"integer"}
RANDOM_SEED = 42
# Estimator configuration.
runConfig = tf.estimator.RunConfig(save_checkpoints_steps=100,
save_summary_steps=100,
tf_random_seed=RANDOM_SEED)
classifier = estimator = adanet.Estimator(
model_dir=OUTPUT_DIR,
adanet_loss_decay=0.99,
head=tf.contrib.estimator.binary_classification_head(),
subnetwork_generator=simple_dnn.Generator(
learn_mixture_weights=True,
dropout=0.5,
feature_columns=bidding_data.get_feature_columns_for_wr_prediction(
),
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(
input_fn=lambda: bidding_data.validation_input_fn_for_predict_wr(
batch_size=BATCH_SIZE, num_epochs=TRAIN_STEPS),
steps=EVAL_STEPS),
config=runConfig)
return classifier
def train_and_evaluate(x_train, y_train, x_test, y_test,
learning_rate,
train_steps,
batch_size,
learn_mixture_weights,
adanet_lambda, boosting_iterations):
"""Trains an adanet.Estimator` to predict housing prices."""
estimator = adanet.Estimator(
# Since we are predicting housing prices, we'll use a regression
# head that optimizes for MSE.
head=tf.contrib.estimator.regression_head(
loss_reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE),
# Define the generator, which defines our search space of subnetworks
# to train as candidates to add to the final AdaNet model.
subnetwork_generator=subnetworkGenerator.SimpleDNNGenerator(
optimizer=tf.train.RMSPropOptimizer(learning_rate),
learn_mixture_weights=learn_mixture_weights,
seed=utils.RANDOM_SEED),
# Lambda is a the strength of complexity regularization. A larger
# value will penalize more complex subnetworks.
adanet_lambda=adanet_lambda,
# The number of train steps per iteration.
max_iteration_steps=train_steps // boosting_iterations,
# The evaluator will evaluate the model on the full training set to
# compute the overall AdaNet loss (train loss + complexity
# regularization) to select the best candidate to include in the
# final AdaNet model.
evaluator=adanet.Evaluator(
input_fn=utils.input_fn(x_train, y_train, x_test, y_test,
"train", training=False, batch_size=batch_size)),
# The report materializer will evaluate the subnetworks' metrics
# using the full training set to generate the reports that the generator
# can use in the next iteration to modify its search space.
report_materializer=adanet.ReportMaterializer(
input_fn=utils.input_fn(x_train, y_train, x_test, y_test, "train", training=False, batch_size=batch_size)),
# Configuration for Estimators.
config=tf.estimator.RunConfig(
save_checkpoints_steps=50000,
save_summary_steps=50000,
tf_random_seed=utils.RANDOM_SEED))
# Train and evaluate using the tf.estimator tooling.
train_spec = tf.estimator.TrainSpec(
input_fn=utils.input_fn(x_train, y_train, x_test, y_test, "train", training=True, batch_size=batch_size),
max_steps=train_steps)
eval_spec = tf.estimator.EvalSpec(
input_fn=utils.input_fn(x_train, y_train, x_test, y_test, "test", training=False, batch_size=batch_size),
steps=None)
return tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def cnn_ada():
print("==============================================")
start = datetime.datetime.now()
print("Start Train Adanet with [CNN Model] on Mnist at %s" %
time_str(start))
print("- - - - - - - - - - - - - - - - - - - - - - - -")
LEARNING_RATE = 0.05 # @param {type:"number"}
TRAIN_STEPS = 5000 # @param {type:"integer"}
BATCH_SIZE = 64 # @param {type:"integer"}
ADANET_ITERATIONS = 2 # @param {type:"integer"}
model_dir = os.path.join(LOG_DIR, "cnn_%s" % time_str(start))
config = tf.estimator.RunConfig(save_checkpoints_steps=50000,
save_summary_steps=50000,
tf_random_seed=RANDOM_SEED,
model_dir=model_dir)
max_iteration_steps = TRAIN_STEPS // ADANET_ITERATIONS
estimator = adanet.Estimator(
head=head,
subnetwork_generator=SimpleCNNGenerator(
learning_rate=LEARNING_RATE,
max_iteration_steps=max_iteration_steps,
seed=RANDOM_SEED),
max_iteration_steps=max_iteration_steps,
evaluator=adanet.Evaluator(input_fn=input_fn("train",
training=False,
batch_size=BATCH_SIZE),
steps=None),
report_materializer=adanet.ReportMaterializer(input_fn=input_fn(
"train", training=False, batch_size=BATCH_SIZE),
steps=None),
adanet_loss_decay=.99,
config=config)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(input_fn=input_fn(
"train", training=True, batch_size=BATCH_SIZE),
max_steps=TRAIN_STEPS),
eval_spec=tf.estimator.EvalSpec(input_fn=input_fn(
"test", training=False, batch_size=BATCH_SIZE),
steps=None))
print("Accuracy:", results["accuracy"])
print("Loss:", results["average_loss"])
end = datetime.datetime.now()
print("Training end at %s" % time_str(end))
print("Time Spend %s" % str(end - start))
print("==============================================")
def train_and_evaluate(experiment_name,
learn_mixture_weights=LEARN_MIXTURE_WEIGHTS,
adanet_lambda=ADANET_LAMBDA):
"""Trains an `adanet.Estimator` to predict housing prices."""
model_dir = os.path.join(LOG_DIR, experiment_name)
estimator = adanet.Estimator(
# Since we are predicting housing prices, we'll use a regression
# head that optimizes for MSE.
head=tf.contrib.estimator.regression_head(
loss_reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE),
# Define the generator, which defines our search space of subnetworks
# to train as candidates to add to the final AdaNet model
subnetwork_generator=SimpleDNNGenerator(
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
learn_mixture_weights=learn_mixture_weights,
seed=RANDOM_SEED),
# Lambda is a the strength of complexity regularization. A larger
# value will penalize more complex subnetworks.
adanet_lambda=adanet_lambda,
# The number of train steps per iteration.
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
# The evaluator will evaluate the model on the full training set to
# compute the overall AdaNet loss (train loss + complexity
# regularization) to select the best candidate to include in the
# final AdaNet model.
evaluator=adanet.Evaluator(
input_fn=input_fn("train", training=False, batch_size=BATCH_SIZE)),
# Configuration for Estimators
config=tf.estimator.RunConfig(save_summary_steps=5000,
save_checkpoints_steps=5000,
tf_random_seed=RANDOM_SEED,
model_dir=model_dir))
# Train and evaluate using using the tf.estimator tooling.
train_spec = tf.estimator.TrainSpec(input_fn=input_fn(
"train", training=True, batch_size=BATCH_SIZE),
max_steps=TRAIN_STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=input_fn("test", training=False, batch_size=BATCH_SIZE),
steps=None,
start_delay_secs=1,
throttle_secs=30,
)
return tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def create_estimator(
self,
subnetwork_generator,
adanet_iterations: int,
evaluator: Evaluator,
config: tf.estimator.RunConfig,
) -> tf.estimator.Estimator:
return adanet.Estimator(
head=self.head,
subnetwork_generator=subnetwork_generator,
max_iteration_steps=self.train_steps // adanet_iterations,
evaluator=evaluator,
config=config,
)
def dnn_ada():
print("==============================================")
start = datetime.datetime.now()
print("Start Train Adanet with [DNN Model] on Mnist at %s" %
time_str(start))
print("- - - - - - - - - - - - - - - - - - - - - - - -")
LEARNING_RATE = 0.003
TRAIN_STEPS = 5000
BATCH_SIZE = 64
ADANET_ITERATIONS = 2
model_dir = os.path.join(LOG_DIR, "dnn_%s" % time_str(start))
config = tf.estimator.RunConfig(save_checkpoints_steps=50000,
save_summary_steps=50000,
tf_random_seed=RANDOM_SEED,
model_dir=model_dir)
estimator = adanet.Estimator(
head=head,
subnetwork_generator=simple_dnn.Generator(
feature_columns=feature_columns,
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(input_fn=input_fn("train",
training=False,
batch_size=BATCH_SIZE),
steps=None),
config=config)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(input_fn=input_fn(
"train", training=True, batch_size=BATCH_SIZE),
max_steps=TRAIN_STEPS),
eval_spec=tf.estimator.EvalSpec(input_fn=input_fn(
"test", training=False, batch_size=BATCH_SIZE),
steps=None))
print("Accuracy:", results["accuracy"])
print("Loss:", results["average_loss"])
end = datetime.datetime.now()
print("Training end at %s" % time_str(end))
print("Time Spend %s" % str(end - start))
print("==============================================")
def get_estimator(self):
estimator = adanet.Estimator(
head=self.head,
subnetwork_generator=simple_dnn.Generator(
feature_columns=self.feature_columns,
optimizer=tf.train.RMSPropOptimizer(
learning_rate=self.LEARNING_RATE),
seed=self.RANDOM_SEED),
max_iteration_steps=self.TRAIN_STEPS // self.ADANET_ITERATIONS,
evaluator=adanet.Evaluator(input_fn=self.input_fn(
"train",
training=False,
batch_size=self.BATCH_SIZE,
RANDOM_SEED=self.RANDOM_SEED),
steps=None),
config=self.config)
return estimator
def train_and_evaluate(output_neurons, LEARNING_RATE,
learn_mixture_weights, adanet_lambda,
max_iteration_steps, TRAIN_STEPS, BATCH_SIZE,
RANDOM_SEED, train_test, model_dir):
AdaNetEstimator = adanet.Estimator(
# head instance computes loss and evaluation metrics for every candidate
head=ah.head_classify(output_neurons),
# defines candidate subnetworks to train and evaluate at every AdaNet iteration
subnetwork_generator=SimpleDNNGenerator(
optimizer=tf.train.AdamOptimizer(
learning_rate=LEARNING_RATE), # adagrad
layer_size=ah.hidden_layer_neurons(2, train_test),
learn_mixture_weights=learn_mixture_weights,
seed=RANDOM_SEED),
adanet_lambda=adanet_lambda,
max_iteration_steps=max_iteration_steps,
# reports are made available to subnetwork_generator in next iteration report_materializer=
# adanet.ReportMaterializer(input_fn=ah.input_fn("train", training=False, batch_size=BATCH_SIZE,
# train_test=train_test)),
config=tf.estimator.RunConfig(
# save_checkpoints_steps=50000,
# save_summary_steps=50000,
tf_random_seed=RANDOM_SEED,
model_dir=model_dir))
# Determines input data for training
train_spec = tf.estimator.TrainSpec(input_fn=ah.input_fn(
"train",
training=True,
batch_size=BATCH_SIZE,
train_test=train_test),
max_steps=TRAIN_STEPS)
# Combines evaluation metrics of trained models
eval_spec = tf.estimator.EvalSpec(input_fn=ah.input_fn(
"test",
training=False,
batch_size=BATCH_SIZE,
train_test=train_test),
steps=None)
return tf.estimator.train_and_evaluate(AdaNetEstimator, train_spec,
eval_spec)
def get_adanet_model():
# Estimator configuration.
# distribution_strategy = tf.contrib.distribute.MirroredStrategy()
session_config = tf.ConfigProto(log_device_placement=True)
session_config.gpu_options.allow_growth = True
session_config.gpu_options.per_process_gpu_memory_fraction = 0.8
runConfig = tf.estimator.RunConfig(
# train_distribute=distribution_strategy,
# eval_distribute=distribution_strategy,
session_config=session_config,
save_checkpoints_steps=100,
save_summary_steps=100,
tf_random_seed=RANDOM_SEED)
estimator = adanet.Estimator(
model_dir=OUTPUT_DIR,
# metric_fn=custom_metrics,
# adanet_loss_decay=0.99,
head=tf.contrib.estimator.multi_label_head(
name="name",
n_classes=len(CONFIG['LABELS']),
# classes_for_class_based_metrics= [5,6]
),
subnetwork_generator=simple_dnn.Generator(
learn_mixture_weights=True,
dropout=CONFIG["DROPOUT"],
feature_columns=data.get_feature_columns(),
optimizer=tf.train.AdamOptimizer(
learning_rate=ADANET_LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=NUM_EPOCHS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(input_fn=lambda: data.validation_input_fn(
batch_size=BATCH_SIZE, num_epochs=NUM_EPOCHS),
steps=EVAL_STEPS),
config=runConfig)
return estimator
print("Training Done.")
print("==" * 20)
print("Accuracy: ", results["accuracy"])
print(" Loss: ", results["loss"])
print("==" * 20)
tf.compat.v1.logging.info("Start to evaluate model with test data.")
tf.compat.v1.logging.info("Restore model.")
network_generator_cls = train.MODEL_MAPPER[args.experiment_name]
network_generator = network_generator_cls(hparams.learning_rate)
estimator = adanet.Estimator(
head=AdanetTrainer(1000).head,
max_iteration_steps=500,
subnetwork_generator=network_generator.build_subnetwork_generator(
hparams.unflatten),
model_dir=args.model_dir,
)
test_dataset = MNISTDataset(tf.estimator.ModeKeys.PREDICT, args.dataset_id)
tf.compat.v1.logging.info("Prepare test data.")
test_predictions = estimator.predict(
test_dataset.get_input_fn(hparams.batch_size))
test_predictions = [pred for pred in test_predictions]
class_ids = np.array([r["class_ids"].item() for r in test_predictions])
answers = test_dataset.data[1]
test_accuracy = (class_ids == answers) / len(answers)
#@title Parameters
LEARNING_RATE = 0.05 #@param {type:"number"}
TRAIN_STEPS = 8000 #@param {type:"integer"}
BATCH_SIZE = 3600 #@param {type:"integer"}
ADANET_ITERATIONS = 3 #@param {type:"integer"}
max_iteration_steps = TRAIN_STEPS // ADANET_ITERATIONS
estimator = adanet.Estimator(
head=head,
subnetwork_generator=SimpleCNNGenerator(
learning_rate=LEARNING_RATE,
max_iteration_steps=max_iteration_steps,
seed=RANDOM_SEED),
max_iteration_steps=max_iteration_steps,
evaluator=adanet.Evaluator(
input_fn=input_fn("train", training=False, batch_size=BATCH_SIZE),
steps=None),
report_materializer=adanet.ReportMaterializer(
input_fn=input_fn("train", training=False, batch_size=BATCH_SIZE),
steps=None),
adanet_loss_decay=.99,
config=config)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(
input_fn=input_fn("train", training=True, batch_size=BATCH_SIZE),
max_steps=TRAIN_STEPS),
eval_spec=tf.estimator.EvalSpec(
input_fn=input_fn("test", training=False, batch_size=BATCH_SIZE),
def run_adanet():
EPOCHS = 10
BATCH_SIZE = 32
#x_train, y_train,x_test = load_images()
x_train, y_train, x_test = load_dog_breed()
y_train = pd.get_dummies(y_train, sparse=True)
y_train = np.asarray(y_train)
x_train = x_train / 255 # map values between 0 and 1
x_test = x_test / 255 # map values between 0 and 1
#x_train = x_train.astype(np.float32) # cast values to float32
#x_test = x_test.astype(np.float32) # cast values to float32
#y_train = y_train.astype(np.int32) # cast values to int32
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": x_train},
y=y_train,
batch_size=BATCH_SIZE,
num_epochs=EPOCHS,
shuffle=False)
adanet_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": x_train},
y=y_train,
batch_size=BATCH_SIZE,
num_epochs=1,
shuffle=False)
test_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": x_test},
batch_size=BATCH_SIZE,
num_epochs=1,
shuffle=False)
head = tf.contrib.estimator.multi_class_head(120)
estimator = adanet.Estimator(head=head,
subnetwork_generator=CNNGenerator(),
max_iteration_steps=200,
evaluator=adanet.Evaluator(
input_fn=adanet_input_fn, steps=None),
adanet_loss_decay=.99)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(input_fn=train_input_fn,
max_steps=200),
eval_spec=tf.estimator.EvalSpec(input_fn=train_input_fn, steps=None))
predictions = estimator.predict(input_fn=test_input_fn)
preds = list()
for i, val in enumerate(predictions):
predicted_class = val['class_ids'][0]
print(val['probabilities'])
preds.append(predicted_class)
prediction_confidence = val['probabilities'][predicted_class] * 100
print("Accuracy:", results["accuracy"])
print("Loss:", results["average_loss"])
id_test = pd.read_csv("datasets/aerial-cactus/sample_submission.csv")
submission = pd.DataFrame({"id": id_test.id.values, "has_cactus": preds})
submission.to_csv("invasive_adanet_submission.csv", index=False)
def map_fun(args, ctx):
from datetime import datetime
import tensorflow as tf
import os
import time
import json
import adanet
from adanet.examples import simple_dnn
worker_num = ctx.worker_num
job_name = ctx.job_name
task_index = ctx.task_index
message = "worker_num: {0}, job_name: {1}, task_index: {2}".format(
worker_num, job_name, task_index)
print(message)
input_dim = int(args.input_dim)
batch_size = args.batch_size
# Fix Random Seed
RANDOM_SEED = 42
FEATURES_KEY = "features"
NUM_CLASSES = 2
loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE
# head = tf.contrib.estimator.multi_class_head(NUM_CLASSES, loss_reduction=loss_reduction)
head = tf.contrib.estimator.binary_classification_head(
loss_reduction=loss_reduction)
# numeric_column do not support SparseTensor
feature_columns = [
tf.feature_column.numeric_column(key=FEATURES_KEY, shape=[input_dim])
]
log_dir = ctx.absolute_path(args.log_dir)
export_dir = ctx.absolute_path(args.export_dir)
pred_dir = ctx.absolute_path(args.prediction_dir)
print("tensorflow log path: {0}".format(log_dir))
print("tensorflow export path: {0}".format(export_dir))
print("tensorflow prediction path: {0}".format(pred_dir))
def generator(ln):
splits = tf.string_split([ln], delimiter=" ")
label = splits.values[0]
label = tf.string_to_number(label, tf.float64)
label = tf.cond(
label >= 1.0,
lambda: tf.constant(1, shape=[1], dtype=tf.float32),
lambda: tf.constant(0, shape=[1], dtype=tf.float32),
)
# SparseTensor output
col_val = tf.string_split(splits.values[1::], delimiter=":")
col = tf.string_to_number(col_val.values[0::2], tf.int64) - 1
vals = col_val.values[1::2]
vals = tf.string_to_number(vals, tf.float32)
# Filter the features which occurs few than given input_dim
vals = tf.boolean_mask(vals, col < input_dim)
col = tf.boolean_mask(col, col < input_dim)
row = tf.cast(tf.fill(tf.shape(col), 0), tf.int64, name="row_cast")
row_col = tf.transpose(tf.stack([row, col]), name="row_col_transpose")
sparse = tf.SparseTensor(row_col, vals, (1, input_dim))
# convert to dense，191106 必须转
features = {FEATURES_KEY: tf.sparse_tensor_to_dense(sparse)}
return features, label
def new_input_fn(partition, training):
def _input_fn():
# path is ok
parse_fn = generator
if partition == "train":
data_dir = ctx.absolute_path(args.data_dir)
file_pattern = os.path.join(data_dir, "part-*")
ds = tf.data.Dataset.list_files(file_pattern, shuffle=False)
ds = ds.apply(
tf.contrib.data.parallel_interleave(
tf.data.TextLineDataset, cycle_length=10))
ds = ds.map(parse_fn, num_parallel_calls=5)
if training:
ds = ds.shuffle(batch_size * 5).repeat()
else:
data_dir = ctx.absolute_path(args.test_dir)
file_pattern = os.path.join(data_dir, "part-*")
ds = tf.data.Dataset.list_files(file_pattern, shuffle=False)
ds = ds.apply(
tf.contrib.data.parallel_interleave(
tf.data.TextLineDataset, cycle_length=10))
ds = ds.map(parse_fn, num_parallel_calls=5)
iterator = ds.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
# ds = ds.apply(tf.contrib.data.batch_and_drop_remainder(batch_size))
# return ds.batch(batch_size)
return _input_fn
print("========= Start Training")
LEARNING_RATE = 0.01
TRAIN_STEPS = 3000
ADANET_ITERATIONS = 3 # AKA Boosting Iteration
# 控制模型复杂度
ADANET_LAMBDA = 0.1
LEARN_MIXTURE_WEIGHTS = False
#strategy = adanet.distributed.RoundRobinStrategy()
# 191125 这里一定要设置
tfc = json.dumps({
"cluster": ctx.cluster_spec,
"task": {
"type": job_name,
"index": task_index
}
})
os.environ["TF_CONFIG"] = tfc
# 191127 尝试不用 device_filter，用了 strategy 后会自动设置为 /job:ps，不需要时候手动设置
config = tf.estimator.RunConfig(
save_checkpoints_steps=5000,
tf_random_seed=RANDOM_SEED,
model_dir=log_dir,
)
# config = tf.estimator.RunConfig(
# save_checkpoints_steps=5000,
# tf_random_seed=RANDOM_SEED,
# model_dir=logdir,
# session_config=tf.ConfigProto(
# log_device_placement=False, device_filters=["/job:ps"]
# ),
# )
# BaseLine Linear
# estimator = tf.estimator.LinearClassifier(
# feature_columns=feature_columns,
# n_classes=NUM_CLASSES,
# optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
# loss_reduction=loss_reduction,
# config=config
# )
# DNN TEST - ADANET
estimator = adanet.Estimator(
head=head,
force_grow=True,
subnetwork_generator=simple_dnn.Generator(
layer_size=128,
initial_num_layers=2,
dropout=0.2,
feature_columns=feature_columns,
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
learn_mixture_weights=LEARN_MIXTURE_WEIGHTS,
seed=RANDOM_SEED,
),
adanet_lambda=ADANET_LAMBDA,
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
#evaluator=adanet.Evaluator(input_fn=new_input_fn("test", False)),
evaluator=adanet.Evaluator(input_fn=new_input_fn("test", False),
steps=1000),
config=config,
#experimental_placement_strategy=strategy,
# 记录 report，实际上没啥用
# report_materializer=adanet.ReportMaterializer(
# input_fn=new_input_fn("train", False),
# ),
)
# 尝试不 return 任何东西，只是计算
tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(input_fn=new_input_fn("train", True),
max_steps=TRAIN_STEPS),
# 这里的 Eval 在分布式场景下，实际上并没有任何作用
eval_spec=tf.estimator.EvalSpec(
input_fn=new_input_fn("test", False),
steps=None,
start_delay_secs=1,
throttle_secs=30,
),
)
# 最后一轮只训练，模型参数会保存到 model.ckpt，并不会再为下一轮去做准备
# 参考 https://github.com/tensorflow/adanet/blob/master/adanet/core/estimator_test.py
# line 2362 def test_export_saved_model_always_uses_replication_placement(self):
def serving_input_receiver_fn():
serialized_sample = tf.compat.v1.placeholder(dtype=tf.float32,
shape=[None, input_dim],
name='features')
tensor_features = {'features': serialized_sample}
return tf.estimator.export.ServingInputReceiver(
features=tensor_features, receiver_tensors=serialized_sample)
# 在 RoundRobinStrategy 下无法执行
if ctx.job_name == "chief":
# 进行预测，分别是测试和训练
print('export test result')
predictions = estimator.predict(new_input_fn("test", False))
print('Writing Predictions to {}'.format(pred_dir))
tf.gfile.MakeDirs(pred_dir)
with tf.gfile.GFile("{}/test".format(pred_dir), 'w') as f:
for pred in predictions:
f.write(str(pred))
f.write('\n')
print('export train result')
predictions = estimator.predict(new_input_fn("train", False))
print('Writing Predictions to {}'.format(pred_dir))
tf.gfile.MakeDirs(pred_dir)
with tf.gfile.GFile("{}/train".format(pred_dir), 'w') as f:
for pred in predictions:
f.write(str(pred))
f.write('\n')
# 导出模型
estimator.export_saved_model(
export_dir,
serving_input_receiver_fn,
experimental_mode=tf.estimator.ModeKeys.PREDICT)
def build_adanet_estimator(subnetwork_generator):
'''Build ADANET estimator.
Input :
* subnetwork_generator : subnetwork generator that generates a candidate
that is provided to ADANET weaklearner algorithm.
* nn_type : type of neural network candidate to be generated. It is used
to select mixture weights type : scalar, vector or matrix.
* feature_shape : shape of features.
Output :
* adanet_estimator : ADANET estimator instance of adanet.Estimator class.
'''
output_dir = './tmp/adanet'
#---------------------------------------------------------------------------
# Get parameters from configuration file.
#---------------------------------------------------------------------------
dict_adanet_config = p8_util_config.dict_adanet_config
feature_shape = dict_adanet_config['adanet_feature_shape']
dict_nn_layer_config = dict_adanet_config['adanet_nn_layer_config']
nn_type = dict_nn_layer_config['nn_type']
#---------------------------------------------------------------------------
# Fixe mixture weights
#---------------------------------------------------------------------------
if 'RNN' == nn_type :
mixture_weight_type=MixtureWeightType.VECTOR
else :
mixture_weight_type=MixtureWeightType.MATRIX
ensembler = ComplexityRegularizedEnsembler(mixture_weight_type=mixture_weight_type\
, adanet_lambda=p8_util_config.dict_adanet_config['adanet_lambda'])
dataset_type = p8_util_config.DATASET_TYPE
input_fn_param={'num_epochs':p8_util_config.NUM_EPOCHS,\
'batch_size':p8_util_config.BATCH_SIZE,\
'feature_shape': feature_shape,\
'dataset_type': dataset_type
}
train_input_fn=input_fn_2("train", input_fn_param)
model_name = build_model_name(nn_type)
adanet_estimator_config, output_dir_log= make_config(model_name\
, output_dir=output_dir\
, is_restored=False)
nb_class = p8_util_config.dict_adanet_config['adanet_nn_layer_config']['nn_logit_dimension']
feature_columns, loss_reduction, tf_head \
= get_tf_head(feature_shape, nb_class, nn_type=nn_type, feature_shape=feature_shape)
p8_util_config.dict_adanet_config['adanet_feature_columns'] = feature_columns
p8_util_config.dict_adanet_config['adanet_feature_shape'] = feature_shape
adanet_estimator = adanet.Estimator(
head=p8_util_config.dict_adanet_config['adanet_tf_head'],
subnetwork_generator=subnetwork_generator,
ensemblers = [ensembler],
max_iteration_steps=p8_util_config.ADANET_MAX_ITERATION_STEPS,
evaluator=adanet.Evaluator(
input_fn=train_input_fn,
steps=None),
config= adanet_estimator_config)
return adanet_estimator, output_dir_log
def map_fun(args, ctx):
from datetime import datetime
import tensorflow as tf
import os
import time
import json
import adanet
from adanet.examples import simple_dnn
worker_num = ctx.worker_num
job_name = ctx.job_name
task_index = ctx.task_index
message = "worker_num: {0}, job_name: {1}, task_index: {2}".format(
worker_num, job_name, task_index)
print(message)
input_dim = int(args.input_dim)
batch_size = args.batch_size
# Fix Random Seed
RANDOM_SEED = 42
FEATURES_KEY = "features"
loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE
def weighted_cross_entropy_with_logits(labels, logits):
return tf.nn.weighted_cross_entropy_with_logits(targets=labels,
logits=logits,
pos_weight=4)
head = tf.contrib.estimator.binary_classification_head(
loss_reduction=loss_reduction,
loss_fn=weighted_cross_entropy_with_logits)
# numeric_column do not support SparseTensor
feature_columns = [
tf.feature_column.numeric_column(key=FEATURES_KEY, shape=[input_dim])
]
log_dir = ctx.absolute_path(args.log_dir)
export_dir = ctx.absolute_path(args.export_dir)
pred_dir = ctx.absolute_path(args.prediction_dir)
print("tensorflow log path: {0}".format(log_dir))
print("tensorflow export path: {0}".format(export_dir))
print("tensorflow prediction path: {0}".format(pred_dir))
def generator(ln):
splits = tf.string_split([ln], delimiter=" ")
label = splits.values[0]
label = tf.string_to_number(label, tf.float64)
label = tf.cond(
label >= 1.0,
lambda: tf.constant(1, shape=[1], dtype=tf.float32),
lambda: tf.constant(0, shape=[1], dtype=tf.float32),
)
# SparseTensor output
col_val = tf.string_split(splits.values[1::], delimiter=":")
col = tf.string_to_number(col_val.values[0::2], tf.int64) - 1
vals = col_val.values[1::2]
vals = tf.string_to_number(vals, tf.float32)
# Filter the features which occurs few than given input_dim
vals = tf.boolean_mask(vals, col < input_dim)
col = tf.boolean_mask(col, col < input_dim)
row = tf.cast(tf.fill(tf.shape(col), 0), tf.int64, name="row_cast")
row_col = tf.transpose(tf.stack([row, col]), name="row_col_transpose")
sparse = tf.SparseTensor(row_col, vals, (1, input_dim))
# convert to dense，191106 必须转
features = {FEATURES_KEY: tf.sparse_tensor_to_dense(sparse)}
return features, label
def new_input_fn(partition, training):
def _input_fn():
# path is ok
parse_fn = generator
if partition == "train":
data_dir = ctx.absolute_path(args.data_dir)
file_pattern = os.path.join(data_dir, "part-*")
ds = tf.data.Dataset.list_files(file_pattern, shuffle=False)
ds = ds.apply(
tf.contrib.data.parallel_interleave(
tf.data.TextLineDataset, cycle_length=10))
ds = ds.map(parse_fn, num_parallel_calls=5)
if training:
ds = ds.shuffle(batch_size * 5).repeat()
else:
data_dir = ctx.absolute_path(args.test_dir)
file_pattern = os.path.join(data_dir, "part-*")
ds = tf.data.Dataset.list_files(file_pattern, shuffle=False)
ds = ds.apply(
tf.contrib.data.parallel_interleave(
tf.data.TextLineDataset, cycle_length=10))
ds = ds.map(parse_fn, num_parallel_calls=5)
iterator = ds.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
# ds = ds.apply(tf.contrib.data.batch_and_drop_remainder(batch_size))
# return ds.batch(batch_size)
return _input_fn
print("========= Start Training")
LEARNING_RATE = 0.01
TRAIN_STEPS = 1000
ADANET_ITERATIONS = 4 # AKA Boosting Iteration
# 控制模型复杂度
ADANET_LAMBDA = 0.1
LEARN_MIXTURE_WEIGHTS = False
#strategy = adanet.distributed.RoundRobinStrategy()
# 191125 这里一定要设置
tfc = json.dumps({
"cluster": ctx.cluster_spec,
"task": {
"type": job_name,
"index": task_index
}
})
os.environ["TF_CONFIG"] = tfc
# 191127 尝试不用 device_filter，用了 strategy 后会自动设置为 /job:ps，不需要时候手动设置
config = tf.estimator.RunConfig(
save_checkpoints_steps=5000,
tf_random_seed=RANDOM_SEED,
model_dir=log_dir,
)
# estimator = tf.estimator.LinearEstimator(
# head=head,
# feature_columns=feature_columns,
# config=config
#
# )
# config = tf.estimator.RunConfig(
# save_checkpoints_steps=5000,
# tf_random_seed=RANDOM_SEED,
# model_dir=logdir,
# session_config=tf.ConfigProto(
# log_device_placement=False, device_filters=["/job:ps"]
# ),
# )
# DNN TEST - ADANET
estimator = adanet.Estimator(
head=head,
force_grow=False,
subnetwork_generator=simple_dnn.Generator(
layer_size=128,
initial_num_layers=1,
dropout=0.2,
feature_columns=feature_columns,
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
learn_mixture_weights=LEARN_MIXTURE_WEIGHTS,
seed=RANDOM_SEED,
),
adanet_lambda=ADANET_LAMBDA,
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(input_fn=new_input_fn("test", False),
steps=1000),
config=config,
)
# ensemble_estimator = adanet.AutoEnsembleEstimator(
# head=head,
# candidate_pool= lambda config: {
# "linear1":
# tf.estimator.LinearEstimator(
# head=head,
# feature_columns=feature_columns,
# optimizer=tf.train.RMSPropOptimizer(learning_rate=0.1),
# config=config,
# ),
# "dnn1":
# tf.estimator.DNNEstimator(
# head=head,
# feature_columns=feature_columns,
# optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001),
# hidden_units=[512, 256, 128],
# config=config,
# ),
# "dnn2":
# tf.estimator.DNNEstimator(
# head=head,
# feature_columns=feature_columns,
# optimizer=tf.train.RMSPropOptimizer(learning_rate=0.01),
# hidden_units=[256, 128],
# config=config,
# ),
# "dnn_linear":
# tf.estimator.DNNLinearCombinedEstimator(
# head=head,
# dnn_feature_columns=feature_columns,
# linear_feature_columns=feature_columns,
# dnn_hidden_units=[512, 256, 128],
# config=config,
# )
# },
# max_iteration_steps=100,
# )
cur_e = estimator
# 尝试不 return 任何东西，只是计算
tf.estimator.train_and_evaluate(
cur_e,
train_spec=tf.estimator.TrainSpec(input_fn=new_input_fn("train", True),
max_steps=TRAIN_STEPS),
# 这里的 Eval 在分布式场景下，实际上并没有任何作用
eval_spec=tf.estimator.EvalSpec(
input_fn=new_input_fn("test", False),
steps=None,
start_delay_secs=1,
throttle_secs=30,
),
)
# 最后一轮只训练，模型参数会保存到 model.ckpt，并不会再为下一轮去做准备
# 这样的保存方式，需要输入是一个 example，不适合 DSP 的输入
# feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
# serving_input_receiver_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
def serving_input_receiver_fn():
indices = tf.placeholder(dtype=tf.int64,
shape=[None, None],
name='indices')
values = tf.placeholder(dtype=tf.float32, shape=[None], name='values')
shape = tf.placeholder(dtype=tf.int64,
shape=[None],
name='dense_shape')
receiver_input = {
'indices': indices,
'values': values,
'dense_shape': shape
}
# 先构成 sparse，然后 sparse_to_dense
sparse = tf.SparseTensor(indices, values, shape)
features = {FEATURES_KEY: tf.sparse_tensor_to_dense(sparse)}
return tf.estimator.export.ServingInputReceiver(
features, receiver_input)
# 在 RoundRobinStrategy 下无法执行
if ctx.job_name == "chief":
# 进行 evaluate，比较慢，跳过
# predictions = cur_e.predict(new_input_fn("test", False))
# result = cur_e.evaluate(new_input_fn("test", False))
# with tf.gfile.GFile("{}/evaluate".format(log_dir), 'w') as f:
# f.write(str(result))
# f.write('\n')
# 进行预测，分别是测试和训练
# print('export test result')
# predictions = estimator.predict(new_input_fn("test", False))
# print('Writing Predictions to {}'.format(pred_dir))
# tf.gfile.MakeDirs(pred_dir)
# with tf.gfile.GFile("{}/test".format(pred_dir), 'w') as f:
# for pred in predictions:
# f.write(str(pred))
# f.write('\n')
# print('export train result')
# predictions = estimator.predict(new_input_fn("train", False))
# print('Writing Predictions to {}'.format(pred_dir))
# tf.gfile.MakeDirs(pred_dir)
# with tf.gfile.GFile("{}/train".format(pred_dir), 'w') as f:
# for pred in predictions:
# f.write(pred['classes'][0])
# f.write('\n')
# 导出模型
# 191204 这样导出没有办法指定 serving 时的输出，
cur_e.export_saved_model(export_dir, serving_input_receiver_fn)
def map_fun_v2(args, ctx):
from datetime import datetime
import tensorflow as tf
import time
worker_num = ctx.worker_num
job_name = ctx.job_name
task_index = ctx.task_index
# Parameters
IMAGE_PIXELS = 28
hidden_units = 128
# Fix Random Seed
RANDOM_SEED = 42
(x_train, y_train), (x_test,
y_test) = (tf.keras.datasets.mnist.load_data())
FEATURES_KEY = "images"
NUM_CLASSES = 10
loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE
head = tf.contrib.estimator.multi_class_head(NUM_CLASSES,
loss_reduction=loss_reduction)
feature_columns = [
tf.feature_column.numeric_column(FEATURES_KEY, shape=[28, 28, 1])
]
# Get TF cluster and server instances
cluster, server = ctx.start_cluster_server(1, args.rdma)
def generator(images, labels):
"""Returns a generator that returns image-label pairs."""
def _gen():
for image, label in zip(images, labels):
yield image, label
return _gen
def preprocess_image(image, label):
"""Preprocesses an image for an `Estimator`."""
image = image / 255.
image = tf.reshape(image, [28, 28, 1])
features = {FEATURES_KEY: image}
return features, label
def input_fn(partition, training):
"""Generate an input_fn for the Estimator."""
def _input_fn():
if partition == "train":
dataset = tf.data.Dataset.from_generator(
generator(x_train, y_train), (tf.float32, tf.int32),
((28, 28), ()))
else:
dataset = tf.data.Dataset.from_generator(
generator(x_test, y_test), (tf.float32, tf.int32),
((28, 28), ()))
if training:
dataset = dataset.shuffle(10 * args.batch_size,
seed=RANDOM_SEED).repeat()
dataset = dataset.map(preprocess_image).batch(args.batch_size)
iterator = dataset.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
return _input_fn
if job_name == "ps":
server.join()
elif job_name == "worker":
# Assigns ops to the local worker by default
# 这里的日志都是看不到的
message = ""
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % task_index,
cluster=cluster)):
print("========= Start Training")
LEARNING_RATE = 0.003
TRAIN_STEPS = 5000
BATCH_SIZE = 64
ADANET_ITERATIONS = 2
logdir = ctx.absolute_path(args.model)
config = tf.estimator.RunConfig(save_checkpoints_steps=50000,
save_summary_steps=50000,
tf_random_seed=RANDOM_SEED,
model_dir=logdir)
# 先测试下线性模型
# estimator = tf.estimator.LinearClassifier(
# feature_columns=feature_columns,
# n_classes=NUM_CLASSES,
# optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
# loss_reduction=loss_reduction,
# config=config
# )
estimator = adanet.Estimator(
head=head,
subnetwork_generator=simple_dnn.Generator(
feature_columns=feature_columns,
optimizer=tf.train.RMSPropOptimizer(
learning_rate=LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(input_fn=input_fn("train",
training=False),
steps=None),
config=config)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(input_fn=input_fn(
"train", training=True),
max_steps=TRAIN_STEPS),
eval_spec=tf.estimator.EvalSpec(input_fn=input_fn(
"test", training=False),
steps=None))
print("Accuracy:", results["accuracy"])
print("Loss:", results["average_loss"])
message = "Accuracy: {}; Loss: {}".format(results["accuracy"],
results["average_loss"])
print("==============================================")
print("{} stopping MonitoredTrainingSession".format(
datetime.now().isoformat()))
# WORKAROUND FOR https://github.com/tensorflow/tensorflow/issues/21745
# wait for all other nodes to complete (via done files)
done_dir = "{}/{}/done".format(ctx.absolute_path(args.model),
args.mode)
print("Writing done file to: {}".format(done_dir))
tf.gfile.MakeDirs(done_dir)
with tf.gfile.GFile("{}/{}".format(done_dir, ctx.task_index),
'w') as done_file:
done_file.write("done")
done_file.write(message)
for i in range(60):
if len(tf.gfile.ListDirectory(done_dir)) < len(
ctx.cluster_spec['worker']):
print("{} Waiting for other nodes {}".format(
datetime.now().isoformat(), i))
time.sleep(1)
else:
print("{} All nodes done".format(datetime.now().isoformat()))
break
def map_fun(args, ctx):
from datetime import datetime
import tensorflow as tf
import os
import time
worker_num = ctx.worker_num
job_name = ctx.job_name
task_index = ctx.task_index
message = 'worker_num: {0}, job_name: {1}, task_index: {2}'.format(worker_num, job_name, task_index)
input_dim = int(args.input_dim)
batch_size = args.batch_size
# Fix Random Seed
RANDOM_SEED = 42
FEATURES_KEY = "ctr"
NUM_CLASSES = 2
loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE
# head = tf.contrib.estimator.multi_class_head(NUM_CLASSES, loss_reduction=loss_reduction)
head = tf.contrib.estimator.binary_classification_head(loss_reduction=loss_reduction)
# 用 numeric_column 是不支持 SparseTensor 的
feature_columns = [
tf.feature_column.numeric_column(FEATURES_KEY, shape=[input_dim])
]
log_dir = ctx.absolute_path(args.log_dir)
export_dir = ctx.absolute_path(args.export_dir)
print("tensorflow log path: {0}".format(log_dir))
print("tensorflow export path: {0}".format(export_dir))
# Get TF cluster and server instances
cluster, server = ctx.start_cluster_server(1, args.rdma)
def generator(ln):
splits = tf.string_split([ln], delimiter=' ')
label = splits.values[0]
label = tf.string_to_number(label, tf.float64)
label = tf.cond(label >= 1.0,
lambda: tf.constant(1, shape=[1], dtype=tf.float32),
lambda: tf.constant(0, shape=[1], dtype=tf.float32))
# SparseTensor output
col_val = tf.string_split(splits.values[1::], delimiter=':')
col = tf.string_to_number(col_val.values[0::2], tf.int64) - 1
vals = col_val.values[1::2]
vals = tf.string_to_number(vals, tf.float32)
# Filter the features which occurs few than given input_dim
vals = tf.boolean_mask(vals, col < input_dim)
col = tf.boolean_mask(col, col < input_dim)
row = tf.cast(tf.fill(tf.shape(col), 0), tf.int64, name='row_cast')
row_col = tf.transpose(tf.stack([row, col]), name='row_col_transpose')
sparse = tf.SparseTensor(row_col, vals, (1, input_dim))
# 转换成 dense
features = {FEATURES_KEY: tf.sparse_tensor_to_dense(sparse)}
return features, label
def input_fn(partition):
"""Generate an input_fn for the Estimator."""
def _input_fn():
num_workers = len(ctx.cluster_spec['worker'])
data_dir = ctx.absolute_path(args.data_dir)
file_pattern = os.path.join(data_dir, 'part-*')
ds = tf.data.Dataset.list_files(file_pattern)
ds = ds.shard(num_workers, task_index).repeat(args.epochs)
if args.format == 'libsvm':
ds = ds.apply(tf.contrib.data.parallel_interleave(tf.data.TextLineDataset, cycle_length=10))
parse_fn = generator
if partition == "train":
ds = ds.map(parse_fn, num_parallel_calls=5).shuffle(batch_size * 5)
else:
ds = ds.map(parse_fn, num_parallel_calls=5)
ds = ds.apply(tf.contrib.data.batch_and_drop_remainder(batch_size)).prefetch(100)
iterator = ds.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
return _input_fn
if job_name == "ps":
server.join()
elif job_name == "worker":
# Assigns ops to the local worker by default
# 这里的日志都是看不到的
message = ""
with tf.device(tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % task_index,
cluster=cluster)):
print("========= Start Training")
LEARNING_RATE = 0.003
TRAIN_STEPS = 1000
ADANET_ITERATIONS = 2
# 目前来看效果不是很好，还不如线性
logdir = ctx.absolute_path(args.log_dir)
config = tf.estimator.RunConfig(
save_checkpoints_steps=50000,
save_summary_steps=50000,
tf_random_seed=RANDOM_SEED,
model_dir=logdir
)
# BaseLine 线性模型
# estimator = tf.estimator.LinearClassifier(
# feature_columns=feature_columns,
# n_classes=NUM_CLASSES,
# optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
# loss_reduction=loss_reduction,
# config=config
# )
# DNN 测试 - ADANET
estimator = adanet.Estimator(
head=head,
subnetwork_generator=simple_dnn.Generator(
layer_size=128,
initial_num_layers=3,
dropout=0.2,
feature_columns=feature_columns,
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(
input_fn=input_fn("train"),
steps=None
),
config=config
)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(
input_fn=input_fn("train"),
max_steps=TRAIN_STEPS),
eval_spec=tf.estimator.EvalSpec(
input_fn=input_fn("test"),
steps=None)
)
print("Accuracy:", results["accuracy"])
print("Loss:", results["average_loss"])
message = "Accuracy: {}; Loss: {}".format(results["accuracy"], results["average_loss"])
arch = results["architecture/adanet/ensembles"]
summary_proto = tf.summary.Summary.FromString(arch)
arch_result = summary_proto.value[0].tensor.string_val[0]
print("==============================================")
print("{} stopping MonitoredTrainingSession".format(datetime.now().isoformat()))
# WORKAROUND for https://github.com/tensorflow/tensorflow/issues/21745
# wait for all other nodes to complete (via done files)
done_dir = "{}/{}/done".format(ctx.absolute_path(args.log_dir), args.mode)
print("Writing done file to: {}".format(done_dir))
tf.gfile.MakeDirs(done_dir)
with tf.gfile.GFile("{}/{}".format(done_dir, ctx.task_index), 'w') as done_file:
done_file.write(message)
done_file.write(arch_result)
for i in range(30):
if len(tf.gfile.ListDirectory(done_dir)) < len(ctx.cluster_spec['worker']):
print("{} Waiting for other nodes {}".format(datetime.now().isoformat(), i))
time.sleep(1)
else:
print("{} All nodes done".format(datetime.now().isoformat()))
break
def estimator(self,
data_provider,
run_config,
hparams,
train_steps=None,
seed=None):
"""Returns an AdaNet `Estimator` for train and evaluation.
Args:
data_provider: Data `Provider` for dataset to model.
run_config: `RunConfig` object to configure the runtime settings.
hparams: `HParams` instance defining custom hyperparameters.
train_steps: number of train steps.
seed: An integer seed if determinism is required.
Returns:
Returns an `Estimator`.
"""
max_iteration_steps = int(train_steps / hparams.boosting_iterations)
optimizer_fn = optimizer.fn_with_name(
hparams.optimizer,
learning_rate_schedule=hparams.learning_rate_schedule,
cosine_decay_steps=max_iteration_steps)
hparams.add_hparam("total_training_steps", max_iteration_steps)
if hparams.generator == GeneratorType.SIMPLE:
subnetwork_generator = improve_nas.Generator(
feature_columns=data_provider.get_feature_columns(),
optimizer_fn=optimizer_fn,
iteration_steps=max_iteration_steps,
checkpoint_dir=run_config.model_dir,
hparams=hparams,
seed=seed)
elif hparams.generator == GeneratorType.DYNAMIC:
subnetwork_generator = improve_nas.DynamicGenerator(
feature_columns=data_provider.get_feature_columns(),
optimizer_fn=optimizer_fn,
iteration_steps=max_iteration_steps,
checkpoint_dir=run_config.model_dir,
hparams=hparams,
seed=seed)
else:
raise ValueError("Invalid generator: `%s`" % hparams.generator)
evaluator = None
if hparams.use_evaluator:
evaluator = adanet.Evaluator(input_fn=data_provider.get_input_fn(
partition="train",
mode=tf.estimator.ModeKeys.EVAL,
batch_size=hparams.evaluator_batch_size),
steps=hparams.evaluator_steps)
return adanet.Estimator(
head=data_provider.get_head(),
subnetwork_generator=subnetwork_generator,
max_iteration_steps=max_iteration_steps,
adanet_lambda=hparams.adanet_lambda,
adanet_beta=hparams.adanet_beta,
mixture_weight_type=hparams.mixture_weight_type,
force_grow=hparams.force_grow,
evaluator=evaluator,
config=run_config,
model_dir=run_config.model_dir)
def dnn_ada():
print("==============================================")
start = datetime.datetime.now()
print("Start Train Adanet with [DNN Model] on Criteo at %s" % time_str(start))
print("- - - - - - - - - - - - - - - - - - - - - - - -")
# 根据论文参数调整
LEARNING_RATE = LR
model_dir = os.path.join(LOG_DIR, "dnn_%s" % time_str(start))
result_file = os.path.join(RESULT_DIR, "dnn_%s" % time_str(start))
valid_file = os.path.join(RESULT_DIR, "valid_%s" % time_str(start))
test_file = os.path.join(RESULT_DIR, "test_%s" % time_str(start))
tpred_file = os.path.join(RESULT_DIR, "tpred_%s" % time_str(start))
vpred_file = os.path.join(RESULT_DIR, "vpred_%s" % time_str(start))
config = tf.estimator.RunConfig(
save_checkpoints_steps=50000,
save_summary_steps=50000,
tf_random_seed=RANDOM_SEED,
model_dir=model_dir
)
# layer size 125 256 512
estimator = adanet.Estimator(
head=head,
subnetwork_generator=simple_dnn.Generator(
feature_columns=feature_columns,
layer_size=LS,
optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),
seed=RANDOM_SEED),
max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,
evaluator=adanet.Evaluator(
input_fn=input_fn("train"),
steps=None),
config=config
)
results, _ = tf.estimator.train_and_evaluate(
estimator,
train_spec=tf.estimator.TrainSpec(
input_fn=input_fn("train"),
max_steps=TRAIN_STEPS),
eval_spec=tf.estimator.EvalSpec(
input_fn=input_fn("test"),
steps=None)
)
print("Accuracy:", results["accuracy"])
print("AUC", results["auc"])
print("Loss:", results["average_loss"])
# 重新获取评测结果
train_spec = estimator.evaluate(input_fn=input_fn("train"))
test_spec = estimator.evaluate(input_fn=input_fn("test"))
end = datetime.datetime.now()
print("Training end at %s" % time_str(end))
print("Time Spend %s" % str(end - start))
print("==============================================")
with open('{}.txt'.format(result_file), 'w') as f:
f.write('Train Configs:\n')
f.write('[Layer Size] {}\n'.format(LS))
f.write('[Learning Rate] {}\n'.format(LR))
f.write('[BATCH SIZE] {}\n'.format(BATCH_SIZE))
f.write('[Train Step] {}\n'.format(TRAIN_STEPS))
f.write('[Adanet Iteration] {}\n'.format(ADANET_ITERATIONS))
f.write('\nResults:\n')
f.write('[Accurary] {}\n'.format(results["accuracy"]))
f.write('[AUC] {}\n'.format(results["auc"]))
f.write('[Loss] {}\n'.format(results["average_loss"]))
f.write('[Time Spend] {}\n'.format(str(end - start)))
f.write('[Train Spec] {}\n'.format(str(train_spec)))
f.write('[Test Spec] {}\n'.format(str(test_spec)))
# 写入测试集
print("export test data")
test.to_csv('{}.txt'.format(test_file))
print("export train data")
train.to_csv('{}.txt'.format(valid_file))
# 进行预测
predictions = estimator.predict(input_fn=input_fn("test"))
# 写入预测集
with open('{}.txt'.format(tpred_file), 'w') as f:
for pred in predictions:
f.write(str(pred))
f.write('\n')
# 进行预测并写入预测集
predictions = estimator.predict(input_fn=input_fn("valid"))
# 写入预测集
with open('{}.txt'.format(vpred_file), 'w') as f:
for pred in predictions:
f.write(str(pred))
f.write('\n')
