def main(unused_argv):
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig())
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
input_fn=input_fn_eval,
every_n_steps=FLAGS.eval_every,
metrics=eval_metrics,
early_stopping_metric="recall_at_1",
early_stopping_metric_minimize=False,
early_stopping_rounds=4000)
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
def main(unused_argv):
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=MODEL_DIR)
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
input_fn=input_fn_eval,
every_n_steps=FLAGS.eval_every,
metrics=eval_metrics)
dbg_hook = tfdbg.LocalCLIDebugHook()
estimator.fit(input_fn=input_fn_train,
steps=FLAGS.num_steps,
monitors=[eval_monitor])
def main(unused_argv):
hparams = udc_hparams.create_hparams()
writer = tf.summary.FileWriter(FLAGS.logdir)
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig())
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
input_fn=input_fn_eval,
every_n_steps=FLAGS.eval_every,
metrics=eval_metrics)
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
def main(unused_argv):
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.estimator.Estimator(model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.estimator.RunConfig())
input_fn_train = udc_inputs.create_input_fn(
mode=tf.estimator.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.estimator.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
estimator.train(input_fn=input_fn_train, steps=FLAGS.num_epochs)
def main(unused_argv):
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(
hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(
model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig())
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
input_fn=input_fn_eval,
every_n_steps=FLAGS.eval_every,
metrics=eval_metrics)
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
def initialize():
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=FLAGS.model_dir)
estimator._targets_info = tf.contrib.learn.estimators.tensor_signature.TensorSignature(
tf.constant(0, shape=[1, 1]))
return estimator
def main(unused_argv):
# 获取udc_hparams文件设置参数
hparams = udc_hparams.create_hparams()
# 设置模型方法，传递参数和dual_encoder_model
model_fn = udc_model.create_model_fn(
hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(
model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig())
'''
1. Create a feature definition that describes the fields in our Example file
2. Read records from the input_files with tf.TFRecordReader
3. Parse the records according to the feature definition
4. Extract the training labels
5. Batch multiple examples and training labels
6. Return the batched examples and training labels
'''
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
# eval_metrics = udc_metrics.create_evaluation_metrics()
eval_metrics = None
"""Runs evaluation of a given estimator, at most every N steps.
Note that the evaluation is done based on the saved checkpoint, which will
usually be older than the current step.
Can do early stopping on validation metrics if `early_stopping_rounds` is
provided.
"""
eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
input_fn=input_fn_eval,
every_n_steps=FLAGS.eval_every,
metrics=eval_metrics)
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
def main(unused_argv):
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(
hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(
model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig())
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
# We need to subclass theis manually for now. The next TF version will
# have support ValidationMonitors with metrics built-in.
# It's already on the master branch.
class EvaluationMonitor(tf.contrib.learn.monitors.EveryN):
def every_n_step_end(self, step, outputs):
self._estimator.evaluate(
input_fn=input_fn_eval,
metrics=eval_metrics,
steps=None)
eval_monitor = EvaluationMonitor(every_n_steps=FLAGS.eval_every)
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
def retrieve_response(input_c):
#input_c = "hello"
potential_responses = get_responses(10, input_c)
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=FLAGS.trained_model_dir)
# Ugly hack, seems to be a bug in Tensorflow
# estimator.predict doesn't work without this line
estimator._targets_info = tf.contrib.learn.estimators.tensor_signature.TensorSignature(
tf.constant(0, shape=[1, 1]))
response = "..."
max = 0
for r in potential_responses:
prob = estimator.predict(input_fn=lambda: get_features(input_c, r),
as_iterable=True)
val = next(prob)[0]
if val > max:
response = r
max = val
#print("{}: {:g}: {}".format(r, prob[0,0], prob))
return response
def main(unused_argv):
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=MODEL_DIR,
config=tf.contrib.learn.RunConfig())
input_fn_train = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.TRAIN,
input_files=[TRAIN_FILE],
batch_size=hparams.batch_size,
num_epochs=FLAGS.num_epochs)
input_fn_eval = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[VALIDATION_FILE],
batch_size=hparams.eval_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
# We need to subclass theis manually for now. The next TF version will
# have support ValidationMonitors with metrics built-in.
# It's already on the master branch.
class EvaluationMonitor(tf.contrib.learn.monitors.EveryN):
def every_n_step_end(self, step, outputs):
self._estimator.evaluate(input_fn=input_fn_eval,
metrics=eval_metrics,
steps=None)
eval_monitor = EvaluationMonitor(every_n_steps=FLAGS.eval_every,
first_n_steps=-1)
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
'''
my test try
'''
if __name__ == "__main__":
import udc_hparams
import udc_model
hparams = udc_hparams.create_hparams()
embeddings_W = get_embeddings(hparams)
# <tf.Variable 'word_embeddings:0' shape=(91620, 100) dtype=float32_ref>
print(embeddings_W)
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
result2 = sess.run(embeddings_W)
print(result2)
hparams = udc_hparams.create_hparams()
# 模型函数
model_fn = udc_model.create_model_fn(hparams,
model_impl=dual_encoder_model)
print(model_fn)
from models.dual_encoder import dual_encoder_model
tf.flags.DEFINE_string("test_file", "./data/test.tfrecords", "Path of test data in TFRecords format")
tf.flags.DEFINE_string("model_dir", None, "Directory to load model checkpoints from")
tf.flags.DEFINE_integer("loglevel", 20, "Tensorflow log level")
tf.flags.DEFINE_integer("test_batch_size", 16, "Batch size for testing")
FLAGS = tf.flags.FLAGS
if not FLAGS.model_dir:
print("You must specify a model directory")
sys.exit(1)
tf.logging.set_verbosity(FLAGS.loglevel)
if __name__ == "__main__":
hparams = udc_hparams.create_hparams()
model_fn = udc_model.create_model_fn(hparams, model_impl=dual_encoder_model)
estimator = tf.contrib.learn.Estimator(
model_fn=model_fn,
model_dir=FLAGS.model_dir,
config=tf.contrib.learn.RunConfig())
input_fn_test = udc_inputs.create_input_fn(
mode=tf.contrib.learn.ModeKeys.EVAL,
input_files=[FLAGS.test_file],
batch_size=FLAGS.test_batch_size,
num_epochs=1)
eval_metrics = udc_metrics.create_evaluation_metrics()
estimator.evaluate(input_fn=input_fn_test, steps=None, metrics=eval_metrics)