Пример #1
0
def main(unused_argv):
    hparams = udc_hparams.create_hparams()

    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)

    estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
                                           model_dir=MODEL_DIR,
                                           config=tf.contrib.learn.RunConfig())

    input_fn_train = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.TRAIN,
        input_files=[TRAIN_FILE],
        batch_size=hparams.batch_size,
        num_epochs=FLAGS.num_epochs)

    input_fn_eval = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.EVAL,
        input_files=[VALIDATION_FILE],
        batch_size=hparams.eval_batch_size,
        num_epochs=1)

    eval_metrics = udc_metrics.create_evaluation_metrics()

    eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
        input_fn=input_fn_eval,
        every_n_steps=FLAGS.eval_every,
        metrics=eval_metrics,
        early_stopping_metric="recall_at_1",
        early_stopping_metric_minimize=False,
        early_stopping_rounds=4000)

    estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
Пример #2
0
def main(unused_argv):
    hparams = udc_hparams.create_hparams()

    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)

    estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
                                           model_dir=MODEL_DIR)

    input_fn_train = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.TRAIN,
        input_files=[TRAIN_FILE],
        batch_size=hparams.batch_size,
        num_epochs=FLAGS.num_epochs)

    input_fn_eval = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.EVAL,
        input_files=[VALIDATION_FILE],
        batch_size=hparams.eval_batch_size,
        num_epochs=1)

    eval_metrics = udc_metrics.create_evaluation_metrics()

    eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
        input_fn=input_fn_eval,
        every_n_steps=FLAGS.eval_every,
        metrics=eval_metrics)

    dbg_hook = tfdbg.LocalCLIDebugHook()
    estimator.fit(input_fn=input_fn_train,
                  steps=FLAGS.num_steps,
                  monitors=[eval_monitor])
Пример #3
0
def main(unused_argv):
    hparams = udc_hparams.create_hparams()
    writer = tf.summary.FileWriter(FLAGS.logdir)

    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)

    estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
                                           model_dir=MODEL_DIR,
                                           config=tf.contrib.learn.RunConfig())

    input_fn_train = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.TRAIN,
        input_files=[TRAIN_FILE],
        batch_size=hparams.batch_size,
        num_epochs=FLAGS.num_epochs)

    input_fn_eval = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.EVAL,
        input_files=[VALIDATION_FILE],
        batch_size=hparams.eval_batch_size,
        num_epochs=1)

    eval_metrics = udc_metrics.create_evaluation_metrics()

    eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
        input_fn=input_fn_eval,
        every_n_steps=FLAGS.eval_every,
        metrics=eval_metrics)

    estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
Пример #4
0
def main(unused_argv):
    hparams = udc_hparams.create_hparams()

    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)

    estimator = tf.estimator.Estimator(model_fn=model_fn,
                                       model_dir=MODEL_DIR,
                                       config=tf.estimator.RunConfig())

    input_fn_train = udc_inputs.create_input_fn(
        mode=tf.estimator.ModeKeys.TRAIN,
        input_files=[TRAIN_FILE],
        batch_size=hparams.batch_size,
        num_epochs=FLAGS.num_epochs)

    input_fn_eval = udc_inputs.create_input_fn(
        mode=tf.estimator.ModeKeys.EVAL,
        input_files=[VALIDATION_FILE],
        batch_size=hparams.eval_batch_size,
        num_epochs=1)

    eval_metrics = udc_metrics.create_evaluation_metrics()

    estimator.train(input_fn=input_fn_train, steps=FLAGS.num_epochs)
Пример #5
0
def main(unused_argv):
  hparams = udc_hparams.create_hparams()

  model_fn = udc_model.create_model_fn(
    hparams,
    model_impl=dual_encoder_model)

  estimator = tf.contrib.learn.Estimator(
    model_fn=model_fn,
    model_dir=MODEL_DIR,
    config=tf.contrib.learn.RunConfig())

  input_fn_train = udc_inputs.create_input_fn(
    mode=tf.contrib.learn.ModeKeys.TRAIN,
    input_files=[TRAIN_FILE],
    batch_size=hparams.batch_size,
    num_epochs=FLAGS.num_epochs)

  input_fn_eval = udc_inputs.create_input_fn(
    mode=tf.contrib.learn.ModeKeys.EVAL,
    input_files=[VALIDATION_FILE],
    batch_size=hparams.eval_batch_size,
    num_epochs=1)

  eval_metrics = udc_metrics.create_evaluation_metrics()
  
  eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
        input_fn=input_fn_eval,
        every_n_steps=FLAGS.eval_every,
        metrics=eval_metrics)

  estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
Пример #6
0
def initialize():
    hparams = udc_hparams.create_hparams()
    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)
    estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
                                           model_dir=FLAGS.model_dir)
    estimator._targets_info = tf.contrib.learn.estimators.tensor_signature.TensorSignature(
        tf.constant(0, shape=[1, 1]))
    return estimator
Пример #7
0
def main(unused_argv):
    # 获取udc_hparams文件设置参数
    hparams = udc_hparams.create_hparams()

    # 设置模型方法,传递参数和dual_encoder_model
    model_fn = udc_model.create_model_fn(
        hparams,
        model_impl=dual_encoder_model)

    estimator = tf.contrib.learn.Estimator(
        model_fn=model_fn,
        model_dir=MODEL_DIR,
        config=tf.contrib.learn.RunConfig())

    '''
    1. Create a feature definition that describes the fields in our Example file
    2. Read records from the input_files with tf.TFRecordReader
    3. Parse the records according to the feature definition
    4. Extract the training labels
    5. Batch multiple examples and training labels
    6. Return the batched examples and training labels
    '''

    input_fn_train = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.TRAIN,
        input_files=[TRAIN_FILE],
        batch_size=hparams.batch_size,
        num_epochs=FLAGS.num_epochs)

    input_fn_eval = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.EVAL,
        input_files=[VALIDATION_FILE],
        batch_size=hparams.eval_batch_size,
        num_epochs=1)

    # eval_metrics = udc_metrics.create_evaluation_metrics()
    eval_metrics = None

    """Runs evaluation of a given estimator, at most every N steps.

      Note that the evaluation is done based on the saved checkpoint, which will
      usually be older than the current step.

      Can do early stopping on validation metrics if `early_stopping_rounds` is
      provided.
      """
    eval_monitor = tf.contrib.learn.monitors.ValidationMonitor(
        input_fn=input_fn_eval,
        every_n_steps=FLAGS.eval_every,
        metrics=eval_metrics)

    estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
Пример #8
0
def main(unused_argv):
  hparams = udc_hparams.create_hparams()

  model_fn = udc_model.create_model_fn(
    hparams,
    model_impl=dual_encoder_model)

  estimator = tf.contrib.learn.Estimator(
    model_fn=model_fn,
    model_dir=MODEL_DIR,
    config=tf.contrib.learn.RunConfig())

  input_fn_train = udc_inputs.create_input_fn(
    mode=tf.contrib.learn.ModeKeys.TRAIN,
    input_files=[TRAIN_FILE],
    batch_size=hparams.batch_size,
    num_epochs=FLAGS.num_epochs)

  input_fn_eval = udc_inputs.create_input_fn(
    mode=tf.contrib.learn.ModeKeys.EVAL,
    input_files=[VALIDATION_FILE],
    batch_size=hparams.eval_batch_size,
    num_epochs=1)

  eval_metrics = udc_metrics.create_evaluation_metrics()

  # We need to subclass theis manually for now. The next TF version will
  # have support ValidationMonitors with metrics built-in.
  # It's already on the master branch.
  class EvaluationMonitor(tf.contrib.learn.monitors.EveryN):
    def every_n_step_end(self, step, outputs):
      self._estimator.evaluate(
        input_fn=input_fn_eval,
        metrics=eval_metrics,
        steps=None)

  eval_monitor = EvaluationMonitor(every_n_steps=FLAGS.eval_every)
  estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
Пример #9
0
def retrieve_response(input_c):
    #input_c = "hello"
    potential_responses = get_responses(10, input_c)
    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)
    estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
                                           model_dir=FLAGS.trained_model_dir)

    # Ugly hack, seems to be a bug in Tensorflow
    # estimator.predict doesn't work without this line
    estimator._targets_info = tf.contrib.learn.estimators.tensor_signature.TensorSignature(
        tf.constant(0, shape=[1, 1]))
    response = "..."
    max = 0
    for r in potential_responses:
        prob = estimator.predict(input_fn=lambda: get_features(input_c, r),
                                 as_iterable=True)
        val = next(prob)[0]
        if val > max:
            response = r
            max = val
        #print("{}: {:g}: {}".format(r, prob[0,0], prob))
    return response
Пример #10
0
def main(unused_argv):
    hparams = udc_hparams.create_hparams()

    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)

    estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
                                           model_dir=MODEL_DIR,
                                           config=tf.contrib.learn.RunConfig())

    input_fn_train = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.TRAIN,
        input_files=[TRAIN_FILE],
        batch_size=hparams.batch_size,
        num_epochs=FLAGS.num_epochs)

    input_fn_eval = udc_inputs.create_input_fn(
        mode=tf.contrib.learn.ModeKeys.EVAL,
        input_files=[VALIDATION_FILE],
        batch_size=hparams.eval_batch_size,
        num_epochs=1)

    eval_metrics = udc_metrics.create_evaluation_metrics()

    # We need to subclass theis manually for now. The next TF version will
    # have support ValidationMonitors with metrics built-in.
    # It's already on the master branch.
    class EvaluationMonitor(tf.contrib.learn.monitors.EveryN):
        def every_n_step_end(self, step, outputs):
            self._estimator.evaluate(input_fn=input_fn_eval,
                                     metrics=eval_metrics,
                                     steps=None)

    eval_monitor = EvaluationMonitor(every_n_steps=FLAGS.eval_every,
                                     first_n_steps=-1)
    estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])

'''
  my test try
'''
if __name__ == "__main__":

    import udc_hparams
    import udc_model

    hparams = udc_hparams.create_hparams()

    embeddings_W = get_embeddings(hparams)

    # <tf.Variable 'word_embeddings:0' shape=(91620, 100) dtype=float32_ref>
    print(embeddings_W)

    import os
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        result2 = sess.run(embeddings_W)
        print(result2)
    hparams = udc_hparams.create_hparams()

    # 模型函数
    model_fn = udc_model.create_model_fn(hparams,
                                         model_impl=dual_encoder_model)

    print(model_fn)
Пример #12
0
from models.dual_encoder import dual_encoder_model

tf.flags.DEFINE_string("test_file", "./data/test.tfrecords", "Path of test data in TFRecords format")
tf.flags.DEFINE_string("model_dir", None, "Directory to load model checkpoints from")
tf.flags.DEFINE_integer("loglevel", 20, "Tensorflow log level")
tf.flags.DEFINE_integer("test_batch_size", 16, "Batch size for testing")
FLAGS = tf.flags.FLAGS

if not FLAGS.model_dir:
  print("You must specify a model directory")
  sys.exit(1)

tf.logging.set_verbosity(FLAGS.loglevel)

if __name__ == "__main__":
  hparams = udc_hparams.create_hparams()
  model_fn = udc_model.create_model_fn(hparams, model_impl=dual_encoder_model)
  estimator = tf.contrib.learn.Estimator(
    model_fn=model_fn,
    model_dir=FLAGS.model_dir,
    config=tf.contrib.learn.RunConfig())

  input_fn_test = udc_inputs.create_input_fn(
    mode=tf.contrib.learn.ModeKeys.EVAL,
    input_files=[FLAGS.test_file],
    batch_size=FLAGS.test_batch_size,
    num_epochs=1)

  eval_metrics = udc_metrics.create_evaluation_metrics()
  estimator.evaluate(input_fn=input_fn_test, steps=None, metrics=eval_metrics)