def test(sess):
    tf.set_random_seed(seed)
    ### Prepare data for training
    print("Prepare vocab dict and read pretrained word embeddings ...")
    vocab_dict, word_embedding_array = DataProcessor(
    ).prepare_vocab_embeddingdict()
    # vocab_dict contains _PAD and _UNK but not word_embedding_array

    print("Prepare test data ...")
    test_batch = batch_load_data(DataProcessor().prepare_news_data(
        vocab_dict, data_type="test"))

    fil_lens_to_test = FLAGS.max_filter_length - FLAGS.min_filter_length + 1
    if FLAGS.handle_filter_output == "concat" and FLAGS.sentembed_size % fil_lens_to_test != 0:
        q = int(FLAGS.sentembed_size // fil_lens_to_test)
        FLAGS.sentembed_size = q * fil_lens_to_test
        print("corrected embedding size: %d" % FLAGS.sentembed_size)

    # Create Model with various operations
    model = MY_Model(sess, len(vocab_dict) - 2)

    # Select the model

    selected_modelpath = FLAGS.train_dir + "/step-a.model.ckpt.epoch-" + str(
        FLAGS.model_to_load)

    # Reload saved model and test
    #print("Reading model parameters from %s" % selected_modelpath)
    model.saver.restore(sess, selected_modelpath)
    #print("Model loaded.")

    # Initialize word embedding before training
    sess.run(model.vocab_embed_variable.assign(word_embedding_array))

    # Test Accuracy and Prediction
    #print("Performance on the test data:")
    FLAGS.authorise_gold_label = False
    FLAGS.use_dropout = False
    test_batch = batch_predict_with_a_model(test_batch,
                                            "test",
                                            model,
                                            session=sess)
    probs = sess.run(model.predictions,
                     feed_dict={model.logits_placeholder: test_batch.logits})

    acc = accuracy_qas_top(probs, test_batch.labels, test_batch.weights,
                           test_batch.isf_score_ids)
    mrr = mrr_metric(probs, test_batch.labels, test_batch.weights,
                     test_batch.isf_score_ids, "test")
    _map = map_score(probs, test_batch.labels, test_batch.weights,
                     test_batch.isf_score_ids, "test")
    print("Metrics: acc: %.4f | mrr: %.4f | map: %.4f" % (acc, mrr, _map))

    # Writing test predictions and final summaries
    if FLAGS.save_preds:
        #print("Writing predictions...")
        modelname = "step-a.model.ckpt.epoch-" + str(FLAGS.model_to_load)
        write_prediction_summaries(test_batch, probs, modelname, "test")
        write_cos_sim(test_batch.cos_sim, modelname, "test")
def test(sess):
  ### Prepare data for training
  print("Prepare vocab dict and read pretrained word embeddings ...")
  vocab_dict, word_embedding_array = DataProcessor().prepare_vocab_embeddingdict()
  # vocab_dict contains _PAD and _UNK but not word_embedding_array

  print("Prepare test data ...")
  train_data = DataProcessor().prepare_news_data(vocab_dict, data_type="training")
  test_batch = batch_load_data(DataProcessor().prepare_news_data(vocab_dict,
                                                                data_type="test",
                                                                normalizer=train_data.normalizer,
                                                                pca_model=train_data.pca_model))
  del train_data

  # Create Model with various operations
  model = MY_Model(sess, len(vocab_dict)-2)

  # # Initialize word embedding before training
  #print("Initialize word embedding vocabulary with pretrained embeddings ...")
  #sess.run(model.vocab_embed_variable.assign(word_embedding_array))

  # Select the model
  if (os.path.isfile(FLAGS.train_dir+"/step-a.model.ckpt.epoch-"+str(FLAGS.model_to_load))):
    selected_modelpath = FLAGS.train_dir+"/step-a.model.ckpt.epoch-"+str(FLAGS.model_to_load)
  else:
    print("Model not found in checkpoint folder.")
    exit(0)

  # Reload saved model and test
  print("Reading model parameters from %s" % selected_modelpath)
  model.saver.restore(sess, selected_modelpath)
  print("Model loaded.")

  # Initialize word embedding before training
  print("Initialize word embedding vocabulary with pretrained embeddings ...")
  sess.run(model.vocab_embed_variable.assign(word_embedding_array))

  # Test Accuracy and Prediction
  print("Performance on the test data:")
  FLAGS.authorise_gold_label = False
  test_batch = batch_predict_with_a_model(test_batch,"test",model, session=sess)
  probs = sess.run(model.probs,feed_dict={model.logits_placeholder: test_batch.logits})

  test_acc = accuracy_qas_top(probs, test_batch.labels, test_batch.weights, test_batch.isf_score_ids)

  # Print Test Summary
  print("Test ("+str(test_batch.docs.shape[0])+") Accuracy = {:.6f}".format(test_acc))
  # Estimate MRR on validation set
  mrr_score = mrr_metric(probs, test_batch.labels, test_batch.weights, test_batch.isf_score_ids)
  print("Test ("+str(test_batch.docs.shape[0])+") MRR = {:.6f}".format(mrr_score))
  # Estimate MAP score on validation set
  mapsc = map_score(probs, test_batch.labels, test_batch.weights, test_batch.isf_score_ids)
  print("Test ("+str(test_batch.docs.shape[0])+") MAP = {:.6f}".format(mapsc))

  # Writing test predictions and final summaries
  modelname = "step-a.model.ckpt.epoch-" + str(FLAGS.model_to_load)
  write_prediction_summaries(test_batch, probs, modelname, "test")
def test_val(sess):
    """
  Test on validation Mode: Loads an existing model and test it on the validation set
  """
    tf.set_random_seed(seed)
    if FLAGS.load_prediction != -1:
        print(
            "====================================== [%d] ======================================"
            % (FLAGS.load_prediction))

    ### Prepare data for training
    #print("Prepare vocab dict and read pretrained word embeddings ...")
    vocab_dict, word_embedding_array = DataProcessor(
    ).prepare_vocab_embeddingdict()
    # vocab_dict contains _PAD and _UNK but not word_embedding_array

    val_batch = batch_load_data(DataProcessor().prepare_news_data(
        vocab_dict, data_type="validation"))

    fil_lens_to_test = FLAGS.max_filter_length - FLAGS.min_filter_length + 1
    if FLAGS.handle_filter_output == "concat" and FLAGS.sentembed_size % fil_lens_to_test != 0:
        q = int(FLAGS.sentembed_size // fil_lens_to_test)
        FLAGS.sentembed_size = q * fil_lens_to_test
        print("corrected embedding size: %d" % FLAGS.sentembed_size)

    # Create Model with various operations
    model = MY_Model(sess, len(vocab_dict) - 2)

    # Select the model
    selected_modelpath = FLAGS.train_dir + "/step-a.model.ckpt.epoch-" + str(
        FLAGS.model_to_load)

    # Reload saved model and test
    model.saver.restore(sess, selected_modelpath)

    # Initialize word embedding before training
    sess.run(model.vocab_embed_variable.assign(word_embedding_array))

    # Test Accuracy and Prediction
    FLAGS.authorise_gold_label = False
    FLAGS.use_dropout = False
    val_batch = batch_predict_with_a_model(val_batch,
                                           "validation",
                                           model,
                                           session=sess)
    FLAGS.authorise_gold_label = True
    probs = sess.run(model.predictions,
                     feed_dict={model.logits_placeholder: val_batch.logits})

    acc = accuracy_qas_top(probs, val_batch.labels, val_batch.weights,
                           val_batch.isf_score_ids)
    mrr = mrr_metric(probs, val_batch.labels, val_batch.weights,
                     val_batch.isf_score_ids, "validation")
    _map = map_score(probs, val_batch.labels, val_batch.weights,
                     val_batch.isf_score_ids, "validation")
    print("Metrics: acc: %.4f | mrr: %.4f | map: %.4f" % (acc, mrr, _map))

    if FLAGS.load_prediction != -1:
        fn = ''
        if FLAGS.filtered_setting:
            fn = "%s/step-a.model.ckpt.%s-top%d-isf-metrics" % (
                FLAGS.train_dir, "validation", FLAGS.topK)
        else:
            fn = "%s/step-a.model.ckpt.%s-metrics" % (FLAGS.train_dir,
                                                      "validation")
        save_metrics(fn, FLAGS.load_prediction, validation_acc, mrr_score,
                     mapsc)

    if FLAGS.save_preds:
        modelname = "step-a.model.ckpt.epoch-" + str(FLAGS.model_to_load)
        write_prediction_summaries(val_batch, probs, modelname, "validation")
        write_cos_sim(val_batch.cos_sim, modelname, "validation")
def train_simple(sess):
    """
  Training Mode: Create a new model and train the network
  """
    tf.set_random_seed(seed)
    ### Prepare data for training
    vocab_dict, word_embedding_array = DataProcessor(
    ).prepare_vocab_embeddingdict()
    # vocab_dict contains _PAD and _UNK but not word_embedding_array

    train_data = DataProcessor().prepare_news_data(vocab_dict,
                                                   data_type="training")

    # data in whole batch with padded matrixes
    val_batch = batch_load_data(DataProcessor().prepare_news_data(
        vocab_dict, data_type="validation"))

    fil_lens_to_test = FLAGS.max_filter_length - FLAGS.min_filter_length + 1
    if FLAGS.handle_filter_output == "concat" and FLAGS.sentembed_size % fil_lens_to_test != 0:
        q = int(FLAGS.sentembed_size // fil_lens_to_test)
        FLAGS.sentembed_size = q * fil_lens_to_test
        print("corrected embedding size: %d" % FLAGS.sentembed_size)

    # Create Model with various operations
    model = MY_Model(sess, len(vocab_dict) - 2)
    init_epoch = 1
    # Resume training if indicated Select the model
    if FLAGS.model_to_load != -1:
        selected_modelpath = FLAGS.train_dir + "/step-a.model.ckpt.epoch-" + str(
            FLAGS.model_to_load)
        init_epoch = FLAGS.model_to_load + 1
        print("Reading model parameters from %s" % selected_modelpath)
        model.saver.restore(sess, selected_modelpath)
        print("Model loaded.")

    # Initialize word embedding before training
    sess.run(model.vocab_embed_variable.assign(word_embedding_array))

    ### STEP A : Start Pretraining the policy with Supervised Labels: Simple Cross Entropy Training
    counter = 0
    max_val_acc = -1
    for epoch in range(init_epoch, FLAGS.train_epoch_crossentropy + 1):
        ep_time = time.time()  # to check duration

        train_data.shuffle_fileindices()
        # Start Batch Training
        step = 1
        total_loss = 0
        while (step * FLAGS.batch_size) <= len(train_data.fileindices):
            # Get batch data as Numpy Arrays
            batch = train_data.get_batch(((step - 1) * FLAGS.batch_size),
                                         (step * FLAGS.batch_size))

            # Run optimizer: optimize policy and reward estimator
            _, ce_loss = sess.run(
                [model.train_op_policynet_withgold, model.cross_entropy_loss],
                feed_dict={
                    model.document_placeholder: batch.docs,
                    model.label_placeholder: batch.labels,
                    model.weight_placeholder: batch.weights
                })
            total_loss += ce_loss
            step += 1
        #END-WHILE-TRAINING  ... but wait there is more
        ## eval metrics
        FLAGS.authorise_gold_label = False
        prev_use_dpt = FLAGS.use_dropout
        total_loss /= step
        FLAGS.use_dropout = False
        # retrieve batch with updated logits in it
        val_batch = batch_predict_with_a_model(val_batch,
                                               "validation",
                                               model,
                                               session=sess)
        FLAGS.authorise_gold_label = True
        FLAGS.use_dropout = prev_use_dpt
        probs = sess.run(
            model.predictions,
            feed_dict={model.logits_placeholder: val_batch.logits})
        validation_acc = accuracy_qas_top(probs, val_batch.labels,
                                          val_batch.weights,
                                          val_batch.isf_score_ids)
        val_mrr = mrr_metric(probs, val_batch.labels, val_batch.weights,
                             val_batch.isf_score_ids, "validation")
        val_map = map_score(probs, val_batch.labels, val_batch.weights,
                            val_batch.isf_score_ids, "validation")

        print(
            "\tEpoch %2d || Train ce_loss: %4.3f || Val acc: %.4f || Val mrr: %.4f || Val mac: %.4f || duration: %3.2f"
            % (epoch, total_loss, validation_acc, val_mrr, val_map,
               time.time() - ep_time))

        if FLAGS.save_models:
            print("Saving model after epoch completion")
            checkpoint_path = os.path.join(
                FLAGS.train_dir, "step-a.model.ckpt.epoch-" + str(epoch))
            model.saver.save(sess, checkpoint_path)
        print(
            "------------------------------------------------------------------------------------------"
        )
    #END-FOR-EPOCH

    print("Optimization Finished!")
def train_debug(sess):
    """
  Training Mode: Create a new model and train the network
  """
    tf.set_random_seed(seed)
    ### Prepare data for training
    print("Prepare vocab dict and read pretrained word embeddings ...")
    vocab_dict, word_embedding_array = DataProcessor(
    ).prepare_vocab_embeddingdict()
    # vocab_dict contains _PAD and _UNK but not word_embedding_array

    print("Prepare training data ...")
    train_data = DataProcessor().prepare_news_data(vocab_dict,
                                                   data_type="training")
    print("Training size: ", len(train_data.fileindices))

    print("Prepare validation data ...")
    # data in whole batch with padded matrixes
    val_batch = batch_load_data(DataProcessor().prepare_news_data(
        vocab_dict, data_type="validation"))
    print("Validation size: ", val_batch.docs.shape[0])

    fil_lens_to_test = FLAGS.max_filter_length - FLAGS.min_filter_length + 1
    if FLAGS.handle_filter_output == "concat" and FLAGS.sentembed_size % fil_lens_to_test != 0:
        q = int(FLAGS.sentembed_size // fil_lens_to_test)
        FLAGS.sentembed_size = q * fil_lens_to_test
        print("corrected embedding size: %d" % FLAGS.sentembed_size)

    # Create Model with various operations
    model = MY_Model(sess, len(vocab_dict) - 2)

    init_epoch = 1
    # Resume training if indicated Select the model
    if FLAGS.model_to_load != -1:
        selected_modelpath = FLAGS.train_dir + "/step-a.model.ckpt.epoch-" + str(
            FLAGS.model_to_load)
        init_epoch = FLAGS.model_to_load + 1
        print("Reading model parameters from %s" % selected_modelpath)
        model.saver.restore(sess, selected_modelpath)
        print("Model loaded.")

    # Initialize word embedding before training
    print(
        "Initialize word embedding vocabulary with pretrained embeddings ...")
    sess.run(model.vocab_embed_variable.assign(word_embedding_array))

    ### STEP A : Start Pretraining the policy with Supervised Labels: Simple Cross Entropy Training
    counter = 0
    max_val_acc = -1
    for epoch in range(init_epoch, FLAGS.train_epoch_crossentropy + 1):
        ep_time = time.time()  # to check duration

        train_data.shuffle_fileindices()

        # Start Batch Training
        step = 1
        total_ce_loss = 0
        total_train_acc = 0
        while (step * FLAGS.batch_size) <= len(train_data.fileindices):
            # Get batch data as Numpy Arrays
            batch = train_data.get_batch(((step - 1) * FLAGS.batch_size),
                                         (step * FLAGS.batch_size))

            # Run optimizer: optimize policy and reward estimator
            sess.run(
                [model.train_op_policynet_withgold],
                feed_dict={
                    model.document_placeholder: batch.docs,
                    model.label_placeholder: batch.labels,
                    model.weight_placeholder: batch.weights
                })

            prev_use_dpt = FLAGS.use_dropout
            FLAGS.use_dropout = False
            batch_logits, ce_loss, merged_summ = sess.run(
                [model.logits, model.cross_entropy_loss, model.merged],
                feed_dict={
                    model.document_placeholder: batch.docs,
                    model.label_placeholder: batch.labels,
                    model.weight_placeholder: batch.weights
                })
            total_ce_loss += ce_loss
            probs = sess.run(
                model.predictions,
                feed_dict={model.logits_placeholder: batch_logits})
            training_acc = accuracy_qas_top(probs, batch.labels, batch.weights,
                                            batch.isf_score_ids)
            FLAGS.use_dropout = prev_use_dpt
            total_train_acc += training_acc
            # Print the progress
            if (step % FLAGS.training_checkpoint) == 0:
                total_train_acc /= FLAGS.training_checkpoint
                acc_sum = sess.run(
                    model.tstepa_accuracy_summary,
                    feed_dict={model.train_acc_placeholder: total_train_acc})

                total_ce_loss /= FLAGS.training_checkpoint
                # Print Summary to Tensor Board
                model.summary_writer.add_summary(merged_summ, counter)
                model.summary_writer.add_summary(acc_sum, counter)

                # Performance on the validation set
                # Get Predictions: Prohibit the use of gold labels
                FLAGS.authorise_gold_label = False
                prev_use_dpt = FLAGS.use_dropout
                FLAGS.use_dropout = False
                val_batch = batch_predict_with_a_model(val_batch,
                                                       "validation",
                                                       model,
                                                       session=sess)
                FLAGS.use_dropout = prev_use_dpt
                FLAGS.authorise_gold_label = True

                # Validation Accuracy and Prediction
                probs = sess.run(
                    model.predictions,
                    feed_dict={model.logits_placeholder: val_batch.logits})
                validation_acc = accuracy_qas_top(probs, val_batch.labels,
                                                  val_batch.weights,
                                                  val_batch.isf_score_ids)

                ce_loss_val, ce_loss_sum, acc_sum = sess.run(
                    [
                        model.cross_entropy_loss_val,
                        model.ce_loss_summary_val,
                        model.vstepa_accuracy_summary
                    ],
                    feed_dict={
                        model.logits_placeholder: val_batch.logits,
                        model.label_placeholder: val_batch.labels,
                        model.weight_placeholder: val_batch.weights,
                        model.val_acc_placeholder: validation_acc
                    })

                # Print Validation Summary
                model.summary_writer.add_summary(acc_sum, counter)
                model.summary_writer.add_summary(ce_loss_sum, counter)
                print(
                    "Epoch %2d, step: %2d(%2d) || CE loss || Train : %4.3f , Val : %4.3f ||| ACC || Train : %.3f , Val : %.3f"
                    % (epoch, step, counter, total_ce_loss, ce_loss_val,
                       training_acc, validation_acc))
                total_ce_loss = 0
                total_train_acc = 0

            if (step % 5) == 0:  # to have comparable tensorboard plots
                counter += 1
            # Increase step
            step += 1
        #END-WHILE-TRAINING  ... but wait there is more
        ## eval metrics
        FLAGS.authorise_gold_label = False
        prev_use_dpt = FLAGS.use_dropout
        FLAGS.use_dropout = False
        val_batch = batch_predict_with_a_model(val_batch,
                                               "validation",
                                               model,
                                               session=sess)
        FLAGS.use_dropout = prev_use_dpt
        FLAGS.authorise_gold_label = True
        # Validation metrics
        probs = sess.run(
            model.predictions,
            feed_dict={model.logits_placeholder: val_batch.logits})
        acc = accuracy_qas_top(probs, val_batch.labels, val_batch.weights,
                               val_batch.isf_score_ids)
        mrr = mrr_metric(probs, val_batch.labels, val_batch.weights,
                         val_batch.isf_score_ids, "validation")
        _map = map_score(probs, val_batch.labels, val_batch.weights,
                         val_batch.isf_score_ids, "validation")
        print("Metrics: acc: %.4f | mrr: %.4f | map: %.4f" % (acc, mrr, _map))

        print("Epoch %2d : Duration: %.4f" % (epoch, time.time() - ep_time))
        if FLAGS.save_models:
            print("Saving model after epoch completion")
            checkpoint_path = os.path.join(
                FLAGS.train_dir, "step-a.model.ckpt.epoch-" + str(epoch))
            model.saver.save(sess, checkpoint_path)
        print(
            "------------------------------------------------------------------------------------------"
        )
    #END-FOR-EPOCH

    print("Optimization Finished!")
def train(sess):
    """
  Training Mode: Create a new model and train the network
  """
    ### Prepare data for training
    print("Prepare vocab dict and read pretrained word embeddings ...")
    vocab_dict, word_embedding_array = DataProcessor(
    ).prepare_vocab_embeddingdict()
    # vocab_dict contains _PAD and _UNK but not word_embedding_array

    print("Prepare training data ...")
    train_data = DataProcessor().prepare_news_data(vocab_dict,
                                                   data_type="training")

    print("Prepare validation data ...")
    # data in whole batch with padded matrixes
    val_batch = batch_load_data(DataProcessor().prepare_news_data(
        vocab_dict, data_type="validation"))

    fil_lens_to_test = FLAGS.max_filter_length - FLAGS.min_filter_length + 1
    if FLAGS.handle_filter_output == "concat" and FLAGS.sentembed_size % fil_lens_to_test != 0:
        q = int(FLAGS.sentembed_size // fil_lens_to_test)
        FLAGS.sentembed_size = q * fil_lens_to_test
        print("corrected embedding size: %d" % FLAGS.sentembed_size)

    # Create Model with various operations
    model = MY_Model(sess, len(vocab_dict) - 2)

    init_epoch = 1
    # Resume training if indicated Select the model
    if FLAGS.model_to_load != -1:
        if (os.path.isfile(FLAGS.train_dir + "/step-a.model.ckpt.epoch-" +
                           str(FLAGS.model_to_load))):
            selected_modelpath = FLAGS.train_dir + "/step-a.model.ckpt.epoch-" + str(
                FLAGS.model_to_load)
        else:
            print("Model not found in checkpoint folder.")
            exit(0)

        # Reload saved model and test
        init_epoch = FLAGS.model_to_load + 1
        print("Reading model parameters from %s" % selected_modelpath)
        model.saver.restore(sess, selected_modelpath)
        print("Model loaded.")

    # Initialize word embedding before training
    print(
        "Initialize word embedding vocabulary with pretrained embeddings ...")
    sess.run(model.vocab_embed_variable.assign(word_embedding_array))

    ### STEP A : Start Pretraining the policy with Supervised Labels: Simple Cross Entropy Training

    for epoch in range(init_epoch, FLAGS.train_epoch_crossentropy + 1):
        ep_time = time.time()  # to check duration

        print("STEP A: Epoch " + str(epoch) +
              " : Start pretraining with supervised labels")

        print("STEP A: Epoch " + str(epoch) +
              " : Reshuffle training document indices")
        train_data.shuffle_fileindices()

        # Start Batch Training
        step = 1
        total_ce_loss = 0
        counter = 0
        while (step * FLAGS.batch_size) <= len(train_data.fileindices):
            # Get batch data as Numpy Arrays
            batch = train_data.get_batch(((step - 1) * FLAGS.batch_size),
                                         (step * FLAGS.batch_size))

            # Run optimizer: optimize policy and reward estimator
            _, batch_logits, ce_loss, merged_summ = sess.run(
                [
                    model.train_op_policynet_withgold, model.logits,
                    model.cross_entropy_loss, model.merged
                ],
                feed_dict={
                    model.document_placeholder: batch.docs,
                    model.label_placeholder: batch.labels,
                    model.weight_placeholder: batch.weights
                })
            total_ce_loss += ce_loss
            # Print the progress
            if (step % FLAGS.training_checkpoint) == 0:
                probs = sess.run(
                    model.predictions,
                    feed_dict={model.logits_placeholder: batch_logits})
                training_acc = accuracy_qas_top(probs, batch.labels,
                                                batch.weights)
                acc_sum = sess.run(
                    model.tstepa_accuracy_summary,
                    feed_dict={model.train_acc_placeholder: training_acc})

                total_ce_loss /= FLAGS.training_checkpoint
                print("STEP A: Epoch " + str(epoch) + " : Covered " +
                      str(step * FLAGS.batch_size) + "/" +
                      str(len(train_data.fileindices)) +
                      " : Minibatch CE Loss= {:.6f}".format(total_ce_loss) +
                      ", Minibatch Accuracy= {:.6f}".format(training_acc))
                total_ce_loss = 0
                # Print Summary to Tensor Board
                model.summary_writer.add_summary(
                    merged_summ, (epoch - 1) * len(train_data.fileindices) +
                    step * FLAGS.batch_size)
                model.summary_writer.add_summary(
                    acc_sum, (epoch - 1) * len(train_data.fileindices) +
                    step * FLAGS.batch_size)
            # Increase step
            step += 1

            # if step == 100:
            # break

            #END-WHILE-TRAINING

        # Save Model
        print("STEP A: Epoch " + str(epoch) +
              " : Saving model after epoch completion")
        checkpoint_path = os.path.join(FLAGS.train_dir,
                                       "step-a.model.ckpt.epoch-" + str(epoch))
        model.saver.save(sess, checkpoint_path)

        # Performance on the validation set
        print("STEP A: Epoch " + str(epoch) +
              " : Performance on the validation data")
        # Get Predictions: Prohibit the use of gold labels
        FLAGS.authorise_gold_label = False
        FLAGS.use_dropout = False
        val_batch = batch_predict_with_a_model(val_batch,
                                               "validation",
                                               model,
                                               session=sess)
        FLAGS.use_dropout = True

        # Validation Accuracy and Prediction
        probs = sess.run(
            model.predictions,
            feed_dict={model.logits_placeholder: val_batch.logits})
        validation_acc = accuracy_qas_top(probs, val_batch.labels,
                                          val_batch.weights,
                                          val_batch.isf_score_ids)

        ce_loss_val, ce_loss_sum, acc_sum = sess.run(
            [
                model.cross_entropy_loss_val, model.ce_loss_summary_val,
                model.vstepa_accuracy_summary
            ],
            feed_dict={
                model.logits_placeholder: val_batch.logits,
                model.label_placeholder: val_batch.labels,
                model.weight_placeholder: val_batch.weights,
                model.val_acc_placeholder: validation_acc
            })

        # Print Validation Summary
        model.summary_writer.add_summary(acc_sum,
                                         epoch * len(train_data.fileindices))
        model.summary_writer.add_summary(ce_loss_sum,
                                         epoch * len(train_data.fileindices))
        print("STEP A: Epoch %s : Validation (%s) CE loss = %.6f" %
              (str(epoch), str(val_batch.docs.shape[0]), ce_loss_val))
        print("STEP A: Epoch %s : Validation (%s) Accuracy= %.6f" %
              (str(epoch), str(val_batch.docs.shape[0]), validation_acc))

        # Estimate MRR on validation set
        mrr_score = mrr_metric(probs, val_batch.labels, val_batch.weights,
                               val_batch.isf_score_ids, "validation")
        print("STEP A: Epoch %s : Validation (%s) MRR= %.6f" %
              (str(epoch), str(val_batch.docs.shape[0]), mrr_score))
        # Estimate MAP score on validation set
        mapsc = map_score(probs, val_batch.labels, val_batch.weights,
                          val_batch.isf_score_ids, "validation")
        print("STEP A: Epoch %s : Validation (%s) MAP= %.6f" %
              (str(epoch), str(val_batch.docs.shape[0]), mapsc))

        fn = "%s/step-a.model.ckpt.validation-metrics" % (FLAGS.train_dir)
        save_metrics(fn, FLAGS.load_prediction, validation_acc, mrr_score,
                     mapsc)

        print("STEP A: Epoch %d : Duration: %.4f" %
              (epoch, time.time() - ep_time))

    #END-FOR-EPOCH

    print("Optimization Finished!")
def test_val(sess):
  """
  Test on validation Mode: Loads an existing model and test it on the validation set
  """
  if FLAGS.load_prediction != -1:
    print("====================================== [%d] ======================================" % (FLAGS.load_prediction))

  ### Prepare data for training
  print("Prepare vocab dict and read pretrained word embeddings ...")
  vocab_dict, word_embedding_array = DataProcessor().prepare_vocab_embeddingdict()
  # vocab_dict contains _PAD and _UNK but not word_embedding_array

  print("Prepare test data ...")
  train_data = DataProcessor().prepare_news_data(vocab_dict, data_type="training")
  val_batch = batch_load_data(DataProcessor().prepare_news_data(vocab_dict,
                                                                data_type="validation",
                                                                normalizer=train_data.normalizer,
                                                                pca_model=train_data.pca_model))
  del train_data

  # Create Model with various operations
  model = MY_Model(sess, len(vocab_dict)-2)

  # # Initialize word embedding before training
  #print("Initialize word embedding vocabulary with pretrained embeddings ...")
  #sess.run(model.vocab_embed_variable.assign(word_embedding_array))

  # Select the model
  if (os.path.isfile(FLAGS.train_dir+"/step-a.model.ckpt.epoch-"+str(FLAGS.model_to_load))):
    selected_modelpath = FLAGS.train_dir+"/step-a.model.ckpt.epoch-"+str(FLAGS.model_to_load)
  else:
    print("Model not found in checkpoint folder.")
    exit(0)

  # Reload saved model and test
  print("Reading model parameters from %s" % selected_modelpath)
  model.saver.restore(sess, selected_modelpath)
  print("Model loaded.")

  # Initialize word embedding before training
  print("Initialize word embedding vocabulary with pretrained embeddings ...")
  sess.run(model.vocab_embed_variable.assign(word_embedding_array))

  # Test Accuracy and Prediction
  print("Performance on the validation data:")
  FLAGS.authorise_gold_label = False
  val_batch = batch_predict_with_a_model(val_batch,"validation", model, session=sess)
  FLAGS.authorise_gold_label = True
  probs = sess.run(model.probs,feed_dict={model.logits_placeholder: val_batch.logits})

  # Validation Accuracy and Prediction
  validation_acc = accuracy_qas_top(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids)

  # Print Validation Summary
  print("Validation (%s) Accuracy= %.6f" % (str(val_batch.docs.shape[0]),validation_acc))
  # Estimate MRR on validation set
  mrr_score = mrr_metric(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids)
  print("Validation (%s) MRR= %.6f" % (str(val_batch.docs.shape[0]),mrr_score))
  # Estimate MAP score on validation set
  mapsc = map_score(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids)
  print("Validation (%s) MAP= %.6f" % (str(val_batch.docs.shape[0]),mapsc))

  if FLAGS.load_prediction != -1:
    fn = ''
    if FLAGS.filtered_setting:
      fn = "%s/step-a.model.ckpt.%s-top%d-isf-metrics" % (FLAGS.train_dir,"validation",FLAGS.topK)
    else:
      fn = "%s/step-a.model.ckpt.%s-metrics" % (FLAGS.train_dir,"validation")
    save_metrics(fn,FLAGS.load_prediction,validation_acc,mrr_score,mapsc)

  # Writing validation predictions, embeddings
  print("Writing final validation summaries and embeddings")
  modelname = "step-a.model.ckpt.epoch-" + str(FLAGS.model_to_load)
  write_prediction_summaries(val_batch, probs, modelname, "validation")
  write_cos_sim(val_batch.cos_sim, modelname + "-cos_sim","validation")
Esempio n. 8
0
            # Increase step
            if step%5000==0:
              print ("\tStep: ",step)
            step += 1
          #END-WHILE-TRAINING
          total_loss /= step
          FLAGS.authorise_gold_label = False
          FLAGS.use_dropout = False
          # retrieve batch with updated logits in it
          val_batch = batch_predict_with_a_model(val_batch, "validation", model, session=sess)
          FLAGS.authorise_gold_label = True
          FLAGS.use_dropout = prev_drpt
          probs = sess.run(model.predictions,feed_dict={model.logits_placeholder: val_batch.logits})
          probs,lab,w = group_by_doc(probs,val_batch.labels,val_batch.qids)
          validation_acc = accuracy_qas_top(probs,lab,w)
          val_mrr = mrr_metric(probs,lab,w,"validation")
 
          print("\tEpoch %2d || Train ce_loss: %4.3f || Val acc: %.4f || Val mrr: %.4f || duration: %3.2f" % 
            (epoch,total_loss,validation_acc,val_mrr,time.time()-ep_time))
          output.write("\tEpoch %2d || Train ce_loss: %4.3f || Val acc: %.4f || Val mrr: %.4f || duration: %3.2f\n" % 
            (epoch,total_loss,validation_acc,val_mrr,time.time()-ep_time))

          if validation_acc > best_acc:
            best_acc = validation_acc
            best_ep = epoch
          if val_mrr > best_mrr:
            best_mrr = val_mrr
            best_ep_mrr = epoch
          #break # for time testing
        #END-FOR-EPOCH
        results_by_id[setup_id] = (best_acc,best_ep)
Esempio n. 9
0
            total_loss += ce_loss
            # Increase step
            if step%500==0:
              print ("\tStep: ",step)
            step += 1
          #END-WHILE-TRAINING
          total_loss /= step
          FLAGS.authorise_gold_label = False
          FLAGS.use_dropout = False
          # retrieve batch with updated logits in it
          val_batch = batch_predict_with_a_model(val_batch, "validation", model, session=sess)
          FLAGS.authorise_gold_label = True
          FLAGS.use_dropout = prev_drpt
          probs = sess.run(model.predictions,feed_dict={model.logits_placeholder: val_batch.logits})
          validation_acc = accuracy_qas_top(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids)
          val_mrr = mrr_metric(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids,"validation")

          print("\tEpoch %2d || Train ce_loss: %4.3f || Val acc: %.4f || Val mrr: %.4f || duration: %3.2f" % 
            (epoch,total_loss,validation_acc,val_mrr,time.time()-ep_time))
          output.write("\tEpoch %2d || Train ce_loss: %4.3f || Val acc: %.4f || Val mrr: %.4f || duration: %3.2f\n" % 
            (epoch,total_loss,validation_acc,val_mrr,time.time()-ep_time))

          if validation_acc > best_acc:
            best_acc = validation_acc
            best_ep = epoch
          if val_mrr > best_mrr:
            best_mrr = val_mrr
            best_ep_mrr = epoch
          #break # for time testing
        #END-FOR-EPOCH
        results_by_id[setup_id] = (best_acc,best_ep)
Esempio n. 10
0
def evaluate_model(assignments,train_data,val_batch,score="mrr"):

  FLAGS.batch_size = assignments["batch_size"]
  FLAGS.learning_rate = math.exp(assignments["log_learning_rate"])
  FLAGS.size = assignments["size"]
  FLAGS.sentembed_size = assignments["sentembed_size"]

  #FLAGS.dropout = setup["dropout"]

  fil_lens_to_test = FLAGS.max_filter_length - FLAGS.min_filter_length + 1
  if FLAGS.handle_filter_output == "concat" and FLAGS.sentembed_size%fil_lens_to_test != 0:
    q = int(FLAGS.sentembed_size // fil_lens_to_test)
    FLAGS.sentembed_size = q * fil_lens_to_test

  print("Setup: bs: %d | lr: %f | size: %d | sent_emb: %d" % 
    (FLAGS.batch_size,
     FLAGS.learning_rate,
     FLAGS.size,
     FLAGS.sentembed_size)
    )

  with tf.Graph().as_default() and tf.device('/gpu:'+FLAGS.gpu_id):
    config = tf.ConfigProto(allow_soft_placement = True)
    tf.set_random_seed(seed)
    with tf.Session(config = config) as sess:
      model = MY_Model(sess, len(vocab_dict)-2)
      init_epoch = 1
      sess.run(model.vocab_embed_variable.assign(word_embedding_array))
      
      best_metric = -1
      best_ep = 0
      for epoch in range(init_epoch, FLAGS.train_epoch_crossentropy+1):
        ep_time = time.time() # to check duration

        train_data.shuffle_fileindices()
        total_loss = 0
        # Start Batch Training
        step = 1
        while (step * FLAGS.batch_size) <= len(train_data.fileindices):
          # Get batch data as Numpy Arrays
          batch = train_data.get_batch(((step-1)*FLAGS.batch_size), (step * FLAGS.batch_size))

          # Run optimizer: optimize policy and reward estimator
          _,ce_loss = sess.run([model.train_op_policynet_withgold,
                                model.cross_entropy_loss],
                                feed_dict={model.document_placeholder: batch.docs,
                                           model.label_placeholder: batch.labels,
                                           model.weight_placeholder: batch.weights,
                                           model.isf_score_placeholder: batch.isf_score,
                                           model.idf_score_placeholder: batch.idf_score,
                                           model.locisf_score_placeholder: batch.locisf_score})
          total_loss += ce_loss
          # Increase step
          if step%500==0:
            print ("\tStep: ",step)
          step += 1
        #END-WHILE-TRAINING
        total_loss /= step
        FLAGS.authorise_gold_label = False
        #FLAGS.use_dropout = False
        # retrieve batch with updated logits in it
        val_batch = batch_predict_with_a_model(val_batch, "validation", model, session=sess)
        FLAGS.authorise_gold_label = True
        #FLAGS.use_dropout = True
        probs = sess.run(model.predictions,feed_dict={model.logits_placeholder: val_batch.logits})
        if score=="acc":
          metric = accuracy_qas_top(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids)
        elif score=="mrr":
          metric = mrr_metric(probs, val_batch.labels, val_batch.weights, val_batch.isf_score_ids,"validation")
        
        print("\tEpoch %2d || Train ce_loss: %4.3f || Val %s: %.4f || duration: %3.2f" % (epoch,total_loss,score,metric,time.time()-ep_time))
        
        if metric > best_metric:
          best_metric = metric
          best_ep = epoch
      #END-FOR-EPOCH
    # clear graph
    tf.reset_default_graph()
  #END-GRAPH
  print("Best metric:%.6f | ep: %d" % (best_metric,best_ep))

  return best_metric