Exemple #1
0
def predict(sdae, data_set, bias_node=False):
    with sdae.session.graph.as_default():
        labels_placeholder = tf.placeholder(tf.int32, shape=1,\
                                            name='labels_placeholder')
        examples_placeholder = tf.placeholder(tf.float32,\
                                              shape=(1, sdae._net_shape[0]),\
                                              name='input_pl')
        
        logits = tf.identity(examples_placeholder)
        
        for layer in sdae.get_layers:
            if bias_node:
                bias_n = tf.ones(shape=[1, 1], dtype=tf.float32)
                logits = tf.concat(1, [bias_n, logits])
            logits = layer.clean_activation(x_in=logits, use_fixed=False)
            
        predictions = tf.argmax(logits, 1)
        
        labels = tf.identity(labels_placeholder)
        
        y_pred = []
        y_true = []
        
        for _ in xrange(data_set.num_examples):
            feed_dict = fill_feed_dict(data_set,
                                       examples_placeholder,
                                       labels_placeholder, 1)
    
            y_prediction, y_trues = sdae.session.run([predictions, labels], feed_dict=feed_dict)
            y_pred += list(y_prediction)
            y_true += list(y_trues)
            
#         print(y_pred)
        return y_pred, y_true
Exemple #2
0
def predict(sdae, data_set, bias_node=False):
    with sdae.session.graph.as_default():
        labels_placeholder = tf.placeholder(tf.int32, shape=1,\
                                            name='labels_placeholder')
        examples_placeholder = tf.placeholder(tf.float32,\
                                              shape=(1, sdae._net_shape[0]),\
                                              name='input_pl')

        logits = tf.identity(examples_placeholder)

        for layer in sdae.get_layers:
            if bias_node:
                bias_n = tf.ones(shape=[1, 1], dtype=tf.float32)
                logits = tf.concat(1, [bias_n, logits])
            logits = layer.clean_activation(x_in=logits, use_fixed=False)

        predictions = tf.argmax(logits, 1)

        labels = tf.identity(labels_placeholder)

        y_pred = []
        y_true = []

        for _ in xrange(data_set.num_examples):
            feed_dict = fill_feed_dict(data_set, examples_placeholder,
                                       labels_placeholder, 1)

            y_prediction, y_trues = sdae.session.run([predictions, labels],
                                                     feed_dict=feed_dict)
            y_pred += list(y_prediction)
            y_true += list(y_trues)


#         print(y_pred)
        return y_pred, y_true
def minibatch_transformation(dataset):
    patches_pl, labels_pl = utils.placeholder_inputs()
    feed_dict = utils.fill_feed_dict(dataset.train,
                                     patches_pl,
                                     labels_pl,
                                     36,
                                     augment=True)

    for patch in feed_dict[patches_pl]:
        cv2.imshow('patch', patch)
        cv2.waitKey(0)
Exemple #4
0
def do_eval_summary(tag, sess, eval_correct, examples_placeholder,
                    labels_placeholder, data_set):
    true_count = 0
    steps_per_epoch = data_set.num_examples // FLAGS.batch_size
    num_examples = steps_per_epoch * FLAGS.batch_size
    for _ in xrange(steps_per_epoch):
        feed_dict = fill_feed_dict(data_set, examples_placeholder,
                                   labels_placeholder)
        true_count += sess.run(eval_correct, feed_dict=feed_dict)
    error = 1 - true_count / num_examples

    return sess.run(tf.scalar_summary(tag, tf.identity(error)))
def do_eval(sess, eval_correct, keep_prob):
    """Runs one evaluation against the full epoch of data.

  Args:
    sess: The session in which the model has been trained.
    eval_correct: The Tensor that returns the number of correct predictions.
    keep_prob: The keep prob placeholder.
  """
    # And run one epoch of eval.
    true_count = 0  # Counts the number of correct predictions.
    steps_per_epoch = data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TEST // FLAGS.batch_size
    num_examples = steps_per_epoch * FLAGS.batch_size
    for step in xrange(steps_per_epoch):
        feed_dict = utils.fill_feed_dict(keep_prob, train=False)
        true_count += sess.run(eval_correct, feed_dict=feed_dict)
    precision = true_count / num_examples
    print("  Num examples: %d  Num correct: %d  Precision @ 1: %0.04f" % (num_examples, true_count, precision))
Exemple #6
0
def do_eval_summary(tag,
                    sess,
                    eval_correct,
                    examples_placeholder,
                    labels_placeholder,
                    data_set):
    true_count = 0
    steps_per_epoch = data_set.num_examples // FLAGS.batch_size
    num_examples = steps_per_epoch * FLAGS.batch_size
    for _ in xrange(steps_per_epoch):
        feed_dict = fill_feed_dict(data_set,
                                   examples_placeholder,
                                   labels_placeholder)
        true_count += sess.run(eval_correct, feed_dict=feed_dict)
    error = 1 - true_count / num_examples
    
    return sess.run(tf.scalar_summary(tag, tf.identity(error)))
Exemple #7
0
def do_eval(sess, eval_correct, keep_prob):
    """Runs one evaluation against the full epoch of data.

  Args:
    sess: The session in which the model has been trained.
    eval_correct: The Tensor that returns the number of correct predictions.
    keep_prob: The keep prob placeholder.
  """
    # And run one epoch of eval.
    true_count = 0  # Counts the number of correct predictions.
    steps_per_epoch = data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TEST // FLAGS.batch_size
    num_examples = steps_per_epoch * FLAGS.batch_size
    for step in xrange(steps_per_epoch):
        feed_dict = utils.fill_feed_dict(keep_prob, train=False)
        true_count += sess.run(eval_correct, feed_dict=feed_dict)
    precision = true_count / num_examples
    print('  Num examples: %d  Num correct: %d  Precision @ 1: %0.04f' %
          (num_examples, true_count, precision))
def run_training():
    """Train BinaryConnect."""
    # Get the sets of images and labels for training, validation, and
    # test on CIFAR10.
    data_sets = cifar10.read_data_sets(dst_dir='./dataset',
                                       validation_size=5000)

    # Tell TensorFlow that the model will be built into the default Graph.
    with tf.Graph().as_default():
        # Generate placeholders for the images and labels.
        images_placeholder, labels_placeholder, train_placeholder = placeholder_inputs(
            FLAGS.batch_size)

        # Build a Graph that computes predictions from the inference model.
        logits = bc.inference_bin(images_placeholder, train_placeholder,
                                  stochastic=FLAGS.stochastic,
                                  use_bnorm=True) \
            if FLAGS.binary \
            else bc.inference_ref(images_placeholder, train_placeholder,
                                  use_bnorm=True)

        # Add to the Graph the Ops for loss calculation.
        loss = bc.loss(logits, labels_placeholder)

        # Add to the Graph the Ops that calculate and apply gradients.
        train_op = bc.training(loss, FLAGS.learning_rate)

        # Add the Op to compare the logits to the labels during evaluation.
        eval_metric = bc.evaluation(logits, labels_placeholder)

        # Add a placeholder for logging execution time
        # frequency_placeholder = tf.placeholder(tf.float32, shape=())
        # tf.summary.scalar('Execution Time', frequency_placeholder)
        # TODO: support a d separate summary for metadata (e.g. execution time)

        # Build the summary Tensor based on the TF collection of Summaries.
        summary = tf.summary.merge_all()

        # Add the variable initializer Op.
        ivars = tf.global_variables() + tf.local_variables()
        init = tf.variables_initializer(ivars)

        # Create a saver for writing training checkpoints.
        saver = tf.train.Saver()

        # Create a logger to the validation accuracy
        val_acc_pl = tf.placeholder(tf.float32, shape=())
        summary_val_acc = tf.summary.scalar(name='validation_acc',
                                            tensor=val_acc_pl,
                                            collections=['validation'])
        summary_val = tf.summary.merge([summary_val_acc])

        # Create a session for running Ops on the Graph.
        sess = tf.Session()

        # Instantiate a SummaryWriter to output summaries and the Graph.
        summary_writer_train = tf.summary.FileWriter(
            os.path.join(FLAGS.log_dir, 'train'), sess.graph)
        summary_writer_val = tf.summary.FileWriter(
            os.path.join(FLAGS.log_dir, 'val'), sess.graph)

        # And then after everything is built:

        # Run the Op to initialize the variables.
        sess.run(init)

        # Start the training loop.
        duration = 0
        tp_value_total = 0
        for step in xrange(FLAGS.max_steps):
            start_time = time.time()

            # Fill a feed dictionary with the actual set of images and labels
            # for this particular training step.
            feed_dict = fill_feed_dict(data_sets.train, images_placeholder,
                                       labels_placeholder, train_placeholder,
                                       True)

            # Run one step of the model.  The return values are the activations
            # from the `train_op` (which is discarded) and the `loss` Op.  To
            # inspect the values of your Ops or variables, you may include them
            # in the list passed to sess.run() and the value tensors will be
            # returned in the tuple from the call.
            _, loss_value, acc_val = sess.run([train_op, loss, eval_metric],
                                              feed_dict=feed_dict)

            duration += time.time() - start_time
            tp_value_total += acc_val

            # Write the summaries and print an overview fairly often.
            if step % 100 == 0 and step > 0:
                # Print status to stdout.
                images_freq = 100 * FLAGS.batch_size / duration
                print(
                    'Step %d: loss = %.2f, correct = %.2f%% (%.3f images/sec)'
                    % (step, loss_value, tp_value_total / FLAGS.batch_size,
                       images_freq))
                duration = time.time() - start_time
                tp_value_total = 0
                duration = 0
                # Update the events file.
                summary_str = sess.run(summary, feed_dict=feed_dict)
                summary_writer_train.add_summary(summary_str, step)
                summary_writer_train.flush()

            # Save a checkpoint and evaluate the model periodically.
            if (step + 1) % 500 == 0 or (step + 1) == FLAGS.max_steps:
                checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt')
                saver.save(sess, checkpoint_file, global_step=step)
                # Evaluate against the training set.
                # print('Training Data Eval:')
                # do_eval(sess,
                #         eval_metric,
                #         images_placeholder,
                #         labels_placeholder,
                #         train_placeholder,
                #         data_sets.train, summary)
                # Evaluate against the validation set.
                print('Validation Data Eval:')
                accuracy_val = do_eval(sess, eval_metric, images_placeholder,
                                       labels_placeholder, train_placeholder,
                                       data_sets.validation)
                # TODO: find a way to collect summaries for validation
                summary_str = sess.run(summary_val,
                                       feed_dict={val_acc_pl: accuracy_val})
                summary_writer_val.add_summary(summary_str, step)
                summary_writer_val.flush()

                # Evaluate against the test set.
                print('Test Data Eval:')
                do_eval(sess, eval_metric, images_placeholder,
                        labels_placeholder, train_placeholder, data_sets.test)
Exemple #9
0
def do_eval(sess,
            eval_correct,
            predictions,
            examples_placeholder,
            labels_placeholder,
            label_map,
            data_set,
            title='Evaluation'):
    """Runs one evaluation against the full epoch of data.
    Args:
      sess: The session in which the model has been trained.
      eval_correct: The Tensor that returns the number of correct predictions.
      images_placeholder: The images placeholder.
      labels_placeholder: The labels placeholder.
      data_set: The set of images and labels to evaluate, from
        utils.read_data_sets().
    """
    # And run one epoch of eval.
    true_count = 0  # Counts the number of correct predictions.
    y_pred = []
    y_true = []
    steps_per_epoch = data_set.num_examples // FLAGS.batch_size
    num_examples = steps_per_epoch * FLAGS.batch_size
    
    labels = tf.identity(labels_placeholder)
    
    for _ in xrange(steps_per_epoch):
        feed_dict = fill_feed_dict(data_set,
                                   examples_placeholder,
                                   labels_placeholder)
        corrects, y_prediction, y_trues = sess.run([eval_correct, predictions,\
                                                    labels], feed_dict=feed_dict)
        true_count += corrects
        y_pred += list(y_prediction)
        y_true += list(y_trues)
        
    accuracy = true_count / num_examples
    print(title + ' - Num examples: %d  Num correct: %d  Accuracy_score @ 1: %0.08f' %
          (num_examples, true_count, accuracy))

#     print("True output:", y_true)
#     print("Pred output:", y_pred)
    
    print("Precision:")
    print("\tNone: ", precision_score(y_true, y_pred, average=None, pos_label=None))
#     print("\tBinary:", precision_score(y_true, y_pred, average='binary'))
    print("\tMicro: %0.08f" % precision_score(y_true, y_pred, average='micro', pos_label=None))
    print("\tMacro: %0.08f" % precision_score(y_true, y_pred, average='macro', pos_label=None))
    print("\tWeighted: %0.08f" % precision_score(y_true, y_pred, average='weighted', pos_label=None))
#     print("\tSamples:", sklearn.metrics.precision_score(y_true, y_pred, average='samples'))
#     print("\tAccuracy_score: %0.08f" % accuracy_score(y_true, y_pred))
     
    print("Recall:")
#     print("\tNone: ", recall_score(y_true, y_pred, average=None, pos_label=None))
#     print("\tBinary:", recall_score(y_true, y_pred, average='binary'))
    print("\tMicro: %0.08f" % recall_score(y_true, y_pred, average='micro', pos_label=None))
    print("\tMacro: %0.08f" % recall_score(y_true, y_pred, average='macro', pos_label=None))
    print("\tWeighted: %0.08f" % recall_score(y_true, y_pred, average='weighted', pos_label=None))
#     print("\tSamples:", sklearn.metrics.recall_score(y_true, y_pred, average='samples'))    
    
#     print("F1_score:")
#     print("\tNone: ", f1_score(y_true, y_pred, average=None, pos_label=None))
#     print("\tBinary:", f1_score(y_true, y_pred, average='binary'))
#     print("\tMicro: %0.08f" % f1_score(y_true, y_pred, average='micro', pos_label=None))
#     print("\tMacro: %0.08f" % f1_score(y_true, y_pred, average='macro', pos_label=None))
    print("\nF1 Score (weighted): %0.08f" % f1_score(y_true, y_pred, average='weighted', pos_label=None))
#     print("\tSamples:", sklearn.metrics.f1_score(y_true, y_pred, average='samples'))

#     print("True Length:", len(y_true))
#     print("Prediction Length:", len(y_pred))

    cm = confusion_matrix(y_true, y_pred)
#     print("\nConfusion Matrix")
#     print(cm)
    
    cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
#     print("\nNormalized confusion_matrix")
#     print(cm_normalized)

    print("")
    print(classification_report(y_true, y_pred, target_names=label_map))

    pcm(cm, target_names=label_map, title=title)
    pcm(cm_normalized, target_names=label_map, title=title+"_Normalized")
    
    roc(y_pred, y_true, n_classes=len(label_map), title=title)
    
    print("\n=====================================================================================================\n")
Exemple #10
0
def run_training():
  """Train model for a number of steps."""
  # Get the sets of images and labels for training, validation, and
  # test on MNIST.

  # Tell TensorFlow that the model will be built into the default Graph.
  train_dir = os.path.join(FLAGS.model_dir,FLAGS.name)

  with tf.Graph().as_default():

    global_step = tf.Variable(0, trainable=False)

    with tf.name_scope('Input'):
      image_batch, label_batch = data_input.distorted_inputs(FLAGS.data_dir, FLAGS.batch_size)
    # Generate placeholders for the images and labels.
      keep_prob = utils.placeholder_inputs(FLAGS.batch_size)

    # Build a Graph that computes predictions from the inference model.
    logits = model.inference(image_batch, keep_prob)

    # Add to the Graph the Ops for loss calculation.
    loss = model.loss(logits, label_batch)

    # Add to the Graph the Ops that calculate and apply gradients.
    train_op = model.training(loss, global_step=global_step, learning_rate=FLAGS.learning_rate)

    # Add the Op to compare the logits to the labels during evaluation.
    eval_correct = model.evaluation(logits, label_batch)

    # Build the summary operation based on the TF collection of Summaries.
    summary_op = tf.merge_all_summaries()

    # Create a saver for writing training checkpoints.
    saver = tf.train.Saver()

    # Create a session for running Ops on the Graph.
    sess = tf.Session()

    # Run the Op to initialize the variables.
    init = tf.initialize_all_variables()
    sess.run(init)

    # Start the queue runners.
    tf.train.start_queue_runners(sess=sess)

    # Instantiate a SummaryWriter to output summaries and the Graph.
    summary_writer = tf.train.SummaryWriter(train_dir,
                                            graph_def=sess.graph_def)

    # And then after everything is built, start the training loop.
    for step in xrange(FLAGS.max_steps):
      start_time = time.time()

      # Fill a feed dictionary with the actual set of images and labels
      # for this particular training step.
      feed_dict = utils.fill_feed_dict(keep_prob,
                                       train = True)

      # Run one step of the model.  The return values are the activations
      # from the `train_op` (which is discarded) and the `loss` Op.  To
      # inspect the values of your Ops or variables, you may include them
      # in the list passed to sess.run() and the value tensors will be
      # returned in the tuple from the call.
      _, loss_value = sess.run([train_op, loss],
                               feed_dict=feed_dict)

      # Write the summaries and print an overview fairly often.
      if step % 100 == 0:
        # Print status to stdout.
        duration = time.time() - start_time
        examples_per_sec = FLAGS.batch_size / duration
        sec_per_batch = float(duration)
        print('Step %d: loss = %.2f ( %.3f sec (per Batch); %.1f examples/sec;)' % (step, loss_value,
                                     sec_per_batch, examples_per_sec))
        # Update the events file.
        summary_str = sess.run(summary_op, feed_dict=feed_dict)
        summary_writer.add_summary(summary_str, step)

      # Save a checkpoint and evaluate the model periodically.
      if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
        checkpoint_path = os.path.join(train_dir, 'model.ckpt')
        saver.save(sess, checkpoint_path , global_step=step)
        # Evaluate against the training set.

      if (step + 1) % 10000 == 0 or (step + 1) == FLAGS.max_steps:  
        print('Training Data Eval:')
        utils.do_eval(sess,
                      eval_correct,
                      keep_prob,
                      data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)
Exemple #11
0
def do_eval(sess,
            eval_correct,
            predictions,
            examples_placeholder,
            labels_placeholder,
            label_map,
            data_set,
            title='Evaluation'):
    """Runs one evaluation against the full epoch of data.
    Args:
      sess: The session in which the model has been trained.
      eval_correct: The Tensor that returns the number of correct predictions.
      images_placeholder: The images placeholder.
      labels_placeholder: The labels placeholder.
      data_set: The set of images and labels to evaluate, from
        utils.read_data_sets().
    """
    # And run one epoch of eval.
    true_count = 0  # Counts the number of correct predictions.
    y_pred = []
    y_true = []
    steps_per_epoch = data_set.num_examples // FLAGS.batch_size
    num_examples = steps_per_epoch * FLAGS.batch_size

    labels = tf.identity(labels_placeholder)

    for _ in xrange(steps_per_epoch):
        feed_dict = fill_feed_dict(data_set, examples_placeholder,
                                   labels_placeholder)
        corrects, y_prediction, y_trues = sess.run([eval_correct, predictions,\
                                                    labels], feed_dict=feed_dict)
        true_count += corrects
        y_pred += list(y_prediction)
        y_true += list(y_trues)

    accuracy = true_count / num_examples
    print(title +
          ' - Num examples: %d  Num correct: %d  Accuracy_score @ 1: %0.08f' %
          (num_examples, true_count, accuracy))

    #     print("True output:", y_true)
    #     print("Pred output:", y_pred)

    print("Precision:")
    print("\tNone: ",
          precision_score(y_true, y_pred, average=None, pos_label=None))
    #     print("\tBinary:", precision_score(y_true, y_pred, average='binary'))
    print("\tMicro: %0.08f" %
          precision_score(y_true, y_pred, average='micro', pos_label=None))
    print("\tMacro: %0.08f" %
          precision_score(y_true, y_pred, average='macro', pos_label=None))
    print("\tWeighted: %0.08f" %
          precision_score(y_true, y_pred, average='weighted', pos_label=None))
    #     print("\tSamples:", sklearn.metrics.precision_score(y_true, y_pred, average='samples'))
    #     print("\tAccuracy_score: %0.08f" % accuracy_score(y_true, y_pred))

    print("Recall:")
    #     print("\tNone: ", recall_score(y_true, y_pred, average=None, pos_label=None))
    #     print("\tBinary:", recall_score(y_true, y_pred, average='binary'))
    print("\tMicro: %0.08f" %
          recall_score(y_true, y_pred, average='micro', pos_label=None))
    print("\tMacro: %0.08f" %
          recall_score(y_true, y_pred, average='macro', pos_label=None))
    print("\tWeighted: %0.08f" %
          recall_score(y_true, y_pred, average='weighted', pos_label=None))
    #     print("\tSamples:", sklearn.metrics.recall_score(y_true, y_pred, average='samples'))

    #     print("F1_score:")
    #     print("\tNone: ", f1_score(y_true, y_pred, average=None, pos_label=None))
    #     print("\tBinary:", f1_score(y_true, y_pred, average='binary'))
    #     print("\tMicro: %0.08f" % f1_score(y_true, y_pred, average='micro', pos_label=None))
    #     print("\tMacro: %0.08f" % f1_score(y_true, y_pred, average='macro', pos_label=None))
    print("\nF1 Score (weighted): %0.08f" %
          f1_score(y_true, y_pred, average='weighted', pos_label=None))
    #     print("\tSamples:", sklearn.metrics.f1_score(y_true, y_pred, average='samples'))

    #     print("True Length:", len(y_true))
    #     print("Prediction Length:", len(y_pred))

    cm = confusion_matrix(y_true, y_pred)
    #     print("\nConfusion Matrix")
    #     print(cm)

    cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    #     print("\nNormalized confusion_matrix")
    #     print(cm_normalized)

    print("")
    print(classification_report(y_true, y_pred, target_names=label_map))

    pcm(cm, target_names=label_map, title=title)
    pcm(cm_normalized, target_names=label_map, title=title + "_Normalized")

    roc(y_pred, y_true, n_classes=len(label_map), title=title)

    print(
        "\n=====================================================================================================\n"
    )
Exemple #12
0
def train(dataset, log_dir):
  with tf.Graph().as_default():
    # gets placeholders for patches and labels
    patches_pl, labels_pl = utils.placeholder_inputs()

    # build train related ops
    net = models.FCN(patches_pl, FLAGS.dropout)
    net.build_loss(labels_pl)
    net.build_train(FLAGS.learning_rate)

    # builds validation inference graph
    val_net = models.FCN(patches_pl, training=False, reuse=True)

    # add summary to plot loss, f score, tdr and fdr
    f_score_pl = tf.placeholder(tf.float32, shape=())
    tdr_pl = tf.placeholder(tf.float32, shape=())
    fdr_pl = tf.placeholder(tf.float32, shape=())
    scores_summary_op = tf.summary.merge([
        tf.summary.scalar('f_score', f_score_pl),
        tf.summary.scalar('tdr', tdr_pl),
        tf.summary.scalar('fdr', fdr_pl)
    ])
    loss_summary_op = tf.summary.scalar('loss', net.loss)

    # add variable initialization to graph
    init = tf.global_variables_initializer()

    # early stopping vars
    best_f_score = 0
    faults = 0
    saver = tf.train.Saver()
    with tf.Session() as sess:
      summary_writer = tf.summary.FileWriter(log_dir, sess.graph)

      sess.run(init)

      for step in range(1, FLAGS.steps + 1):
        feed_dict = utils.fill_feed_dict(dataset.train, patches_pl, labels_pl,
                                         FLAGS.batch_size)

        _, loss_value = sess.run([net.train, net.loss], feed_dict=feed_dict)

        # write loss summary periodically
        if step % 100 == 0:
          print('Step {}: loss = {}'.format(step, loss_value))
          summary_str = sess.run(loss_summary_op, feed_dict=feed_dict)
          summary_writer.add_summary(summary_str, step)

        # evaluate the model periodically
        if step % 1000 == 0:
          print('Evaluation:')
          f_score, fdr, tdr = validate.by_patches(sess, val_net.predictions,
                                                  FLAGS.batch_size, patches_pl,
                                                  labels_pl, dataset.val)
          print('TDR = {}'.format(tdr))
          print('FDR = {}'.format(fdr))
          print('F score = {}'.format(f_score))

          # early stopping
          if f_score > best_f_score:
            best_f_score = f_score
            saver.save(
                sess, os.path.join(log_dir, 'model.ckpt'), global_step=step)
            faults = 0
          else:
            faults += 1
            if faults >= FLAGS.tolerance:
              print('Training stopped early')
              break

          # write f score, tdr and fdr to summary
          scores_summary = sess.run(
              scores_summary_op,
              feed_dict={
                  f_score_pl: f_score,
                  tdr_pl: tdr,
                  fdr_pl: fdr
              })
          summary_writer.add_summary(scores_summary, global_step=step)

  print('Finished')
  print('best F score = {}'.format(best_f_score))
def train(dataset, log_dir):
    with tf.Graph().as_default():
        # gets placeholders for images and labels
        images_pl, labels_pl = utils.placeholder_inputs()

        # build net graph
        net = description.Net(images_pl, FLAGS.dropout)

        # build training related ops
        net.build_loss(labels_pl, FLAGS.weight_decay)
        net.build_train(FLAGS.learning_rate)

        # builds validation graph
        val_net = description.Net(images_pl, training=False, reuse=True)

        # add summary to plot loss and rank
        eer_pl = tf.placeholder(tf.float32, shape=(), name='eer_pl')
        loss_pl = tf.placeholder(tf.float32, shape=(), name='loss_pl')
        eer_summary_op = tf.summary.scalar('eer', eer_pl)
        loss_summary_op = tf.summary.scalar('loss', loss_pl)

        # early stopping vars
        best_eer = 1
        faults = 0
        saver = tf.train.Saver()
        with tf.Session() as sess:
            # initialize summary and variables
            summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
            sess.run(tf.global_variables_initializer())

            # 'compute_descriptors' function for validation
            compute_descriptors = lambda img, pts: utils.trained_descriptors(
                img,
                pts,
                patch_size=dataset.train.images_shape[1],
                session=sess,
                imgs_pl=images_pl,
                descs_op=val_net.descriptors)

            # train loop
            for step in range(1, FLAGS.steps + 1):
                # fill feed dict
                feed_dict = utils.fill_feed_dict(dataset.train, images_pl,
                                                 labels_pl, FLAGS.batch_size,
                                                 FLAGS.augment)
                # train step
                loss_value, _ = sess.run([net.loss, net.train],
                                         feed_dict=feed_dict)

                # write loss summary periodically
                if step % 100 == 0:
                    print('Step {}: loss = {}'.format(step, loss_value))

                    # summarize loss
                    loss_summary = sess.run(loss_summary_op,
                                            feed_dict={loss_pl: loss_value})
                    summary_writer.add_summary(loss_summary, step)

                # evaluate model periodically
                if step % 500 == 0 and dataset.val is not None:
                    print('Validation:')
                    eer = validate.matching.validation_eer(
                        dataset.val, compute_descriptors)
                    print('EER = {}'.format(eer))

                    # summarize eer
                    eer_summary = sess.run(eer_summary_op,
                                           feed_dict={eer_pl: eer})
                    summary_writer.add_summary(eer_summary, global_step=step)

                    # early stopping
                    if eer < best_eer:
                        # update early stopping vars
                        best_eer = eer
                        faults = 0
                        saver.save(sess,
                                   os.path.join(log_dir, 'model.ckpt'),
                                   global_step=step)
                    else:
                        faults += 1
                        if faults >= FLAGS.tolerance:
                            print('Training stopped early')
                            break

            # if no validation set, save model when training completes
            if dataset.val is None:
                saver.save(sess, os.path.join(log_dir, 'model.ckpt'))

    print('Finished')
    print('best EER = {}'.format(best_eer))
def by_patches(sess, preds, batch_size, patches_pl, labels_pl, dataset):
    '''
  Computes detection parameters that optimize the patch-based keypoint
  detection F-score in the dataset with a grid search. This is done
  efficiently, by sorting probability scores and iterating over them.

  Args:
    sess: tf session with loaded preds variables.
    preds: tf op for detection probability prediction.
    batch_size: size of mini-batch.
    patches_pl: patch input placeholder for preds op.
    labels_pl: label input placeholder to retrieve labels from
      tf input feed dict.
    dataset: dataset to perform grid-search on.

  Returns:
    best_f_score: value of best found F-score.
    best_fdr: corresponding value of False Detection Rate.
    best_tdr: corresponding value of True Detection Rate.
  '''
    # initialize dataset statistics
    true_preds = []
    false_preds = []
    total = 0

    steps_per_epoch = (dataset.num_samples + batch_size - 1) // batch_size
    for _ in range(steps_per_epoch):
        feed_dict = utils.fill_feed_dict(dataset, patches_pl, labels_pl,
                                         batch_size)

        # evaluate batch
        batch_preds = sess.run(preds, feed_dict=feed_dict)
        batch_labels = feed_dict[labels_pl]
        batch_total = np.sum(batch_labels)

        # update dataset statistics
        total += batch_total
        if batch_total > 0:
            true_preds.extend(batch_preds[batch_labels == 1].flatten())
        if batch_total < batch_labels.shape[0]:
            false_preds.extend(batch_preds[batch_labels == 0].flatten())

    # sort for efficient computation of tdr/fdr over thresholds
    true_preds.sort()
    true_preds.reverse()
    false_preds.sort()
    false_preds.reverse()

    # compute tdr/fdr score over thresholds
    best_f_score = 0
    best_fdr = None
    best_tdr = None

    true_pointer = 0
    false_pointer = 0

    eps = 1e-5
    thrs = np.arange(1.01, -0.01, -0.01)
    for thr in thrs:
        # compute true positives
        while true_pointer < len(
                true_preds) and true_preds[true_pointer] >= thr:
            true_pointer += 1

        # compute false positives
        while false_pointer < len(
                false_preds) and false_preds[false_pointer] >= thr:
            false_pointer += 1

        # compute tdr and fdr
        tdr = true_pointer / (total + eps)
        fdr = false_pointer / (true_pointer + false_pointer + eps)

        # compute and update f score
        f_score = 2 * (tdr * (1 - fdr)) / (tdr + 1 - fdr)
        if f_score > best_f_score:
            best_tdr = tdr
            best_fdr = fdr
            best_f_score = f_score

    return best_f_score, best_fdr, best_tdr
def run_testing():
    """Test BinaryConnect."""
    # Get the sets of images and labels for training, validation, and
    # test on CIFAR10.
    data_sets = cifar10.read_data_sets(dst_dir='./dataset',
                                       validation_size=5000)

    # Tell TensorFlow that the model will be built into the default Graph.
    with tf.Graph().as_default():
        # Generate placeholders for the images and labels.
        images_placeholder, labels_placeholder, train_placeholder = placeholder_inputs(
            FLAGS.batch_size)

        # Build a Graph that computes predictions from the inference model.
        logits = bc.inference_bin(images_placeholder, train_placeholder,
                                  stochastic=FLAGS.stochastic,
                                  use_bnorm=True) \
            if FLAGS.binary \
            else bc.inference_ref(images_placeholder, train_placeholder,
                                  use_bnorm=True)

        # Add the Op to compare the logits to the labels during evaluation.
        eval_metric = bc.evaluation(logits, labels_placeholder)

        # Add the variable initializer Op.
        ivars = tf.global_variables() + tf.local_variables()
        init = tf.variables_initializer(ivars)

        # Create a session for running Ops on the Graph.
        sess = tf.Session()

        # Load trained model.
        saver = tf.train.Saver()
        saver.restore(sess, FLAGS.model_path)
        print("Model loaded.")

        # And then after everything is built:

        # Run the Op to initialize the variables.
        sess.run(init)

        # Start the training loop
        duration = 0
        tp_value_total = 0
        for step in xrange(FLAGS.max_steps):
            start_time = time.time()

            # Fill a feed dictionary with the actual set of images and labels
            # for this particular testing step.
            feed_dict = fill_feed_dict(data_sets.test, images_placeholder,
                                       labels_placeholder, train_placeholder,
                                       False)

            # Run one step of the model.
            acc_val = sess.run(eval_metric, feed_dict=feed_dict)
            duration += time.time() - start_time
            tp_value_total += acc_val

            # Print an overview
            if step % 100 == 0:
                # Print status to stdout.
                images_freq = 100 * FLAGS.batch_size / duration
                print('Step %d: correct = %.2f%% (%.3f images/sec)' %
                      (step, tp_value_total / step, images_freq))
                duration = time.time() - start_time
                duration = 0