Exemplo n.º 1
0
    def _get_eval_ops(self, features, targets, metrics=None):
        """See base class."""
        logits = self._logits(features)
        result = {
            "loss":
            metrics_lib.streaming_mean(
                self._loss(logits,
                           targets,
                           weight_tensor=self._get_weight_tensor(features)))
        }

        # Adding default metrics
        if metrics is None:
            metrics = {"accuracy": metrics_lib.streaming_accuracy}

        if self._n_classes == 2:
            predictions = math_ops.sigmoid(logits)
            result["eval_auc"] = metrics_lib.streaming_auc(
                predictions, targets)

        if metrics:
            predictions = self._logits_to_predictions(logits, proba=False)
            result.update(
                self._run_metrics(predictions, targets, metrics,
                                  self._get_weight_tensor(features)))

        return result
Exemplo n.º 2
0
def _streaming_auc(predictions, labels, weights=None, class_id=None):
  if class_id is not None:
    predictions = predictions[:, class_id]
    labels = labels[:, class_id]
  return metrics_lib.streaming_auc(
      predictions, math_ops.cast(labels, dtypes.bool),
      weights=_float_weights_or_none(weights))
Exemplo n.º 3
0
    def __init__(self,
                 feature_dims,
                 hidden_units=None,
                 layers=1,
                 model_dir=None):
        self.model_dir = model_dir

        x = tf.placeholder(tf.float32, [None, None, feature_dims])
        y = tf.placeholder(tf.float32, [None, 1])
        length = tf.placeholder(tf.int32, [None])
        self.x, self.y, self.length = x, y, length

        cell = tf.nn.rnn_cell.LSTMCell(hidden_units, state_is_tuple=True)
        if layers > 1:
            cell = tf.nn.rnn_cell.MultiRNNCell([cell] * layers,
                                               state_is_tuple=True)

        output, state = tf.nn.dynamic_rnn(
            cell=cell,
            inputs=x,
            dtype=tf.float32,
            sequence_length=length,
        )

        batch_size = tf.shape(output)[0]
        max_length = tf.shape(output)[1]
        out_size = int(output.get_shape()[2])
        index = tf.range(0, batch_size) * max_length + (length - 1)
        flat = tf.reshape(output, [-1, out_size])
        relevant = tf.gather(flat, index)

        logit = tf.contrib.layers.fully_connected(inputs=relevant,
                                                  num_outputs=1,
                                                  activation_fn=None,
                                                  biases_initializer=None)

        predictions = tf.sigmoid(logit)

        loss = tf.nn.sigmoid_cross_entropy_with_logits(logit, y)

        auc, update_auc = metrics.streaming_auc(predictions,
                                                y,
                                                num_thresholds=10)
        stream_loss, update_stream_loss = metrics.streaming_mean(loss)

        self.update_metrics = [update_stream_loss, update_auc]
        self.summaries = [auc, stream_loss]
        self.summary_labels = ['auc', 'loss']
        self.predictions = predictions

        self.learning_rate = tf.placeholder(tf.float32, [])
        self._global_step = framework.create_global_step()
        self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(
            loss, global_step=self._global_step)
        self.saver = tf.train.Saver()

        sess = tf.Session()
        sess.run(tf.initialize_local_variables())
        sess.run(tf.initialize_all_variables())
        self.sess = sess
Exemplo n.º 4
0
    def _get_eval_ops(self, features, targets, metrics=None):
        """See base class."""
        logits = self._logits(features)
        result = {
            "loss":
            metrics_lib.streaming_mean(
                self._loss(logits,
                           targets,
                           weight_tensor=self._get_weight_tensor(features)))
        }

        # Adding default metrics
        if metrics is None:
            metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}

        if self._n_classes == 2:
            predictions = math_ops.sigmoid(logits)
            result["auc"] = metrics_lib.streaming_auc(predictions, targets)

        if metrics:
            class_metrics = {}
            proba_metrics = {}
            for name, metric_op in six.iteritems(metrics):
                if isinstance(name, tuple):
                    if len(name) != 2:
                        raise ValueError(
                            "Ignoring metric {}. It returned a tuple with "
                            "len {}, expected 2.".format(name, len(name)))
                    else:
                        if name[1] not in ["classes", "probabilities"]:
                            raise ValueError(
                                "Ignoring metric {}. The 2nd element of its "
                                "name should be either 'classes' or "
                                "'probabilities'.".format(name))
                        elif name[1] == "classes":
                            class_metrics[name[0]] = metric_op
                        else:
                            proba_metrics[name[0]] = metric_op
                elif isinstance(name, str):
                    class_metrics[name] = metric_op
                else:
                    raise ValueError(
                        "Ignoring metric {}. Its name is not in the correct "
                        "form.".format(name))

            if class_metrics:
                predictions = self._logits_to_predictions(logits, proba=False)
                result.update(
                    self._run_metrics(predictions, targets, class_metrics,
                                      self._get_weight_tensor(features)))
            if proba_metrics:
                predictions = self._logits_to_predictions(logits, proba=True)
                result.update(
                    self._run_metrics(predictions, targets, proba_metrics,
                                      self._get_weight_tensor(features)))

        return result
Exemplo n.º 5
0
def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
    """Returns a dictionary of evaluation metric ops for logistic regression.

  Args:
    labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
    predictions: The predictions `Tensor`.
    thresholds: List of floating point thresholds to use for accuracy,
      precision, and recall metrics.

  Returns:
    A dict of metric results keyed by name.
  """
    # If labels is a dict with a single key, unpack into a single tensor.
    labels_tensor = labels
    if isinstance(labels, dict) and len(labels) == 1:
        labels_tensor = labels.values()[0]

    metrics = {}
    metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
        predictions)
    metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
        labels_tensor)
    # Also include the streaming mean of the label as an accuracy baseline, as
    # a reminder to users.
    metrics[metric_key.MetricKey.
            ACCURACY_BASELINE] = metrics_lib.streaming_mean(labels_tensor)

    metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
        labels=labels_tensor, predictions=predictions)

    for threshold in thresholds:
        predictions_at_threshold = math_ops.cast(
            math_ops.greater_equal(predictions, threshold),
            dtypes.float32,
            name='predictions_at_threshold_%f' % threshold)
        metrics[metric_key.MetricKey.ACCURACY_MEAN %
                threshold] = (metrics_lib.streaming_accuracy(
                    labels=labels_tensor,
                    predictions=predictions_at_threshold))
        # Precision for positive examples.
        metrics[metric_key.MetricKey.PRECISION_MEAN %
                threshold] = (metrics_lib.streaming_precision(
                    labels=labels_tensor,
                    predictions=predictions_at_threshold))
        # Recall for positive examples.
        metrics[metric_key.MetricKey.RECALL_MEAN %
                threshold] = (metrics_lib.streaming_recall(
                    labels=labels_tensor,
                    predictions=predictions_at_threshold))

    return metrics
Exemplo n.º 6
0
  def _get_eval_ops(self, features, targets, metrics=None):
    """See base class."""
    logits = self._logits(features)
    result = {"loss": metrics_lib.streaming_mean(self._loss(
        logits, targets,
        weight_tensor=self._get_weight_tensor(features)))}

    # Adding default metrics
    if metrics is None:
      metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}

    if self._n_classes == 2:
      predictions = math_ops.sigmoid(logits)
      result["auc"] = metrics_lib.streaming_auc(predictions, targets)

    if metrics:
      class_metrics = {}
      proba_metrics = {}
      for name, metric_op in six.iteritems(metrics):
        if isinstance(name, tuple):
          if len(name) != 2:
            raise ValueError("Ignoring metric {}. It returned a tuple with "
                             "len {}, expected 2.".format(name, len(name)))
          else:
            if name[1] not in ["classes", "probabilities"]:
              raise ValueError("Ignoring metric {}. The 2nd element of its "
                               "name should be either 'classes' or "
                               "'probabilities'.".format(name))
            elif name[1] == "classes":
              class_metrics[name[0]] = metric_op
            else:
              proba_metrics[name[0]] = metric_op
        elif isinstance(name, str):
          class_metrics[name] = metric_op
        else:
          raise ValueError("Ignoring metric {}. Its name is not in the correct "
                           "form.".format(name))

      if class_metrics:
        predictions = self._logits_to_predictions(logits, proba=False)
        result.update(self._run_metrics(predictions, targets, class_metrics,
                                        self._get_weight_tensor(features)))
      if proba_metrics:
        predictions = self._logits_to_predictions(logits, proba=True)
        result.update(self._run_metrics(predictions, targets, proba_metrics,
                                        self._get_weight_tensor(features)))

    return result
Exemplo n.º 7
0
def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
  """Returns a dictionary of evaluation metric ops for logistic regression.

  Args:
    labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
    predictions: The predictions `Tensor`.
    thresholds: List of floating point thresholds to use for accuracy,
      precision, and recall metrics.

  Returns:
    A dict of metric results keyed by name.
  """
  # If labels is a dict with a single key, unpack into a single tensor.
  labels_tensor = labels
  if isinstance(labels, dict) and len(labels) == 1:
    labels_tensor = labels.values()[0]

  metrics = {}
  metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
      predictions)
  metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
      labels_tensor)
  # Also include the streaming mean of the label as an accuracy baseline, as
  # a reminder to users.
  metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean(
      labels_tensor)

  metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
      labels=labels_tensor, predictions=predictions)

  for threshold in thresholds:
    predictions_at_threshold = math_ops.cast(
        math_ops.greater_equal(predictions, threshold),
        dtypes.float32,
        name='predictions_at_threshold_%f' % threshold)
    metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = (
        metrics_lib.streaming_accuracy(labels=labels_tensor,
                                       predictions=predictions_at_threshold))
    # Precision for positive examples.
    metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = (
        metrics_lib.streaming_precision(labels=labels_tensor,
                                        predictions=predictions_at_threshold))
    # Recall for positive examples.
    metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = (
        metrics_lib.streaming_recall(labels=labels_tensor,
                                     predictions=predictions_at_threshold))

  return metrics
            def build_test_metrics(self):

                raw_labels = multihot_labels(self.mmsis)
                mask = tf.to_float(tf.equal(tf.reduce_sum(raw_labels, 1), 1))
                labels = tf.to_int32(tf.argmax(raw_labels, 1))

                predictions = tf.to_int32(tf.argmax(self.prediction, 1))

                metrics_map = {
                    '%s/Test-accuracy' % self.name:
                    metrics.streaming_accuracy(predictions,
                                               labels,
                                               weights=mask)
                }

                if self.metrics == 'all':
                    for i, cls in enumerate(self.classes):
                        cls_name = cls.replace(' ', '-')
                        trues = tf.to_int32(tf.equal(labels, i))
                        preds = tf.to_int32(tf.equal(predictions, i))
                        recall = metrics.streaming_recall(preds,
                                                          trues,
                                                          weights=mask)
                        precision = metrics.streaming_precision(preds,
                                                                trues,
                                                                weights=mask)
                        metrics_map["%s/Class-%s-Precision" %
                                    (self.name, cls_name)] = recall
                        metrics_map["%s/Class-%s-Recall" %
                                    (self.name, cls_name)] = precision
                        metrics_map["%s/Class-%s-F1-Score" %
                                    (self.name, cls_name)] = f1(
                                        recall, precision)
                        metrics_map["%s/Class-%s-ROC-AUC" %
                                    (self.name,
                                     cls_name)] = metrics.streaming_auc(
                                         self.prediction[:, i],
                                         trues,
                                         weights=mask)

                return metrics.aggregate_metric_map(metrics_map)
  def _get_eval_ops(self, features, targets, metrics=None):
    """See base class."""
    logits = self._logits(features)
    result = {"loss": metrics_lib.streaming_mean(self._loss(
        logits, targets,
        weight_tensor=self._get_weight_tensor(features)))}

    # Adding default metrics
    if metrics is None:
      metrics = {"accuracy": metrics_lib.streaming_accuracy}

    if self._n_classes == 2:
      predictions = math_ops.sigmoid(logits)
      result["eval_auc"] = metrics_lib.streaming_auc(predictions, targets)

    if metrics:
      predictions = self._logits_to_predictions(logits, proba=False)
      result.update(self._run_metrics(predictions, targets, metrics,
                                      self._get_weight_tensor(features)))

    return result
Exemplo n.º 10
0
def _streaming_auc(predictions, labels, weights=None):
  return metrics_lib.streaming_auc(predictions, labels,
                                   weights=_float_weights_or_none(weights))
Exemplo n.º 11
0
def _streaming_auc(predictions, target, weights=None):
  return metrics_lib.streaming_auc(predictions, target,
                                   weights=_float_weights_or_none(weights))
Exemplo n.º 12
0
    def __init__(self,
                 limits,
                 embed_size=8,
                 fc1_size=200,
                 fc2_size=100,
                 linear_size=800,
                 l2_reg_lambda=0.0,
                 NUM_CLASSES=2,
                 inds=None,
                 vals=None,
                 labels=None,
                 linear=False):

        field_size = len(limits) - 1
        feature_size = int(limits[-1])

        if inds == None:
            self.inds_ = tf.placeholder(tf.int32, shape=[None, field_size])
        else:
            self.inds_ = inds

        if vals == None:
            self.vals_ = tf.placeholder(tf.float32, shape=[None, field_size])
        else:
            self.vals_ = vals

        if labels == None:
            self.labels_ = tf.placeholder(tf.int64, shape=[None])
        else:
            self.labels_ = labels

        init_weights = tf.truncated_normal_initializer(0.0, stddev=0.1)
        init_biases = tf.constant_initializer(0.1)
        self.l2_loss = tf.constant(0.00)
        epsilon = 1e-3

        # Field_embed
        with tf.name_scope("field_embed"):
            weights = tf.Variable(init_weights(
                [feature_size, field_size, embed_size]),
                                  name="weights")
            biases = tf.Variable(init_biases(
                [field_size, field_size, embed_size]),
                                 name='biases')

            self.field_embed = self.act_summary(
                field_embed_op(self.inds_, self.vals_, weights, biases,
                               limits))

            self.l2_loss = tf.nn.l2_loss(weights)
            self.l2_loss = tf.nn.l2_loss(biases)
            variable_summaries(weights, 'weights')
            variable_summaries(biases, 'biases')
            variable_summaries(self.field_embed, 'output')

        # Product_layer
        with tf.name_scope("product_layer"):
            self.product_layer = self.act_summary(
                product_layer_op(self.field_embed))
            product_layer_size = int(self.product_layer.get_shape()[-1])
            variable_summaries(self.product_layer, 'output')

        if linear:
            # Linear feature
            with tf.name_scope("linear"):
                batch_size = int(self.inds_.get_shape()[0])

                sparse_cols = tf.reshape(
                    tf.add(self.inds_, [limits[:-1]] * batch_size),
                    [-1, field_size, 1])
                sparse_rows = [[[i]] * field_size for i in range(batch_size)]
                sparse_inds = tf.reshape(
                    tf.cast(tf.concat([sparse_rows, sparse_cols], 2),
                            tf.int64), [-1, 2])
                sparse_vals = tf.reshape(self.vals_, [-1])
                sparse_input = tf.SparseTensor(sparse_inds, sparse_vals,
                                               [batch_size, feature_size])

                weights = tf.Variable(init_weights([feature_size,
                                                    linear_size]),
                                      name="weights")
                z_BN = tf.sparse_tensor_dense_matmul(sparse_input, weights)
                batch_mean, batch_var = tf.nn.moments(z_BN, [0])
                scale = tf.Variable(tf.ones([linear_size]))
                beta = tf.Variable(tf.zeros([linear_size]))
                prelu = keras.PReLU()
                self.linear = prelu(
                    tf.nn.batch_normalization(z_BN, batch_mean, batch_var,
                                              beta, scale, epsilon))
                self.l2_loss = tf.nn.l2_loss(weights)
                variable_summaries(weights, 'weights')
                variable_summaries(self.linear, 'output')
            # Full Connect 1 with BN
            with tf.name_scope("fc1"):
                weights = tf.Variable(init_weights(
                    [product_layer_size + linear_size, fc1_size]),
                                      name="weights")
                z_BN = tf.matmul(
                    tf.concat([self.product_layer, self.linear], 1), weights)
                batch_mean, batch_var = tf.nn.moments(z_BN, [0])
                scale = tf.Variable(tf.ones([fc1_size]))
                beta = tf.Variable(tf.zeros([fc1_size]))
                self.fc1 = self.act_summary(
                    tf.nn.batch_normalization(z_BN, batch_mean, batch_var,
                                              beta, scale, epsilon))

                self.l2_loss = tf.nn.l2_loss(weights)
                variable_summaries(weights, 'weights')
                variable_summaries(self.fc1, 'output')
        else:
            # Full Connect 1 with BN
            with tf.name_scope("fc1"):
                weights = tf.Variable(init_weights(
                    [product_layer_size, fc1_size]),
                                      name="weights")
                z_BN = tf.matmul(self.product_layer, weights)
                batch_mean, batch_var = tf.nn.moments(z_BN, [0])
                scale = tf.Variable(tf.ones([fc1_size]))
                beta = tf.Variable(tf.zeros([fc1_size]))
                self.fc1 = self.act_summary(
                    tf.nn.batch_normalization(z_BN, batch_mean, batch_var,
                                              beta, scale, epsilon))

                self.l2_loss = tf.nn.l2_loss(weights)
                variable_summaries(weights, 'weights')
                variable_summaries(self.fc1, 'output')

        # Linear 2 with BN
        with tf.name_scope("fc2"):
            weights = tf.Variable(init_weights([fc1_size, fc2_size]),
                                  name="weights")
            z_BN = tf.matmul(self.fc1, weights)
            batch_mean, batch_var = tf.nn.moments(z_BN, [0])
            scale = tf.Variable(tf.ones([fc2_size]))
            beta = tf.Variable(tf.zeros([fc2_size]))
            self.fc2 = self.act_summary(
                tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta,
                                          scale, epsilon))
            self.l2_loss = tf.nn.l2_loss(weights)
            variable_summaries(weights, 'weights')
            variable_summaries(self.fc2, 'output')

        # Softmax
        with tf.name_scope("Softmax"):
            weights = tf.Variable(init_weights([fc2_size, NUM_CLASSES]),
                                  name="weights")
            biases = tf.Variable(init_biases([NUM_CLASSES]), name='biases')
            self.logits = tf.nn.softmax(tf.matmul(self.fc2, weights) + biases,
                                        name="scores")
            self.predictions = tf.argmax(self.logits, 1, name="predictions")

            self.l2_loss = tf.nn.l2_loss(weights)
            self.l2_loss = tf.nn.l2_loss(biases)
            variable_summaries(weights, 'weights')
            variable_summaries(biases, 'biases')
            variable_summaries(self.logits, 'output')

        # Loss
        with tf.name_scope("loss"):
            cross_entropy = tf.reduce_mean(
                tf.nn.sparse_softmax_cross_entropy_with_logits(
                    logits=self.logits, labels=self.labels_))
            reg_loss = l2_reg_lambda * self.l2_loss
            self.loss = cross_entropy + reg_loss
            tf.summary.scalar('cross_entropy', cross_entropy)
            tf.summary.scalar('reg_loss', reg_loss)
            tf.summary.scalar('loss', self.loss)

        # Accuracy
        with tf.name_scope("accuracy"):
            correct_predictions = tf.equal(self.predictions, self.labels_)
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions,
                                                   "float"),
                                           name="accuracy")
            tf.summary.scalar('accuracy', self.accuracy)

        # Auc
        with tf.name_scope("auc"):
            _, self.auc = streaming_auc(self.logits[:, 1], self.labels_)
            tf.summary.scalar('auc', self.auc)
Exemplo n.º 13
0
Arquivo: model.py Projeto: ml-lab/PNC
    def _model_fn(features, labels, mode):
        """

        :param features:
        :param labels:
        :param mode:
        :return:
        """

        # Pop the name of the signal.
        if 'FN' in features:
            names = features.pop('FN')

        if 'FT' in features:
            labels = features.pop('FT')

        # Define the type of the inputs (they are all numeric).
        columns = [
            layers.real_valued_column(key) for key, value in features.items()
        ]
        #
        inputs = layers.input_from_feature_columns(features, columns)

        # Declare the hidden_layers variable.
        hidden_layers = None

        # Iterate all over the hidden units.
        for unit in hidden_units:
            # Create a new hidden layer.
            hidden_layers = tf.layers.dense(
                inputs=inputs if hidden_layers is None else hidden_layers,
                activation=tf.nn.relu,
                units=unit,
            )

        # Create a dropout layer.
        dropout_layer = layers.dropout(inputs=hidden_layers,
                                       keep_prob=1.0 - dropout)

        # Create the logits layer.
        logits = tf.layers.dense(inputs=dropout_layer,
                                 activation=None,
                                 units=2)

        if mode in (ModeKeys.PREDICT, ModeKeys.EVAL):
            # Calculate the probabilities.
            probabilities = tf.nn.softmax(logits)
            # And their indexes.
            predictions = tf.argmax(logits, 1)

        if mode in (ModeKeys.EVAL, ModeKeys.TRAIN):
            # Convert the labels in the one_hot format.
            onehot_labels = tf.one_hot(indices=labels, depth=2)
            # Define the class weights.
            class_weights = tf.constant(weights)
            # Deduce weights for batch samples based on their true label.
            reduced_weights = tf.reduce_sum(class_weights * onehot_labels,
                                            axis=1)
            # Compute your (unweighted) softmax cross entropy loss.
            unweighted_losses = tf.nn.softmax_cross_entropy_with_logits(
                labels=onehot_labels, logits=logits)
            # Apply the weights, relying on broadcasting of the multiplication.
            weighted_losses = unweighted_losses * reduced_weights
            # Reduce the result to get your final loss.
            loss = tf.reduce_mean(weighted_losses)

        if mode == ModeKeys.PREDICT:

            # Convert predicted_indices back into strings
            predictions = {
                'classes': predictions,
                'scores': probabilities,
            }

            # export_outputs = {
            #     'prediction': tf.estimator.export.PredictOutput(predictions)
            # }

            # return tf.estimator.EstimatorSpec(
            #     mode=mode,
            #     predictions=predictions,
            #     # export_outputs=export_outputs,
            # )

            return tf.estimator.EstimatorSpec(
                mode=mode,
                predictions=predictions,
            )

        if mode == ModeKeys.TRAIN:
            # Define the training rule.
            train_op = layers.optimize_loss(
                loss=loss,
                global_step=framework.get_global_step(),
                learning_rate=learning_rate,
                optimizer='SGD')

            return tf.estimator.EstimatorSpec(mode=mode,
                                              loss=loss,
                                              train_op=train_op)

        if mode == ModeKeys.EVAL:

            # Define the metrics to show up in the evaluation process.
            eval_metric_ops = {
                'accuracy':
                metrics.streaming_accuracy(predictions=predictions,
                                           labels=labels),
                'auroc':
                metrics.streaming_auc(predictions=predictions, labels=labels),
                'recall':
                metrics.streaming_recall(predictions=predictions,
                                         labels=labels),
                'precision':
                metrics.streaming_precision(predictions=predictions,
                                            labels=labels),
                'TP':
                metrics.streaming_true_positives(predictions=predictions,
                                                 labels=labels),
                'FN':
                metrics.streaming_false_negatives(predictions=predictions,
                                                  labels=labels),
                'FP':
                metrics.streaming_false_positives(predictions=predictions,
                                                  labels=labels),
                'TN':
                metrics.streaming_true_negatives(predictions=predictions,
                                                 labels=labels),
                #'gaccuracy' : metrics.streaming_accuracy(predictions=GP, labels=GL)
            }

            return tf.estimator.EstimatorSpec(mode=mode,
                                              predictions=predictions,
                                              loss=loss,
                                              eval_metric_ops=eval_metric_ops)