def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = { "loss": metrics_lib.streaming_mean( self._loss(logits, targets, weight_tensor=self._get_weight_tensor(features))) } # Adding default metrics if metrics is None: metrics = {"accuracy": metrics_lib.streaming_accuracy} if self._n_classes == 2: predictions = math_ops.sigmoid(logits) result["eval_auc"] = metrics_lib.streaming_auc( predictions, targets) if metrics: predictions = self._logits_to_predictions(logits, proba=False) result.update( self._run_metrics(predictions, targets, metrics, self._get_weight_tensor(features))) return result
def _streaming_auc(predictions, labels, weights=None, class_id=None): if class_id is not None: predictions = predictions[:, class_id] labels = labels[:, class_id] return metrics_lib.streaming_auc( predictions, math_ops.cast(labels, dtypes.bool), weights=_float_weights_or_none(weights))
def __init__(self, feature_dims, hidden_units=None, layers=1, model_dir=None): self.model_dir = model_dir x = tf.placeholder(tf.float32, [None, None, feature_dims]) y = tf.placeholder(tf.float32, [None, 1]) length = tf.placeholder(tf.int32, [None]) self.x, self.y, self.length = x, y, length cell = tf.nn.rnn_cell.LSTMCell(hidden_units, state_is_tuple=True) if layers > 1: cell = tf.nn.rnn_cell.MultiRNNCell([cell] * layers, state_is_tuple=True) output, state = tf.nn.dynamic_rnn( cell=cell, inputs=x, dtype=tf.float32, sequence_length=length, ) batch_size = tf.shape(output)[0] max_length = tf.shape(output)[1] out_size = int(output.get_shape()[2]) index = tf.range(0, batch_size) * max_length + (length - 1) flat = tf.reshape(output, [-1, out_size]) relevant = tf.gather(flat, index) logit = tf.contrib.layers.fully_connected(inputs=relevant, num_outputs=1, activation_fn=None, biases_initializer=None) predictions = tf.sigmoid(logit) loss = tf.nn.sigmoid_cross_entropy_with_logits(logit, y) auc, update_auc = metrics.streaming_auc(predictions, y, num_thresholds=10) stream_loss, update_stream_loss = metrics.streaming_mean(loss) self.update_metrics = [update_stream_loss, update_auc] self.summaries = [auc, stream_loss] self.summary_labels = ['auc', 'loss'] self.predictions = predictions self.learning_rate = tf.placeholder(tf.float32, []) self._global_step = framework.create_global_step() self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize( loss, global_step=self._global_step) self.saver = tf.train.Saver() sess = tf.Session() sess.run(tf.initialize_local_variables()) sess.run(tf.initialize_all_variables()) self.sess = sess
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = { "loss": metrics_lib.streaming_mean( self._loss(logits, targets, weight_tensor=self._get_weight_tensor(features))) } # Adding default metrics if metrics is None: metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy} if self._n_classes == 2: predictions = math_ops.sigmoid(logits) result["auc"] = metrics_lib.streaming_auc(predictions, targets) if metrics: class_metrics = {} proba_metrics = {} for name, metric_op in six.iteritems(metrics): if isinstance(name, tuple): if len(name) != 2: raise ValueError( "Ignoring metric {}. It returned a tuple with " "len {}, expected 2.".format(name, len(name))) else: if name[1] not in ["classes", "probabilities"]: raise ValueError( "Ignoring metric {}. The 2nd element of its " "name should be either 'classes' or " "'probabilities'.".format(name)) elif name[1] == "classes": class_metrics[name[0]] = metric_op else: proba_metrics[name[0]] = metric_op elif isinstance(name, str): class_metrics[name] = metric_op else: raise ValueError( "Ignoring metric {}. Its name is not in the correct " "form.".format(name)) if class_metrics: predictions = self._logits_to_predictions(logits, proba=False) result.update( self._run_metrics(predictions, targets, class_metrics, self._get_weight_tensor(features))) if proba_metrics: predictions = self._logits_to_predictions(logits, proba=True) result.update( self._run_metrics(predictions, targets, proba_metrics, self._get_weight_tensor(features))) return result
def _make_logistic_eval_metric_ops(labels, predictions, thresholds): """Returns a dictionary of evaluation metric ops for logistic regression. Args: labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name. predictions: The predictions `Tensor`. thresholds: List of floating point thresholds to use for accuracy, precision, and recall metrics. Returns: A dict of metric results keyed by name. """ # If labels is a dict with a single key, unpack into a single tensor. labels_tensor = labels if isinstance(labels, dict) and len(labels) == 1: labels_tensor = labels.values()[0] metrics = {} metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean( predictions) metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean( labels_tensor) # Also include the streaming mean of the label as an accuracy baseline, as # a reminder to users. metrics[metric_key.MetricKey. ACCURACY_BASELINE] = metrics_lib.streaming_mean(labels_tensor) metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc( labels=labels_tensor, predictions=predictions) for threshold in thresholds: predictions_at_threshold = math_ops.cast( math_ops.greater_equal(predictions, threshold), dtypes.float32, name='predictions_at_threshold_%f' % threshold) metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = (metrics_lib.streaming_accuracy( labels=labels_tensor, predictions=predictions_at_threshold)) # Precision for positive examples. metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = (metrics_lib.streaming_precision( labels=labels_tensor, predictions=predictions_at_threshold)) # Recall for positive examples. metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = (metrics_lib.streaming_recall( labels=labels_tensor, predictions=predictions_at_threshold)) return metrics
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = {"loss": metrics_lib.streaming_mean(self._loss( logits, targets, weight_tensor=self._get_weight_tensor(features)))} # Adding default metrics if metrics is None: metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy} if self._n_classes == 2: predictions = math_ops.sigmoid(logits) result["auc"] = metrics_lib.streaming_auc(predictions, targets) if metrics: class_metrics = {} proba_metrics = {} for name, metric_op in six.iteritems(metrics): if isinstance(name, tuple): if len(name) != 2: raise ValueError("Ignoring metric {}. It returned a tuple with " "len {}, expected 2.".format(name, len(name))) else: if name[1] not in ["classes", "probabilities"]: raise ValueError("Ignoring metric {}. The 2nd element of its " "name should be either 'classes' or " "'probabilities'.".format(name)) elif name[1] == "classes": class_metrics[name[0]] = metric_op else: proba_metrics[name[0]] = metric_op elif isinstance(name, str): class_metrics[name] = metric_op else: raise ValueError("Ignoring metric {}. Its name is not in the correct " "form.".format(name)) if class_metrics: predictions = self._logits_to_predictions(logits, proba=False) result.update(self._run_metrics(predictions, targets, class_metrics, self._get_weight_tensor(features))) if proba_metrics: predictions = self._logits_to_predictions(logits, proba=True) result.update(self._run_metrics(predictions, targets, proba_metrics, self._get_weight_tensor(features))) return result
def _make_logistic_eval_metric_ops(labels, predictions, thresholds): """Returns a dictionary of evaluation metric ops for logistic regression. Args: labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name. predictions: The predictions `Tensor`. thresholds: List of floating point thresholds to use for accuracy, precision, and recall metrics. Returns: A dict of metric results keyed by name. """ # If labels is a dict with a single key, unpack into a single tensor. labels_tensor = labels if isinstance(labels, dict) and len(labels) == 1: labels_tensor = labels.values()[0] metrics = {} metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean( predictions) metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean( labels_tensor) # Also include the streaming mean of the label as an accuracy baseline, as # a reminder to users. metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean( labels_tensor) metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc( labels=labels_tensor, predictions=predictions) for threshold in thresholds: predictions_at_threshold = math_ops.cast( math_ops.greater_equal(predictions, threshold), dtypes.float32, name='predictions_at_threshold_%f' % threshold) metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = ( metrics_lib.streaming_accuracy(labels=labels_tensor, predictions=predictions_at_threshold)) # Precision for positive examples. metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = ( metrics_lib.streaming_precision(labels=labels_tensor, predictions=predictions_at_threshold)) # Recall for positive examples. metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = ( metrics_lib.streaming_recall(labels=labels_tensor, predictions=predictions_at_threshold)) return metrics
def build_test_metrics(self): raw_labels = multihot_labels(self.mmsis) mask = tf.to_float(tf.equal(tf.reduce_sum(raw_labels, 1), 1)) labels = tf.to_int32(tf.argmax(raw_labels, 1)) predictions = tf.to_int32(tf.argmax(self.prediction, 1)) metrics_map = { '%s/Test-accuracy' % self.name: metrics.streaming_accuracy(predictions, labels, weights=mask) } if self.metrics == 'all': for i, cls in enumerate(self.classes): cls_name = cls.replace(' ', '-') trues = tf.to_int32(tf.equal(labels, i)) preds = tf.to_int32(tf.equal(predictions, i)) recall = metrics.streaming_recall(preds, trues, weights=mask) precision = metrics.streaming_precision(preds, trues, weights=mask) metrics_map["%s/Class-%s-Precision" % (self.name, cls_name)] = recall metrics_map["%s/Class-%s-Recall" % (self.name, cls_name)] = precision metrics_map["%s/Class-%s-F1-Score" % (self.name, cls_name)] = f1( recall, precision) metrics_map["%s/Class-%s-ROC-AUC" % (self.name, cls_name)] = metrics.streaming_auc( self.prediction[:, i], trues, weights=mask) return metrics.aggregate_metric_map(metrics_map)
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = {"loss": metrics_lib.streaming_mean(self._loss( logits, targets, weight_tensor=self._get_weight_tensor(features)))} # Adding default metrics if metrics is None: metrics = {"accuracy": metrics_lib.streaming_accuracy} if self._n_classes == 2: predictions = math_ops.sigmoid(logits) result["eval_auc"] = metrics_lib.streaming_auc(predictions, targets) if metrics: predictions = self._logits_to_predictions(logits, proba=False) result.update(self._run_metrics(predictions, targets, metrics, self._get_weight_tensor(features))) return result
def _streaming_auc(predictions, labels, weights=None): return metrics_lib.streaming_auc(predictions, labels, weights=_float_weights_or_none(weights))
def _streaming_auc(predictions, target, weights=None): return metrics_lib.streaming_auc(predictions, target, weights=_float_weights_or_none(weights))
def __init__(self, limits, embed_size=8, fc1_size=200, fc2_size=100, linear_size=800, l2_reg_lambda=0.0, NUM_CLASSES=2, inds=None, vals=None, labels=None, linear=False): field_size = len(limits) - 1 feature_size = int(limits[-1]) if inds == None: self.inds_ = tf.placeholder(tf.int32, shape=[None, field_size]) else: self.inds_ = inds if vals == None: self.vals_ = tf.placeholder(tf.float32, shape=[None, field_size]) else: self.vals_ = vals if labels == None: self.labels_ = tf.placeholder(tf.int64, shape=[None]) else: self.labels_ = labels init_weights = tf.truncated_normal_initializer(0.0, stddev=0.1) init_biases = tf.constant_initializer(0.1) self.l2_loss = tf.constant(0.00) epsilon = 1e-3 # Field_embed with tf.name_scope("field_embed"): weights = tf.Variable(init_weights( [feature_size, field_size, embed_size]), name="weights") biases = tf.Variable(init_biases( [field_size, field_size, embed_size]), name='biases') self.field_embed = self.act_summary( field_embed_op(self.inds_, self.vals_, weights, biases, limits)) self.l2_loss = tf.nn.l2_loss(weights) self.l2_loss = tf.nn.l2_loss(biases) variable_summaries(weights, 'weights') variable_summaries(biases, 'biases') variable_summaries(self.field_embed, 'output') # Product_layer with tf.name_scope("product_layer"): self.product_layer = self.act_summary( product_layer_op(self.field_embed)) product_layer_size = int(self.product_layer.get_shape()[-1]) variable_summaries(self.product_layer, 'output') if linear: # Linear feature with tf.name_scope("linear"): batch_size = int(self.inds_.get_shape()[0]) sparse_cols = tf.reshape( tf.add(self.inds_, [limits[:-1]] * batch_size), [-1, field_size, 1]) sparse_rows = [[[i]] * field_size for i in range(batch_size)] sparse_inds = tf.reshape( tf.cast(tf.concat([sparse_rows, sparse_cols], 2), tf.int64), [-1, 2]) sparse_vals = tf.reshape(self.vals_, [-1]) sparse_input = tf.SparseTensor(sparse_inds, sparse_vals, [batch_size, feature_size]) weights = tf.Variable(init_weights([feature_size, linear_size]), name="weights") z_BN = tf.sparse_tensor_dense_matmul(sparse_input, weights) batch_mean, batch_var = tf.nn.moments(z_BN, [0]) scale = tf.Variable(tf.ones([linear_size])) beta = tf.Variable(tf.zeros([linear_size])) prelu = keras.PReLU() self.linear = prelu( tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta, scale, epsilon)) self.l2_loss = tf.nn.l2_loss(weights) variable_summaries(weights, 'weights') variable_summaries(self.linear, 'output') # Full Connect 1 with BN with tf.name_scope("fc1"): weights = tf.Variable(init_weights( [product_layer_size + linear_size, fc1_size]), name="weights") z_BN = tf.matmul( tf.concat([self.product_layer, self.linear], 1), weights) batch_mean, batch_var = tf.nn.moments(z_BN, [0]) scale = tf.Variable(tf.ones([fc1_size])) beta = tf.Variable(tf.zeros([fc1_size])) self.fc1 = self.act_summary( tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta, scale, epsilon)) self.l2_loss = tf.nn.l2_loss(weights) variable_summaries(weights, 'weights') variable_summaries(self.fc1, 'output') else: # Full Connect 1 with BN with tf.name_scope("fc1"): weights = tf.Variable(init_weights( [product_layer_size, fc1_size]), name="weights") z_BN = tf.matmul(self.product_layer, weights) batch_mean, batch_var = tf.nn.moments(z_BN, [0]) scale = tf.Variable(tf.ones([fc1_size])) beta = tf.Variable(tf.zeros([fc1_size])) self.fc1 = self.act_summary( tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta, scale, epsilon)) self.l2_loss = tf.nn.l2_loss(weights) variable_summaries(weights, 'weights') variable_summaries(self.fc1, 'output') # Linear 2 with BN with tf.name_scope("fc2"): weights = tf.Variable(init_weights([fc1_size, fc2_size]), name="weights") z_BN = tf.matmul(self.fc1, weights) batch_mean, batch_var = tf.nn.moments(z_BN, [0]) scale = tf.Variable(tf.ones([fc2_size])) beta = tf.Variable(tf.zeros([fc2_size])) self.fc2 = self.act_summary( tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta, scale, epsilon)) self.l2_loss = tf.nn.l2_loss(weights) variable_summaries(weights, 'weights') variable_summaries(self.fc2, 'output') # Softmax with tf.name_scope("Softmax"): weights = tf.Variable(init_weights([fc2_size, NUM_CLASSES]), name="weights") biases = tf.Variable(init_biases([NUM_CLASSES]), name='biases') self.logits = tf.nn.softmax(tf.matmul(self.fc2, weights) + biases, name="scores") self.predictions = tf.argmax(self.logits, 1, name="predictions") self.l2_loss = tf.nn.l2_loss(weights) self.l2_loss = tf.nn.l2_loss(biases) variable_summaries(weights, 'weights') variable_summaries(biases, 'biases') variable_summaries(self.logits, 'output') # Loss with tf.name_scope("loss"): cross_entropy = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.logits, labels=self.labels_)) reg_loss = l2_reg_lambda * self.l2_loss self.loss = cross_entropy + reg_loss tf.summary.scalar('cross_entropy', cross_entropy) tf.summary.scalar('reg_loss', reg_loss) tf.summary.scalar('loss', self.loss) # Accuracy with tf.name_scope("accuracy"): correct_predictions = tf.equal(self.predictions, self.labels_) self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy") tf.summary.scalar('accuracy', self.accuracy) # Auc with tf.name_scope("auc"): _, self.auc = streaming_auc(self.logits[:, 1], self.labels_) tf.summary.scalar('auc', self.auc)
def _model_fn(features, labels, mode): """ :param features: :param labels: :param mode: :return: """ # Pop the name of the signal. if 'FN' in features: names = features.pop('FN') if 'FT' in features: labels = features.pop('FT') # Define the type of the inputs (they are all numeric). columns = [ layers.real_valued_column(key) for key, value in features.items() ] # inputs = layers.input_from_feature_columns(features, columns) # Declare the hidden_layers variable. hidden_layers = None # Iterate all over the hidden units. for unit in hidden_units: # Create a new hidden layer. hidden_layers = tf.layers.dense( inputs=inputs if hidden_layers is None else hidden_layers, activation=tf.nn.relu, units=unit, ) # Create a dropout layer. dropout_layer = layers.dropout(inputs=hidden_layers, keep_prob=1.0 - dropout) # Create the logits layer. logits = tf.layers.dense(inputs=dropout_layer, activation=None, units=2) if mode in (ModeKeys.PREDICT, ModeKeys.EVAL): # Calculate the probabilities. probabilities = tf.nn.softmax(logits) # And their indexes. predictions = tf.argmax(logits, 1) if mode in (ModeKeys.EVAL, ModeKeys.TRAIN): # Convert the labels in the one_hot format. onehot_labels = tf.one_hot(indices=labels, depth=2) # Define the class weights. class_weights = tf.constant(weights) # Deduce weights for batch samples based on their true label. reduced_weights = tf.reduce_sum(class_weights * onehot_labels, axis=1) # Compute your (unweighted) softmax cross entropy loss. unweighted_losses = tf.nn.softmax_cross_entropy_with_logits( labels=onehot_labels, logits=logits) # Apply the weights, relying on broadcasting of the multiplication. weighted_losses = unweighted_losses * reduced_weights # Reduce the result to get your final loss. loss = tf.reduce_mean(weighted_losses) if mode == ModeKeys.PREDICT: # Convert predicted_indices back into strings predictions = { 'classes': predictions, 'scores': probabilities, } # export_outputs = { # 'prediction': tf.estimator.export.PredictOutput(predictions) # } # return tf.estimator.EstimatorSpec( # mode=mode, # predictions=predictions, # # export_outputs=export_outputs, # ) return tf.estimator.EstimatorSpec( mode=mode, predictions=predictions, ) if mode == ModeKeys.TRAIN: # Define the training rule. train_op = layers.optimize_loss( loss=loss, global_step=framework.get_global_step(), learning_rate=learning_rate, optimizer='SGD') return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) if mode == ModeKeys.EVAL: # Define the metrics to show up in the evaluation process. eval_metric_ops = { 'accuracy': metrics.streaming_accuracy(predictions=predictions, labels=labels), 'auroc': metrics.streaming_auc(predictions=predictions, labels=labels), 'recall': metrics.streaming_recall(predictions=predictions, labels=labels), 'precision': metrics.streaming_precision(predictions=predictions, labels=labels), 'TP': metrics.streaming_true_positives(predictions=predictions, labels=labels), 'FN': metrics.streaming_false_negatives(predictions=predictions, labels=labels), 'FP': metrics.streaming_false_positives(predictions=predictions, labels=labels), 'TN': metrics.streaming_true_negatives(predictions=predictions, labels=labels), #'gaccuracy' : metrics.streaming_accuracy(predictions=GP, labels=GL) } return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, loss=loss, eval_metric_ops=eval_metric_ops)