def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {
"loss":
metrics_lib.streaming_mean(
self._loss(logits,
targets,
weight_tensor=self._get_weight_tensor(features)))
}
# Adding default metrics
if metrics is None:
metrics = {"accuracy": metrics_lib.streaming_accuracy}
if self._n_classes == 2:
predictions = math_ops.sigmoid(logits)
result["eval_auc"] = metrics_lib.streaming_auc(
predictions, targets)
if metrics:
predictions = self._logits_to_predictions(logits, proba=False)
result.update(
self._run_metrics(predictions, targets, metrics,
self._get_weight_tensor(features)))
return result
def _streaming_auc(predictions, labels, weights=None, class_id=None):
if class_id is not None:
predictions = predictions[:, class_id]
labels = labels[:, class_id]
return metrics_lib.streaming_auc(
predictions, math_ops.cast(labels, dtypes.bool),
weights=_float_weights_or_none(weights))
def __init__(self,
feature_dims,
hidden_units=None,
layers=1,
model_dir=None):
self.model_dir = model_dir
x = tf.placeholder(tf.float32, [None, None, feature_dims])
y = tf.placeholder(tf.float32, [None, 1])
length = tf.placeholder(tf.int32, [None])
self.x, self.y, self.length = x, y, length
cell = tf.nn.rnn_cell.LSTMCell(hidden_units, state_is_tuple=True)
if layers > 1:
cell = tf.nn.rnn_cell.MultiRNNCell([cell] * layers,
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(
cell=cell,
inputs=x,
dtype=tf.float32,
sequence_length=length,
)
batch_size = tf.shape(output)[0]
max_length = tf.shape(output)[1]
out_size = int(output.get_shape()[2])
index = tf.range(0, batch_size) * max_length + (length - 1)
flat = tf.reshape(output, [-1, out_size])
relevant = tf.gather(flat, index)
logit = tf.contrib.layers.fully_connected(inputs=relevant,
num_outputs=1,
activation_fn=None,
biases_initializer=None)
predictions = tf.sigmoid(logit)
loss = tf.nn.sigmoid_cross_entropy_with_logits(logit, y)
auc, update_auc = metrics.streaming_auc(predictions,
y,
num_thresholds=10)
stream_loss, update_stream_loss = metrics.streaming_mean(loss)
self.update_metrics = [update_stream_loss, update_auc]
self.summaries = [auc, stream_loss]
self.summary_labels = ['auc', 'loss']
self.predictions = predictions
self.learning_rate = tf.placeholder(tf.float32, [])
self._global_step = framework.create_global_step()
self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(
loss, global_step=self._global_step)
self.saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.initialize_local_variables())
sess.run(tf.initialize_all_variables())
self.sess = sess
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {
"loss":
metrics_lib.streaming_mean(
self._loss(logits,
targets,
weight_tensor=self._get_weight_tensor(features)))
}
# Adding default metrics
if metrics is None:
metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}
if self._n_classes == 2:
predictions = math_ops.sigmoid(logits)
result["auc"] = metrics_lib.streaming_auc(predictions, targets)
if metrics:
class_metrics = {}
proba_metrics = {}
for name, metric_op in six.iteritems(metrics):
if isinstance(name, tuple):
if len(name) != 2:
raise ValueError(
"Ignoring metric {}. It returned a tuple with "
"len {}, expected 2.".format(name, len(name)))
else:
if name[1] not in ["classes", "probabilities"]:
raise ValueError(
"Ignoring metric {}. The 2nd element of its "
"name should be either 'classes' or "
"'probabilities'.".format(name))
elif name[1] == "classes":
class_metrics[name[0]] = metric_op
else:
proba_metrics[name[0]] = metric_op
elif isinstance(name, str):
class_metrics[name] = metric_op
else:
raise ValueError(
"Ignoring metric {}. Its name is not in the correct "
"form.".format(name))
if class_metrics:
predictions = self._logits_to_predictions(logits, proba=False)
result.update(
self._run_metrics(predictions, targets, class_metrics,
self._get_weight_tensor(features)))
if proba_metrics:
predictions = self._logits_to_predictions(logits, proba=True)
result.update(
self._run_metrics(predictions, targets, proba_metrics,
self._get_weight_tensor(features)))
return result
def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
"""Returns a dictionary of evaluation metric ops for logistic regression.
Args:
labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
predictions: The predictions `Tensor`.
thresholds: List of floating point thresholds to use for accuracy,
precision, and recall metrics.
Returns:
A dict of metric results keyed by name.
"""
# If labels is a dict with a single key, unpack into a single tensor.
labels_tensor = labels
if isinstance(labels, dict) and len(labels) == 1:
labels_tensor = labels.values()[0]
metrics = {}
metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
predictions)
metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
labels_tensor)
# Also include the streaming mean of the label as an accuracy baseline, as
# a reminder to users.
metrics[metric_key.MetricKey.
ACCURACY_BASELINE] = metrics_lib.streaming_mean(labels_tensor)
metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
labels=labels_tensor, predictions=predictions)
for threshold in thresholds:
predictions_at_threshold = math_ops.cast(
math_ops.greater_equal(predictions, threshold),
dtypes.float32,
name='predictions_at_threshold_%f' % threshold)
metrics[metric_key.MetricKey.ACCURACY_MEAN %
threshold] = (metrics_lib.streaming_accuracy(
labels=labels_tensor,
predictions=predictions_at_threshold))
# Precision for positive examples.
metrics[metric_key.MetricKey.PRECISION_MEAN %
threshold] = (metrics_lib.streaming_precision(
labels=labels_tensor,
predictions=predictions_at_threshold))
# Recall for positive examples.
metrics[metric_key.MetricKey.RECALL_MEAN %
threshold] = (metrics_lib.streaming_recall(
labels=labels_tensor,
predictions=predictions_at_threshold))
return metrics
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {"loss": metrics_lib.streaming_mean(self._loss(
logits, targets,
weight_tensor=self._get_weight_tensor(features)))}
# Adding default metrics
if metrics is None:
metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}
if self._n_classes == 2:
predictions = math_ops.sigmoid(logits)
result["auc"] = metrics_lib.streaming_auc(predictions, targets)
if metrics:
class_metrics = {}
proba_metrics = {}
for name, metric_op in six.iteritems(metrics):
if isinstance(name, tuple):
if len(name) != 2:
raise ValueError("Ignoring metric {}. It returned a tuple with "
"len {}, expected 2.".format(name, len(name)))
else:
if name[1] not in ["classes", "probabilities"]:
raise ValueError("Ignoring metric {}. The 2nd element of its "
"name should be either 'classes' or "
"'probabilities'.".format(name))
elif name[1] == "classes":
class_metrics[name[0]] = metric_op
else:
proba_metrics[name[0]] = metric_op
elif isinstance(name, str):
class_metrics[name] = metric_op
else:
raise ValueError("Ignoring metric {}. Its name is not in the correct "
"form.".format(name))
if class_metrics:
predictions = self._logits_to_predictions(logits, proba=False)
result.update(self._run_metrics(predictions, targets, class_metrics,
self._get_weight_tensor(features)))
if proba_metrics:
predictions = self._logits_to_predictions(logits, proba=True)
result.update(self._run_metrics(predictions, targets, proba_metrics,
self._get_weight_tensor(features)))
return result
def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
"""Returns a dictionary of evaluation metric ops for logistic regression.
Args:
labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
predictions: The predictions `Tensor`.
thresholds: List of floating point thresholds to use for accuracy,
precision, and recall metrics.
Returns:
A dict of metric results keyed by name.
"""
# If labels is a dict with a single key, unpack into a single tensor.
labels_tensor = labels
if isinstance(labels, dict) and len(labels) == 1:
labels_tensor = labels.values()[0]
metrics = {}
metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
predictions)
metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
labels_tensor)
# Also include the streaming mean of the label as an accuracy baseline, as
# a reminder to users.
metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean(
labels_tensor)
metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
labels=labels_tensor, predictions=predictions)
for threshold in thresholds:
predictions_at_threshold = math_ops.cast(
math_ops.greater_equal(predictions, threshold),
dtypes.float32,
name='predictions_at_threshold_%f' % threshold)
metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = (
metrics_lib.streaming_accuracy(labels=labels_tensor,
predictions=predictions_at_threshold))
# Precision for positive examples.
metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = (
metrics_lib.streaming_precision(labels=labels_tensor,
predictions=predictions_at_threshold))
# Recall for positive examples.
metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = (
metrics_lib.streaming_recall(labels=labels_tensor,
predictions=predictions_at_threshold))
return metrics
def build_test_metrics(self):
raw_labels = multihot_labels(self.mmsis)
mask = tf.to_float(tf.equal(tf.reduce_sum(raw_labels, 1), 1))
labels = tf.to_int32(tf.argmax(raw_labels, 1))
predictions = tf.to_int32(tf.argmax(self.prediction, 1))
metrics_map = {
'%s/Test-accuracy' % self.name:
metrics.streaming_accuracy(predictions,
labels,
weights=mask)
}
if self.metrics == 'all':
for i, cls in enumerate(self.classes):
cls_name = cls.replace(' ', '-')
trues = tf.to_int32(tf.equal(labels, i))
preds = tf.to_int32(tf.equal(predictions, i))
recall = metrics.streaming_recall(preds,
trues,
weights=mask)
precision = metrics.streaming_precision(preds,
trues,
weights=mask)
metrics_map["%s/Class-%s-Precision" %
(self.name, cls_name)] = recall
metrics_map["%s/Class-%s-Recall" %
(self.name, cls_name)] = precision
metrics_map["%s/Class-%s-F1-Score" %
(self.name, cls_name)] = f1(
recall, precision)
metrics_map["%s/Class-%s-ROC-AUC" %
(self.name,
cls_name)] = metrics.streaming_auc(
self.prediction[:, i],
trues,
weights=mask)
return metrics.aggregate_metric_map(metrics_map)
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {"loss": metrics_lib.streaming_mean(self._loss(
logits, targets,
weight_tensor=self._get_weight_tensor(features)))}
# Adding default metrics
if metrics is None:
metrics = {"accuracy": metrics_lib.streaming_accuracy}
if self._n_classes == 2:
predictions = math_ops.sigmoid(logits)
result["eval_auc"] = metrics_lib.streaming_auc(predictions, targets)
if metrics:
predictions = self._logits_to_predictions(logits, proba=False)
result.update(self._run_metrics(predictions, targets, metrics,
self._get_weight_tensor(features)))
return result
def _streaming_auc(predictions, labels, weights=None):
return metrics_lib.streaming_auc(predictions, labels,
weights=_float_weights_or_none(weights))
def _streaming_auc(predictions, target, weights=None):
return metrics_lib.streaming_auc(predictions, target,
weights=_float_weights_or_none(weights))
def __init__(self,
limits,
embed_size=8,
fc1_size=200,
fc2_size=100,
linear_size=800,
l2_reg_lambda=0.0,
NUM_CLASSES=2,
inds=None,
vals=None,
labels=None,
linear=False):
field_size = len(limits) - 1
feature_size = int(limits[-1])
if inds == None:
self.inds_ = tf.placeholder(tf.int32, shape=[None, field_size])
else:
self.inds_ = inds
if vals == None:
self.vals_ = tf.placeholder(tf.float32, shape=[None, field_size])
else:
self.vals_ = vals
if labels == None:
self.labels_ = tf.placeholder(tf.int64, shape=[None])
else:
self.labels_ = labels
init_weights = tf.truncated_normal_initializer(0.0, stddev=0.1)
init_biases = tf.constant_initializer(0.1)
self.l2_loss = tf.constant(0.00)
epsilon = 1e-3
# Field_embed
with tf.name_scope("field_embed"):
weights = tf.Variable(init_weights(
[feature_size, field_size, embed_size]),
name="weights")
biases = tf.Variable(init_biases(
[field_size, field_size, embed_size]),
name='biases')
self.field_embed = self.act_summary(
field_embed_op(self.inds_, self.vals_, weights, biases,
limits))
self.l2_loss = tf.nn.l2_loss(weights)
self.l2_loss = tf.nn.l2_loss(biases)
variable_summaries(weights, 'weights')
variable_summaries(biases, 'biases')
variable_summaries(self.field_embed, 'output')
# Product_layer
with tf.name_scope("product_layer"):
self.product_layer = self.act_summary(
product_layer_op(self.field_embed))
product_layer_size = int(self.product_layer.get_shape()[-1])
variable_summaries(self.product_layer, 'output')
if linear:
# Linear feature
with tf.name_scope("linear"):
batch_size = int(self.inds_.get_shape()[0])
sparse_cols = tf.reshape(
tf.add(self.inds_, [limits[:-1]] * batch_size),
[-1, field_size, 1])
sparse_rows = [[[i]] * field_size for i in range(batch_size)]
sparse_inds = tf.reshape(
tf.cast(tf.concat([sparse_rows, sparse_cols], 2),
tf.int64), [-1, 2])
sparse_vals = tf.reshape(self.vals_, [-1])
sparse_input = tf.SparseTensor(sparse_inds, sparse_vals,
[batch_size, feature_size])
weights = tf.Variable(init_weights([feature_size,
linear_size]),
name="weights")
z_BN = tf.sparse_tensor_dense_matmul(sparse_input, weights)
batch_mean, batch_var = tf.nn.moments(z_BN, [0])
scale = tf.Variable(tf.ones([linear_size]))
beta = tf.Variable(tf.zeros([linear_size]))
prelu = keras.PReLU()
self.linear = prelu(
tf.nn.batch_normalization(z_BN, batch_mean, batch_var,
beta, scale, epsilon))
self.l2_loss = tf.nn.l2_loss(weights)
variable_summaries(weights, 'weights')
variable_summaries(self.linear, 'output')
# Full Connect 1 with BN
with tf.name_scope("fc1"):
weights = tf.Variable(init_weights(
[product_layer_size + linear_size, fc1_size]),
name="weights")
z_BN = tf.matmul(
tf.concat([self.product_layer, self.linear], 1), weights)
batch_mean, batch_var = tf.nn.moments(z_BN, [0])
scale = tf.Variable(tf.ones([fc1_size]))
beta = tf.Variable(tf.zeros([fc1_size]))
self.fc1 = self.act_summary(
tf.nn.batch_normalization(z_BN, batch_mean, batch_var,
beta, scale, epsilon))
self.l2_loss = tf.nn.l2_loss(weights)
variable_summaries(weights, 'weights')
variable_summaries(self.fc1, 'output')
else:
# Full Connect 1 with BN
with tf.name_scope("fc1"):
weights = tf.Variable(init_weights(
[product_layer_size, fc1_size]),
name="weights")
z_BN = tf.matmul(self.product_layer, weights)
batch_mean, batch_var = tf.nn.moments(z_BN, [0])
scale = tf.Variable(tf.ones([fc1_size]))
beta = tf.Variable(tf.zeros([fc1_size]))
self.fc1 = self.act_summary(
tf.nn.batch_normalization(z_BN, batch_mean, batch_var,
beta, scale, epsilon))
self.l2_loss = tf.nn.l2_loss(weights)
variable_summaries(weights, 'weights')
variable_summaries(self.fc1, 'output')
# Linear 2 with BN
with tf.name_scope("fc2"):
weights = tf.Variable(init_weights([fc1_size, fc2_size]),
name="weights")
z_BN = tf.matmul(self.fc1, weights)
batch_mean, batch_var = tf.nn.moments(z_BN, [0])
scale = tf.Variable(tf.ones([fc2_size]))
beta = tf.Variable(tf.zeros([fc2_size]))
self.fc2 = self.act_summary(
tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta,
scale, epsilon))
self.l2_loss = tf.nn.l2_loss(weights)
variable_summaries(weights, 'weights')
variable_summaries(self.fc2, 'output')
# Softmax
with tf.name_scope("Softmax"):
weights = tf.Variable(init_weights([fc2_size, NUM_CLASSES]),
name="weights")
biases = tf.Variable(init_biases([NUM_CLASSES]), name='biases')
self.logits = tf.nn.softmax(tf.matmul(self.fc2, weights) + biases,
name="scores")
self.predictions = tf.argmax(self.logits, 1, name="predictions")
self.l2_loss = tf.nn.l2_loss(weights)
self.l2_loss = tf.nn.l2_loss(biases)
variable_summaries(weights, 'weights')
variable_summaries(biases, 'biases')
variable_summaries(self.logits, 'output')
# Loss
with tf.name_scope("loss"):
cross_entropy = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.logits, labels=self.labels_))
reg_loss = l2_reg_lambda * self.l2_loss
self.loss = cross_entropy + reg_loss
tf.summary.scalar('cross_entropy', cross_entropy)
tf.summary.scalar('reg_loss', reg_loss)
tf.summary.scalar('loss', self.loss)
# Accuracy
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(self.predictions, self.labels_)
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions,
"float"),
name="accuracy")
tf.summary.scalar('accuracy', self.accuracy)
# Auc
with tf.name_scope("auc"):
_, self.auc = streaming_auc(self.logits[:, 1], self.labels_)
tf.summary.scalar('auc', self.auc)
def _model_fn(features, labels, mode):
"""
:param features:
:param labels:
:param mode:
:return:
"""
# Pop the name of the signal.
if 'FN' in features:
names = features.pop('FN')
if 'FT' in features:
labels = features.pop('FT')
# Define the type of the inputs (they are all numeric).
columns = [
layers.real_valued_column(key) for key, value in features.items()
]
#
inputs = layers.input_from_feature_columns(features, columns)
# Declare the hidden_layers variable.
hidden_layers = None
# Iterate all over the hidden units.
for unit in hidden_units:
# Create a new hidden layer.
hidden_layers = tf.layers.dense(
inputs=inputs if hidden_layers is None else hidden_layers,
activation=tf.nn.relu,
units=unit,
)
# Create a dropout layer.
dropout_layer = layers.dropout(inputs=hidden_layers,
keep_prob=1.0 - dropout)
# Create the logits layer.
logits = tf.layers.dense(inputs=dropout_layer,
activation=None,
units=2)
if mode in (ModeKeys.PREDICT, ModeKeys.EVAL):
# Calculate the probabilities.
probabilities = tf.nn.softmax(logits)
# And their indexes.
predictions = tf.argmax(logits, 1)
if mode in (ModeKeys.EVAL, ModeKeys.TRAIN):
# Convert the labels in the one_hot format.
onehot_labels = tf.one_hot(indices=labels, depth=2)
# Define the class weights.
class_weights = tf.constant(weights)
# Deduce weights for batch samples based on their true label.
reduced_weights = tf.reduce_sum(class_weights * onehot_labels,
axis=1)
# Compute your (unweighted) softmax cross entropy loss.
unweighted_losses = tf.nn.softmax_cross_entropy_with_logits(
labels=onehot_labels, logits=logits)
# Apply the weights, relying on broadcasting of the multiplication.
weighted_losses = unweighted_losses * reduced_weights
# Reduce the result to get your final loss.
loss = tf.reduce_mean(weighted_losses)
if mode == ModeKeys.PREDICT:
# Convert predicted_indices back into strings
predictions = {
'classes': predictions,
'scores': probabilities,
}
# export_outputs = {
# 'prediction': tf.estimator.export.PredictOutput(predictions)
# }
# return tf.estimator.EstimatorSpec(
# mode=mode,
# predictions=predictions,
# # export_outputs=export_outputs,
# )
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
)
if mode == ModeKeys.TRAIN:
# Define the training rule.
train_op = layers.optimize_loss(
loss=loss,
global_step=framework.get_global_step(),
learning_rate=learning_rate,
optimizer='SGD')
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
if mode == ModeKeys.EVAL:
# Define the metrics to show up in the evaluation process.
eval_metric_ops = {
'accuracy':
metrics.streaming_accuracy(predictions=predictions,
labels=labels),
'auroc':
metrics.streaming_auc(predictions=predictions, labels=labels),
'recall':
metrics.streaming_recall(predictions=predictions,
labels=labels),
'precision':
metrics.streaming_precision(predictions=predictions,
labels=labels),
'TP':
metrics.streaming_true_positives(predictions=predictions,
labels=labels),
'FN':
metrics.streaming_false_negatives(predictions=predictions,
labels=labels),
'FP':
metrics.streaming_false_positives(predictions=predictions,
labels=labels),
'TN':
metrics.streaming_true_negatives(predictions=predictions,
labels=labels),
#'gaccuracy' : metrics.streaming_accuracy(predictions=GP, labels=GL)
}
return tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss,
eval_metric_ops=eval_metric_ops)
