def main(_):
assert FLAGS.train_dir, "--train_dir is required."
if tf.gfile.Exists(FLAGS.summaries_dir):
tf.gfile.DeleteRecursively(FLAGS.summaries_dir)
tf.gfile.MakeDirs(FLAGS.summaries_dir)
config = configuration.Config()
dataset_eval = loader.get_split(FLAGS.split_name,
dataset_dir=FLAGS.data_dir)
if FLAGS.preprocess_abs:
preprocess_fn = tf.abs
else:
preprocess_fn = None
# whther it is a 2d input
is_2D = common.is_2D(FLAGS.model)
series, labels, labels_one_hot = loader.load_batch(
dataset_eval,
batch_size=config.batch_size,
is_2D=is_2D,
preprocess_fn=preprocess_fn)
# Build lazy model
model = common.convert_name_to_instance(FLAGS.model, config, 'eval')
endpoints = model.build(inputs=series, is_training=False)
predictions = tf.to_int64(tf.argmax(endpoints.logits, 1))
slim.get_or_create_global_step()
# Choose the metrics to compute:
names_to_values, names_to_updates = metrics.aggregate_metric_map({
'accuracy':
metrics.streaming_accuracy(predictions, labels),
'precision':
metrics.streaming_precision(predictions, labels),
'recall':
metrics.streaming_recall(predictions, labels),
})
# Create the summary ops such that they also print out to std output:
summary_ops = []
for metric_name, metric_value in names_to_values.iteritems():
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
slim.evaluation.evaluation_loop(
master='',
checkpoint_dir=FLAGS.train_dir,
logdir=FLAGS.summaries_dir,
eval_op=names_to_updates.values(),
num_evals=min(FLAGS.num_batches, dataset_eval.num_samples),
eval_interval_secs=FLAGS.eval_interval_secs,
max_number_of_evaluations=FLAGS.num_of_steps,
summary_op=tf.summary.merge(summary_ops),
session_config=config.session_config,
)
def rnn_segment(features, targets, mode, params):
seq_feature = features['seq_feature']
seq_length = features['seq_length']
with tf.variable_scope("emb"):
embeddings = tf.get_variable(
"char_emb", shape=[params['num_char'], params['emb_size']])
seq_emb = tf.nn.embedding_lookup(embeddings, seq_feature)
batch_size = tf.shape(seq_feature)[0]
time_step = tf.shape(seq_feature)[1]
flat_seq_emb = tf.reshape(
seq_emb,
shape=[batch_size, time_step, (params['k'] + 1) * params['emb_size']])
cell = rnn.LSTMCell(params['rnn_units'])
if mode == ModeKeys.TRAIN:
cell = rnn.DropoutWrapper(cell, params['input_keep_prob'],
params['output_keep_prob'])
projection_cell = rnn.OutputProjectionWrapper(cell, params['num_class'])
logits, _ = tf.nn.dynamic_rnn(projection_cell,
flat_seq_emb,
sequence_length=seq_length,
dtype=tf.float32)
weight_mask = tf.to_float(tf.sequence_mask(seq_length))
loss = seq2seq.sequence_loss(logits, targets, weights=weight_mask)
train_op = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=params["learning_rate"],
optimizer=tf.train.AdamOptimizer,
clip_gradients=params['grad_clip'],
summaries=[
"learning_rate",
"loss",
"gradients",
"gradient_norm",
])
pred_classes = tf.to_int32(tf.argmax(input=logits, axis=2))
pred_words = tf.logical_or(tf.equal(pred_classes, 0),
tf.equal(pred_classes, 3))
target_words = tf.logical_or(tf.equal(targets, 0), tf.equal(targets, 3))
precision = metrics.streaming_precision(pred_words,
target_words,
weights=weight_mask)
recall = metrics.streaming_recall(pred_words,
target_words,
weights=weight_mask)
predictions = {"classes": pred_classes}
eval_metric_ops = {"precision": precision, "recall": recall}
return learn.ModelFnOps(mode,
predictions,
loss,
train_op,
eval_metric_ops=eval_metric_ops)
def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
"""Returns a dictionary of evaluation metric ops for logistic regression.
Args:
labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
predictions: The predictions `Tensor`.
thresholds: List of floating point thresholds to use for accuracy,
precision, and recall metrics.
Returns:
A dict of metric results keyed by name.
"""
# If labels is a dict with a single key, unpack into a single tensor.
labels_tensor = labels
if isinstance(labels, dict) and len(labels) == 1:
labels_tensor = labels.values()[0]
metrics = {}
metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
predictions)
metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
labels_tensor)
# Also include the streaming mean of the label as an accuracy baseline, as
# a reminder to users.
metrics[metric_key.MetricKey.
ACCURACY_BASELINE] = metrics_lib.streaming_mean(labels_tensor)
metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
labels=labels_tensor, predictions=predictions)
for threshold in thresholds:
predictions_at_threshold = math_ops.cast(
math_ops.greater_equal(predictions, threshold),
dtypes.float32,
name='predictions_at_threshold_%f' % threshold)
metrics[metric_key.MetricKey.ACCURACY_MEAN %
threshold] = (metrics_lib.streaming_accuracy(
labels=labels_tensor,
predictions=predictions_at_threshold))
# Precision for positive examples.
metrics[metric_key.MetricKey.PRECISION_MEAN %
threshold] = (metrics_lib.streaming_precision(
labels=labels_tensor,
predictions=predictions_at_threshold))
# Recall for positive examples.
metrics[metric_key.MetricKey.RECALL_MEAN %
threshold] = (metrics_lib.streaming_recall(
labels=labels_tensor,
predictions=predictions_at_threshold))
return metrics
def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
"""Returns a dictionary of evaluation metric ops for logistic regression.
Args:
labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
predictions: The predictions `Tensor`.
thresholds: List of floating point thresholds to use for accuracy,
precision, and recall metrics.
Returns:
A dict of metric results keyed by name.
"""
# If labels is a dict with a single key, unpack into a single tensor.
labels_tensor = labels
if isinstance(labels, dict) and len(labels) == 1:
labels_tensor = labels.values()[0]
metrics = {}
metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
predictions)
metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
labels_tensor)
# Also include the streaming mean of the label as an accuracy baseline, as
# a reminder to users.
metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean(
labels_tensor)
metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
labels=labels_tensor, predictions=predictions)
for threshold in thresholds:
predictions_at_threshold = math_ops.cast(
math_ops.greater_equal(predictions, threshold),
dtypes.float32,
name='predictions_at_threshold_%f' % threshold)
metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = (
metrics_lib.streaming_accuracy(labels=labels_tensor,
predictions=predictions_at_threshold))
# Precision for positive examples.
metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = (
metrics_lib.streaming_precision(labels=labels_tensor,
predictions=predictions_at_threshold))
# Recall for positive examples.
metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = (
metrics_lib.streaming_recall(labels=labels_tensor,
predictions=predictions_at_threshold))
return metrics
def evaluate():
with tf.Graph().as_default():
config = tf.ConfigProto(device_count={'GPU': 0})
images, labels = utils.load_batch(shards=VAL_SHARDS,
batch_size=FLAGS.batch_size,
train=False,
crop=False,
flip=False)
predictions = alexnet.AlexNet(images,
batch_size=FLAGS.batch_size).model
prediction = tf.to_int64(tf.argmax(predictions,
1)) # Returns index of largest
mse_op = metrics.streaming_mean_squared_error(prediction, labels)
rmse_op = metrics.streaming_root_mean_squared_error(prediction, labels)
accuracy_op = metrics.streaming_accuracy(prediction, labels)
precision_op = metrics.streaming_precision(prediction, labels)
metrics_to_values, metrics_to_updates = metrics.aggregate_metric_map({
'mse':
mse_op,
'rmse':
rmse_op,
'accuracy':
accuracy_op,
'precision':
precision_op,
})
for metric_name, metric_value in metrics_to_values.items():
tf.summary.scalar(metric_name, metric_value)
slim.evaluation.evaluation_loop('',
FLAGS.trainlog_dir,
FLAGS.evallog_dir,
num_evals=FLAGS.num_evals,
eval_op=list(
metrics_to_updates.values()),
eval_interval_secs=5,
session_config=config)
'''checkpoint_list = [FLAGS.trainlog_momentum_dir,
def build_test_metrics(self):
raw_labels = multihot_labels(self.mmsis)
mask = tf.to_float(tf.equal(tf.reduce_sum(raw_labels, 1), 1))
labels = tf.to_int32(tf.argmax(raw_labels, 1))
predictions = tf.to_int32(tf.argmax(self.prediction, 1))
metrics_map = {
'%s/Test-accuracy' % self.name:
metrics.streaming_accuracy(predictions,
labels,
weights=mask)
}
if self.metrics == 'all':
for i, cls in enumerate(self.classes):
cls_name = cls.replace(' ', '-')
trues = tf.to_int32(tf.equal(labels, i))
preds = tf.to_int32(tf.equal(predictions, i))
recall = metrics.streaming_recall(preds,
trues,
weights=mask)
precision = metrics.streaming_precision(preds,
trues,
weights=mask)
metrics_map["%s/Class-%s-Precision" %
(self.name, cls_name)] = recall
metrics_map["%s/Class-%s-Recall" %
(self.name, cls_name)] = precision
metrics_map["%s/Class-%s-F1-Score" %
(self.name, cls_name)] = f1(
recall, precision)
metrics_map["%s/Class-%s-ROC-AUC" %
(self.name,
cls_name)] = metrics.streaming_auc(
self.prediction[:, i],
trues,
weights=mask)
return metrics.aggregate_metric_map(metrics_map)
def _model_fn(features, labels, mode):
"""
:param features:
:param labels:
:param mode:
:return:
"""
# Pop the name of the signal.
if 'FN' in features:
names = features.pop('FN')
if 'FT' in features:
labels = features.pop('FT')
# Define the type of the inputs (they are all numeric).
columns = [
layers.real_valued_column(key) for key, value in features.items()
]
#
inputs = layers.input_from_feature_columns(features, columns)
# Declare the hidden_layers variable.
hidden_layers = None
# Iterate all over the hidden units.
for unit in hidden_units:
# Create a new hidden layer.
hidden_layers = tf.layers.dense(
inputs=inputs if hidden_layers is None else hidden_layers,
activation=tf.nn.relu,
units=unit,
)
# Create a dropout layer.
dropout_layer = layers.dropout(inputs=hidden_layers,
keep_prob=1.0 - dropout)
# Create the logits layer.
logits = tf.layers.dense(inputs=dropout_layer,
activation=None,
units=2)
if mode in (ModeKeys.PREDICT, ModeKeys.EVAL):
# Calculate the probabilities.
probabilities = tf.nn.softmax(logits)
# And their indexes.
predictions = tf.argmax(logits, 1)
if mode in (ModeKeys.EVAL, ModeKeys.TRAIN):
# Convert the labels in the one_hot format.
onehot_labels = tf.one_hot(indices=labels, depth=2)
# Define the class weights.
class_weights = tf.constant(weights)
# Deduce weights for batch samples based on their true label.
reduced_weights = tf.reduce_sum(class_weights * onehot_labels,
axis=1)
# Compute your (unweighted) softmax cross entropy loss.
unweighted_losses = tf.nn.softmax_cross_entropy_with_logits(
labels=onehot_labels, logits=logits)
# Apply the weights, relying on broadcasting of the multiplication.
weighted_losses = unweighted_losses * reduced_weights
# Reduce the result to get your final loss.
loss = tf.reduce_mean(weighted_losses)
if mode == ModeKeys.PREDICT:
# Convert predicted_indices back into strings
predictions = {
'classes': predictions,
'scores': probabilities,
}
# export_outputs = {
# 'prediction': tf.estimator.export.PredictOutput(predictions)
# }
# return tf.estimator.EstimatorSpec(
# mode=mode,
# predictions=predictions,
# # export_outputs=export_outputs,
# )
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
)
if mode == ModeKeys.TRAIN:
# Define the training rule.
train_op = layers.optimize_loss(
loss=loss,
global_step=framework.get_global_step(),
learning_rate=learning_rate,
optimizer='SGD')
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
if mode == ModeKeys.EVAL:
# Define the metrics to show up in the evaluation process.
eval_metric_ops = {
'accuracy':
metrics.streaming_accuracy(predictions=predictions,
labels=labels),
'auroc':
metrics.streaming_auc(predictions=predictions, labels=labels),
'recall':
metrics.streaming_recall(predictions=predictions,
labels=labels),
'precision':
metrics.streaming_precision(predictions=predictions,
labels=labels),
'TP':
metrics.streaming_true_positives(predictions=predictions,
labels=labels),
'FN':
metrics.streaming_false_negatives(predictions=predictions,
labels=labels),
'FP':
metrics.streaming_false_positives(predictions=predictions,
labels=labels),
'TN':
metrics.streaming_true_negatives(predictions=predictions,
labels=labels),
#'gaccuracy' : metrics.streaming_accuracy(predictions=GP, labels=GL)
}
return tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss,
eval_metric_ops=eval_metric_ops)