def _make_logistic_eval_metric_ops(labels, predictions, thresholds):
"""Returns a dictionary of evaluation metric ops for logistic regression.
Args:
labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name.
predictions: The predictions `Tensor`.
thresholds: List of floating point thresholds to use for accuracy,
precision, and recall metrics.
Returns:
A dict of metric results keyed by name.
"""
# If labels is a dict with a single key, unpack into a single tensor.
labels_tensor = labels
if isinstance(labels, dict) and len(labels) == 1:
labels_tensor = labels.values()[0]
metrics = {}
metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean(
predictions)
metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean(
labels_tensor)
# Also include the streaming mean of the label as an accuracy baseline, as
# a reminder to users.
metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean(
labels_tensor)
metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc(
labels=labels_tensor, predictions=predictions)
for threshold in thresholds:
predictions_at_threshold = math_ops.cast(
math_ops.greater_equal(predictions, threshold),
dtypes.float32,
name='predictions_at_threshold_%f' % threshold)
metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = (
metrics_lib.streaming_accuracy(labels=labels_tensor,
predictions=predictions_at_threshold))
# Precision for positive examples.
metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = (
metrics_lib.streaming_precision(labels=labels_tensor,
predictions=predictions_at_threshold))
# Recall for positive examples.
metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = (
metrics_lib.streaming_recall(labels=labels_tensor,
predictions=predictions_at_threshold))
return metrics
def _get_eval_ops(self, features, targets, metrics):
"""Method that builds model graph and returns evaluation ops.
Expected to be overriden by sub-classes that require custom support.
This implementation uses `model_fn` passed as parameter to constructor to
build model.
Args:
features: `Tensor` or `dict` of `Tensor` objects.
targets: `Tensor` or `dict` of `Tensor` objects.
metrics: Dict of metrics to run. If None, the default metric functions
are used; if {}, no metrics are used. Otherwise, `metrics` should map
friendly names for the metric to a `MetricSpec` object defining which
model outputs to evaluate against which targets with which metric
function. Metric ops should support streaming, e.g., returning
update_op and value tensors. See more details in
`../../../../metrics/python/metrics/ops/streaming_metrics.py` and
`../metric_spec.py`.
Returns:
metrics: `dict` of `Tensor` objects.
Raises:
ValueError: if `metrics` don't match `targets`.
"""
predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL)
result = {'loss': metrics_lib.streaming_mean(loss)}
result.update(_make_metrics_ops(metrics, features, targets, predictions))
return result
def _get_eval_ops(self, features, targets, metrics=None):
logits = self._model.build_model(
features, self._feature_columns, is_training=False)
model_fn_ops = self._head.head_ops(features, targets,
tf.contrib.learn.ModeKeys.TRAIN,
_noop_training_fn, logits=logits)
return {'loss': metrics_lib.streaming_mean(model_fn_ops.loss)}
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {"loss": metrics_lib.streaming_mean(self._loss(
logits, targets, features))}
# Adds default metrics.
if metrics is None:
# TODO(b/29366811): This currently results in both an "accuracy" and an
# "accuracy/threshold_0.500000_mean" metric for binary classification.
metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}
# Adds additional useful metrics for the special case of binary
# classification.
# TODO(zakaria): Move LogisticRegressor.get_default_metrics to metrics
# and handle eval metric from targetcolumn.
if self._target_column.num_label_columns == 1:
predictions = math_ops.sigmoid(logits)
targets_float = math_ops.to_float(targets)
default_metrics = (
logistic_regressor.LogisticRegressor.get_default_metrics())
for metric_name, metric_op in default_metrics.items():
result[metric_name] = metric_op(predictions, targets_float)
if metrics:
class_metrics = {}
proba_metrics = {}
for name, metric_op in six.iteritems(metrics):
if isinstance(name, tuple):
if len(name) != 2:
raise ValueError("Ignoring metric {}. It returned a tuple with "
"len {}, expected 2.".format(name, len(name)))
else:
if name[1] not in ["classes", "probabilities"]:
raise ValueError("Ignoring metric {}. The 2nd element of its "
"name should be either 'classes' or "
"'probabilities'.".format(name))
elif name[1] == "classes":
class_metrics[name[0]] = metric_op
else:
proba_metrics[name[0]] = metric_op
elif isinstance(name, str):
class_metrics[name] = metric_op
else:
raise ValueError("Ignoring metric {}. Its name is not in the correct "
"form.".format(name))
if class_metrics:
predictions = self._target_column.logits_to_predictions(logits,
proba=False)
result.update(self._run_metrics(predictions, targets, class_metrics,
self._target_column.get_weight_tensor(
features)))
if proba_metrics:
predictions = self._target_column.logits_to_predictions(logits,
proba=True)
result.update(self._run_metrics(predictions, targets, proba_metrics,
self._target_column.get_weight_tensor(
features)))
return result
def _get_eval_ops(self, features, targets, metrics):
"""Method that builds model graph and returns evaluation ops.
Expected to be overriden by sub-classes that require custom support.
This implementation uses `model_fn` passed as parameter to constructor to
build model.
Args:
features: `Tensor` or `dict` of `Tensor` objects.
targets: `Tensor` or `dict` of `Tensor` objects.
metrics: Dict of metric ops to run. If None, the default metric functions
are used; if {}, no metrics are used. If model has one output (i.e.,
returning single predction), keys are `str`, e.g. `'accuracy'` - just a
name of the metric that will show up in the logs / summaries.
Otherwise, keys are tuple of two `str`, e.g. `('accuracy', 'classes')`
- name of the metric and name of `Tensor` in the predictions to run
this metric on. Metric ops should support streaming, e.g., returning
update_op and value tensors. See more details in
../../../../metrics/python/metrics/ops/streaming_metrics.py.
Returns:
metrics: `dict` of `Tensor` objects.
Raises:
ValueError: if `metrics` don't match `targets`.
"""
predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL)
result = {"loss": metrics_lib.streaming_mean(loss)}
weights = self._get_weight_tensor(features)
metrics = metrics or {}
if isinstance(targets, dict) and len(targets) == 1:
# Unpack single target into just tensor.
targets = targets[list(targets.keys())[0]]
for name, metric in six.iteritems(metrics):
if isinstance(name, tuple):
# Multi-head metrics.
if not isinstance(predictions, dict):
raise ValueError(
"Metrics passed provide (name, prediction), "
"but predictions are not dict. "
"Metrics: %s, Predictions: %s." % (metrics, predictions)
)
# Here are two options: targets are single Tensor or a dict.
if isinstance(targets, dict) and name[1] in targets:
# If targets are dict and the prediction name is in it, apply metric.
result[name[0]] = metrics_lib.run_metric(metric, predictions[name[1]], targets[name[1]], weights)
else:
# Otherwise pass the targets to the metric.
result[name[0]] = metrics_lib.run_metric(metric, predictions[name[1]], targets, weights)
else:
# Single head metrics.
if isinstance(predictions, dict):
raise ValueError(
"Metrics passed provide only name, no prediction, "
"but predictions are dict. "
"Metrics: %s, Targets: %s." % (metrics, targets)
)
result[name] = metrics_lib.run_metric(metric, predictions, targets, weights)
return result
def get_eval_ops(self, features, logits, targets, metrics=None):
loss = self.loss(logits, targets, features)
result = {"loss": metrics_lib.streaming_mean(loss)}
if metrics:
predictions = self.logits_to_predictions(logits, proba=False)
result.update(_run_metrics(predictions, targets, metrics, self.get_weight_tensor(features)))
return result
def _streaming_weighted_average_loss(predictions, target, weights=None):
loss_unweighted = loss_fn(predictions, target)
if weights is not None:
weights = math_ops.to_float(weights)
_, weighted_average_loss = _loss(loss_unweighted,
weights,
name="eval_loss")
return metrics_lib.streaming_mean(weighted_average_loss)
def _class_predictions_streaming_mean(
predictions, labels, weights=None, class_id=None):
del labels
return metrics_lib.streaming_mean(
array_ops.where(
math_ops.equal(
math_ops.to_int32(class_id),
math_ops.to_int32(predictions)),
array_ops.ones_like(predictions),
array_ops.zeros_like(predictions)),
weights=weights)
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {"loss": metrics_lib.streaming_mean(self._loss(logits, targets, features))}
if metrics:
predictions = self._target_column.logits_to_predictions(logits, proba=False)
result.update(
self._run_metrics(predictions, targets, metrics, self._target_column.get_weight_tensor(features))
)
return result
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {"loss": metrics_lib.streaming_mean(self._loss(
logits, targets,
weight_tensor=self._get_weight_tensor(features)))}
# Adding default metrics
if metrics is None:
metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}
if self._n_classes == 2:
predictions = math_ops.sigmoid(logits)
result["auc"] = metrics_lib.streaming_auc(predictions, targets)
if metrics:
class_metrics = {}
proba_metrics = {}
for name, metric_op in six.iteritems(metrics):
if isinstance(name, tuple):
if len(name) != 2:
raise ValueError("Ignoring metric {}. It returned a tuple with "
"len {}, expected 2.".format(name, len(name)))
else:
if name[1] not in ["classes", "probabilities"]:
raise ValueError("Ignoring metric {}. The 2nd element of its "
"name should be either 'classes' or "
"'probabilities'.".format(name))
elif name[1] == "classes":
class_metrics[name[0]] = metric_op
else:
proba_metrics[name[0]] = metric_op
elif isinstance(name, str):
class_metrics[name] = metric_op
else:
raise ValueError("Ignoring metric {}. Its name is not in the correct "
"form.".format(name))
if class_metrics:
predictions = self._logits_to_predictions(logits, proba=False)
result.update(self._run_metrics(predictions, targets, class_metrics,
self._get_weight_tensor(features)))
if proba_metrics:
predictions = self._logits_to_predictions(logits, proba=True)
result.update(self._run_metrics(predictions, targets, proba_metrics,
self._get_weight_tensor(features)))
return result
def get_eval_ops(self, features, logits, targets, metrics=None):
loss = self.loss(logits, targets, features)
result = {"loss": metrics_lib.streaming_mean(loss)}
# Adds default metrics.
if metrics is None:
# TODO(b/29366811): This currently results in both an "accuracy" and an
# "accuracy/threshold_0.500000_mean" metric for binary classification.
metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}
predictions = math_ops.sigmoid(logits)
targets_float = math_ops.to_float(targets)
default_metrics = self._default_eval_metrics()
for metric_name, metric_op in default_metrics.items():
result[metric_name] = metric_op(predictions, targets_float)
class_metrics = {}
proba_metrics = {}
for name, metric_op in six.iteritems(metrics):
if isinstance(name, tuple):
if len(name) != 2:
raise ValueError("Ignoring metric {}. It returned a tuple with "
"len {}, expected 2.".format(name, len(name)))
else:
if name[1] not in ["classes", "probabilities"]:
raise ValueError("Ignoring metric {}. The 2nd element of its "
"name should be either 'classes' or "
"'probabilities'.".format(name))
elif name[1] == "classes":
class_metrics[name[0]] = metric_op
else:
proba_metrics[name[0]] = metric_op
elif isinstance(name, str):
class_metrics[name] = metric_op
else:
raise ValueError("Ignoring metric {}. Its name is not in the correct "
"form.".format(name))
if class_metrics:
class_predictions = self.logits_to_predictions(logits, proba=False)
result.update(_run_metrics(class_predictions, targets,
class_metrics,
self.get_weight_tensor(features)))
if proba_metrics:
predictions = self.logits_to_predictions(logits, proba=True)
result.update(_run_metrics(predictions, targets, proba_metrics,
self.get_weight_tensor(features)))
return result
def _evaluate_model(self, input_fn, hooks=None, checkpoint_path=None, name=''):
# Check that model has been trained (if nothing has been set explicitly).
if not checkpoint_path:
latest_path = saver.latest_checkpoint(self._model_dir)
if not latest_path:
error_message = "Could not find trained model at {}.".format(self._model_dir)
raise EstimatorNotTrainedError(error_message)
checkpoint_path = latest_path
# Setup output directory.
eval_dir = os.path.join(self._model_dir, 'eval' if not name else 'eval_' + name)
with ops.Graph().as_default() as g:
random_seed.set_random_seed(self._config.tf_random_seed)
global_step = training.create_global_step(g)
features, labels = input_fn()
estimator_spec = self._call_model_fn(features, labels, Modes.EVAL)
if MetricKeys.LOSS in estimator_spec.eval_metric_ops:
raise ValueError("Metric with name `{}` is not allowed, because Estimator "
"already defines a default metric "
"with the same name.".format(MetricKeys.LOSS))
estimator_spec.eval_metric_ops[
MetricKeys.LOSS] = metrics_lib.streaming_mean(estimator_spec.loss)
update_op, eval_dict = self._extract_metric_update_ops(estimator_spec.eval_metric_ops)
if ops.GraphKeys.GLOBAL_STEP in eval_dict:
raise ValueError("Metric with name `global_step` is not allowed, because "
"Estimator already defines a default metric with the same name.")
eval_dict[ops.GraphKeys.GLOBAL_STEP] = global_step
eval_results = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
master=self._config.evaluation_master,
scaffold=estimator_spec.scaffold,
eval_ops=update_op,
final_ops=eval_dict,
hooks=hooks,
config=self._session_config)
self._write_dict_to_summary(
output_dir=eval_dir,
dictionary=eval_results,
current_global_step=eval_results[ops.GraphKeys.GLOBAL_STEP])
return eval_results
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {"loss": metrics_lib.streaming_mean(self._loss(
logits, targets,
weight_tensor=self._get_weight_tensor(features)))}
# Adding default metrics
if metrics is None:
metrics = {"accuracy": metrics_lib.streaming_accuracy}
if self._n_classes == 2:
predictions = math_ops.sigmoid(logits)
result["eval_auc"] = metrics_lib.streaming_auc(predictions, targets)
if metrics:
predictions = self._logits_to_predictions(logits, proba=False)
result.update(self._run_metrics(predictions, targets, metrics,
self._get_weight_tensor(features)))
return result
def _get_eval_ops(self, features, labels, metrics):
"""Method that builds model graph and returns evaluation ops.
Expected to be overriden by sub-classes that require custom support.
This implementation uses `model_fn` passed as parameter to constructor to
build model.
Args:
features: `Tensor` or `dict` of `Tensor` objects.
labels: `Tensor` or `dict` of `Tensor` objects.
metrics: Dict of metrics to run. If None, the default metric functions
are used; if {}, no metrics are used. Otherwise, `metrics` should map
friendly names for the metric to a `MetricSpec` object defining which
model outputs to evaluate against which labels with which metric
function. Metric ops should support streaming, e.g., returning
update_op and value tensors. See more details in
`../../../../metrics/python/metrics/ops/streaming_metrics.py` and
`../metric_spec.py`.
Returns:
metrics: `dict` of `Tensor` objects.
Raises:
ValueError: if `metrics` don't match `labels`.
"""
model_fn_ops = self._call_model_fn(features, labels, ModeKeys.EVAL)
all_metrics = model_fn_ops.default_metrics
# Custom metrics should overwrite defaults.
if metrics:
all_metrics.update(metrics)
result = _make_metrics_ops(all_metrics, features, labels,
model_fn_ops.predictions)
if metric_key.MetricKey.LOSS not in result:
result[metric_key.MetricKey.LOSS] = metrics_lib.streaming_mean(
model_fn_ops.loss)
return result
def _get_eval_ops(self, features, labels, metrics):
"""Method that builds model graph and returns evaluation ops.
Expected to be overriden by sub-classes that require custom support.
This implementation uses `model_fn` passed as parameter to constructor to
build model.
Args:
features: `Tensor` or `dict` of `Tensor` objects.
labels: `Tensor` or `dict` of `Tensor` objects.
metrics: Dict of metrics to run. If None, the default metric functions
are used; if {}, no metrics are used. Otherwise, `metrics` should map
friendly names for the metric to a `MetricSpec` object defining which
model outputs to evaluate against which labels with which metric
function. Metric ops should support streaming, e.g., returning
update_op and value tensors. See more details in
`../../../../metrics/python/metrics/ops/streaming_metrics.py` and
`../metric_spec.py`.
Returns:
`ModelFnOps` object.
Raises:
ValueError: if `metrics` don't match `labels`.
"""
model_fn_ops = self._call_model_fn(
features, labels, model_fn_lib.ModeKeys.EVAL)
# Custom metrics should overwrite defaults.
if metrics:
model_fn_ops.eval_metric_ops.update(_make_metrics_ops(
metrics, features, labels, model_fn_ops.predictions))
if metric_key.MetricKey.LOSS not in model_fn_ops.eval_metric_ops:
model_fn_ops.eval_metric_ops[metric_key.MetricKey.LOSS] = (
metrics_lib.streaming_mean(model_fn_ops.loss))
return model_fn_ops
def _predictions_streaming_mean(predictions, unused_labels): return metrics_lib.streaming_mean(predictions)
def _predictions_streaming_mean(predictions, unused_labels):
return metrics_lib.streaming_mean(predictions)
def _target_streaming_mean(unused_predictions, target, weights=None):
return metrics_lib.streaming_mean(target, weights=weights)
def _labels_streaming_mean(unused_predictions, labels, weights=None): return metrics_lib.streaming_mean(labels, weights=weights)
def _targets_streaming_mean(unused_predictions, targets):
return metrics_lib.streaming_mean(targets)
def _get_eval_ops(self, features, targets, metrics=None):
logits = self._model.build_model(features,
self._feature_columns,
is_training=False)
loss = self._target_column.loss(logits, targets, features)
return {'loss': metrics_lib.streaming_mean(loss)}
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir,
config.dataset,
config.datasubset,
num_folds=config.fold_count,
fold=config.fold,
holdout=True)
init_op, init_feed_dict, image = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images = tf.train.batch([image],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for loss in tf.losses.get_losses():
metrics_map[loss.op.name] = metrics.streaming_mean(loss)
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
total_loss = tf.losses.get_total_loss()
metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
images = input_utils.reshape_images(images, config.dataset)
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
# Generate the canvases that lead to the final output image
with tf.name_scope('canvases'):
for step, canvas in enumerate(outputs):
canvas = input_utils.reshape_images(canvas, config.dataset)
tiled_images = image_utils.tile_images(canvas)
summaries.append(
tf.summary.image('step{0}'.format(step), tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
hooks = [
training.SummaryAtEndHook(log_dir=FLAGS.log_dir,
summary_op=summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def __init__(self, run_context=None):
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard')
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images')
self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels')
self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(self.global_step,
self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(sigmoid_rampup_value,
self.hyper['max_consistency_cost'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length'])
self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
(
(self.class_logits_1, self.cons_logits_1),
(self.class_logits_2, self.cons_logits_2),
(self.class_logits_ema, self.cons_logits_ema)
) = inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
student_dropout_probability=self.hyper['student_dropout_probability'],
teacher_dropout_probability=self.hyper['teacher_dropout_probability'],
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'],
translate=self.hyper['translate'],
num_logits=self.hyper['num_logits'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels)
self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels)
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency)
self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs(
self.cons_logits_1, self.class_logits_2, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'])
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'])
def l2_norms(matrix):
l2s = tf.reduce_sum(matrix ** 2, axis=1)
mean_l2 = tf.reduce_mean(l2s)
return mean_l2, l2s
self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1)
self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema)
self.res_costs_1 = self.hyper['logit_distance_cost'] * self.res_l2s_1
self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1)
self.res_costs_ema = self.hyper['logit_distance_cost'] * self.res_l2s_ema
self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema)
self.mean_total_cost_pi, self.total_costs_pi = total_costs(
self.class_costs_1, self.cons_costs_pi, self.res_costs_1)
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt, self.res_costs_1)
assert_shape(self.total_costs_pi, [3])
assert_shape(self.total_costs_mt, [3])
self.cost_to_be_minimized = tf.cond(self.hyper['ema_consistency'],
lambda: self.mean_total_cost_mt,
lambda: self.mean_total_cost_pi)
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/pi": self.mean_cons_cost_pi,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/res_cost/1": self.mean_res_cost_1,
"train/res_cost/ema": self.mean_res_cost_ema,
"train/total_cost/pi": self.mean_total_cost_pi,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema": streaming_mean(self.class_costs_ema),
"eval/res_cost/1": streaming_mean(self.res_costs_1),
"eval/res_cost/ema": streaming_mean(self.res_costs_ema),
})
metric_variables = slim.get_local_variables(scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables() if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def _predictions_streaming_mean(
predictions, labels, weights=None, class_id=None):
del labels
if class_id is not None:
predictions = predictions[:, class_id]
return metrics_lib.streaming_mean(predictions, weights=weights)
def _indicator_labels_streaming_mean(
predictions, labels, weights=None, class_id=None):
del predictions
if class_id is not None:
labels = labels[:, class_id]
return metrics_lib.streaming_mean(labels, weights=weights)
def create_metric_ops(self, _inputs, labels, predictions):
"""Creates the metric op"""
loss_mask = tf.sequence_mask(
lengths=tf.to_int32(labels["target_len"] - 1),
maxlen=tf.to_int32(tf.shape(predictions["losses"])[1]))
return metrics.streaming_mean(predictions["losses"], loss_mask)
def _streaming_weighted_average_loss(predictions, target, weights=None):
loss_unweighted = loss_fn(predictions, target)
_, weighted_average_loss = _loss(loss_unweighted,
weights,
name="eval_loss")
return metrics_lib.streaming_mean(weighted_average_loss)
def _get_eval_ops(self, features, targets, metrics=None):
logits = self._model.build_model(
features, self._feature_columns, is_training=False)
loss = self._target_column.loss(logits, targets, features)
return {'loss': metrics_lib.streaming_mean(loss)}
def model_fn(features, labels, mode, params):
is_training = mode == learn.ModeKeys.TRAIN
optimizer_class = find_class_by_name(params.optimizer, [tf.train])
label_loss_fn = find_class_by_name(params.label_loss, [losses])()
model = find_class_by_name(params.model,
[frame_level_models, video_level_models])()
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.train.exponential_decay(
params.base_learning_rate,
global_step * params.batch_size * params.num_towers,
params.learning_rate_decay_examples,
params.learning_rate_decay,
staircase=True,
)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
tf.summary.histogram("model/input_raw", features['model_input'])
feature_dim = len(features['model_input'].get_shape()) - 1
model_input = tf.nn.l2_normalize(features['model_input'], feature_dim)
tower_inputs = tf.split(model_input, params.num_towers)
if mode == learn.ModeKeys.INFER:
# ***
# this is a quick hack so that the existing model_fn code,
# taken from train.py, doesn't break in inference (or serving) mode.
# Normally, we would write model_fn such that the 'labels' input arg
# can be None in inference mode, but this existing model code was not written this
# way. See the serving_input_fn() defined below, to see where 'labels_batch'
# is added to the features dict, just to make this code work properly
labels = features['labels_batch']
tower_labels = tf.split(labels, params.num_towers)
tower_num_frames = tf.split(features['num_frames'], params.num_towers)
tower_gradients = []
tower_predictions = []
tower_label_losses = []
tower_reg_losses = []
for i in range(params.num_towers):
# For some reason these 'with' statements can't be combined onto the same
# line. They have to be nested.
with tf.device(params.device_string % i):
with (tf.variable_scope(("tower"), reuse=True if i > 0 else None)):
with (slim.arg_scope([slim.model_variable, slim.variable],
device="/cpu:0"
if params.num_gpus != 1 else "/gpu:0")):
result = model.create_model(
tower_inputs[i],
num_frames=tower_num_frames[i],
vocab_size=params.reader.num_classes,
labels=tower_labels[i],
is_training=is_training)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
tower_predictions.append(predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(
predictions, tower_labels[i])
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
tower_reg_losses.append(reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
tower_label_losses.append(label_loss)
final_loss = params.regularization_penalty * reg_loss + label_loss
gradients = optimizer.compute_gradients(
final_loss, colocate_gradients_with_ops=False)
tower_gradients.append(gradients)
pred_dict = {}
label_loss = tf.reduce_mean(tf.stack(tower_label_losses))
predictions = tf.concat(tower_predictions, 0)
pred_dict['predictions'] = predictions
tf.summary.scalar("label_loss", label_loss)
if params.regularization_penalty != 0:
reg_loss = tf.reduce_mean(tf.stack(tower_reg_losses))
tf.summary.scalar("reg_loss", reg_loss)
if is_training:
# Incorporate the L2 weight penalties, etc.
merged_gradients = utils.combine_gradients(tower_gradients)
if params.clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
merged_gradients = utils.clip_gradient_norms(
merged_gradients, params.clip_gradient_norm)
train_op = optimizer.apply_gradients(merged_gradients,
global_step=global_step)
else:
train_op = None
eval_metric_ops = {}
if mode == learn.ModeKeys.EVAL or is_training:
eval_metric_ops['hit_at_one'] = metrics.streaming_mean(
tf.py_func(
lambda x, y: np.float32(eval_util.calculate_hit_at_one(x, y)),
[predictions, labels],
tf.float32,
stateful=False,
))
eval_metric_ops['perr'] = metrics.streaming_mean(
tf.py_func(
lambda x, y: np.float32(
eval_util.calculate_precision_at_equal_recall_rate(x, y)),
[predictions, labels],
tf.float32,
stateful=False,
))
eval_metric_ops['gap'] = metrics.streaming_mean(
tf.py_func(
lambda x, y: np.float32(eval_util.calculate_gap(x, y)),
[predictions, labels],
tf.float32,
stateful=False,
))
else:
pass
top_predictions, top_indices = tf.nn.top_k(predictions,
_TOP_PREDICTIONS_IN_OUTPUT)
pred_dict['top_predictions'] = top_predictions
pred_dict['top_indices'] = top_indices
#add eval summaries and update ops for training
for key, val in eval_metric_ops.items():
tf.summary.scalar(key, val[0]) #create summary for each eval op
tf.add_to_collection(
tf.GraphKeys.UPDATE_OPS, val[1]
) # add the update op for each eval up to update ops collection, so that it will be run every train_op call
# tf.add_to_collection("global_step", global_step)
# tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", tf.concat(tower_predictions, 0))
# tf.add_to_collection("input_batch_raw", model_input_raw)
# tf.add_to_collection("input_batch", model_input)
# tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels, tf.float32))
# tf.add_to_collection("train_op", train_op)
tf.summary.scalar("loss", label_loss)
export_outputs = {
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
tf.estimator.export.PredictOutput(pred_dict)
}
return tf.estimator.EstimatorSpec(mode=mode,
predictions=pred_dict,
loss=label_loss,
train_op=train_op,
export_outputs=export_outputs,
eval_metric_ops=eval_metric_ops)
def _maybe_add_streaming_mean(result, key, value):
if key in result:
logging.warning('Metrics already contains %s, skipping.', key)
return
result[key] = metrics_lib.streaming_mean(value)
def perplexity_metric(losses, target_len):
"""Calculates the mean log perplexity based on losses and target_len"""
loss_mask = tf.sequence_mask(lengths=tf.to_int32(target_len - 1),
maxlen=tf.to_int32(tf.shape(losses)[1]))
return metrics.streaming_mean(losses, loss_mask)
def __init__(self, result_dir):
self.checkpoint_dir = os.path.join(result_dir, 'checkpoints')
self.summary_dir = os.path.join(result_dir, 'summaries')
os.makedirs(self.checkpoint_dir)
os.makedirs(self.summary_dir)
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32,
shape=(None, 32, 32, 3),
name='images')
self.labels = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name='labels')
self.is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
sigmoid_rampup_value = sigmoid_rampup(self.global_step,
self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(
self.global_step, self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value *
sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(
sigmoid_rampdown_value *
self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) *
self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(
sigmoid_rampup_value,
self.hyper['max_consistency_coefficient'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step,
self.hyper['rampup_length'])
self.adam_beta_2 = tf.add(
(1 - step_rampup_value) *
self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add(
(1 - step_rampup_value) *
self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
self.logits_1, self.logits_2, self.logits_ema = inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.logits_1,
self.labels)
self.mean_error_ema, self.errors_ema = errors(
self.logits_ema, self.labels)
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1),
labeled_consistency)
self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs(
self.logits_1, self.logits_2, self.cons_coefficient,
consistency_mask)
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.logits_1, self.logits_ema, self.cons_coefficient,
consistency_mask)
self.mean_total_cost_pi, self.total_costs_pi = total_costs(
self.class_costs_1, self.cons_costs_pi)
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt)
self.cost_to_be_minimized = tf.cond(
self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt,
lambda: self.mean_total_cost_pi)
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = nn.adam_optimizer(
self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/pi": self.mean_cons_cost_pi,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/pi": self.mean_total_cost_pi,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map(
{
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
# Note that the evaluation costs are not directly comparable
# to the training costs. Training batches contain unlabeled
# samples but the evaluation batches do not. Because
# classification cost is zero for unlabeled samples, the
# training costs are smaller than evaluation costs when
# doing semi-supervised learning.
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema":
streaming_mean(self.class_costs_ema),
"eval/cons_cost/pi": streaming_mean(self.cons_costs_pi),
"eval/cons_cost/mt": streaming_mean(self.cons_costs_mt),
"eval/total_cost/pi": streaming_mean(self.total_costs_pi),
"eval/total_cost/mt": streaming_mean(self.total_costs_mt)
})
metric_variables = slim.get_local_variables(
scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables()
if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def streaming_mean(name, value):
tf.summary.scalar(
name,
tfm.streaming_mean(value, name='stream/{}'.format(name))[1])
def _compute_loss(logits, targets, weights=None):
return metrics_lib.streaming_mean(self._loss(
logits, targets, weight_tensor=weights))
def create_metric_ops(self, _inputs, labels, predictions):
"""Creates the metric op"""
loss_mask = tf.sequence_mask(
lengths=tf.to_int32(labels["target_len"] - 1),
maxlen=tf.to_int32(tf.shape(predictions["losses"])[1]))
return metrics.streaming_mean(predictions["losses"], loss_mask)
def _streaming_weighted_average_loss(predictions, target, weights=None):
loss_unweighted = loss_fn(predictions, target)
_, weighted_average_loss = _loss(loss_unweighted,
weights,
name="eval_loss")
return metrics_lib.streaming_mean(weighted_average_loss)
def _get_eval_ops(self, features, targets, metrics=None):
"""See base class."""
logits = self._logits(features)
result = {
"loss":
metrics_lib.streaming_mean(self._loss(logits, targets, features))
}
# Adds default metrics.
if metrics is None:
# TODO(b/29366811): This currently results in both an "accuracy" and an
# "accuracy/threshold_0.500000_mean" metric for binary classification.
metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy}
# Adds additional useful metrics for the special case of binary
# classification.
# TODO(zakaria): Move LogisticRegressor.get_default_metrics to metrics
# and handle eval metric from targetcolumn.
if self._target_column.num_label_columns == 1:
predictions = math_ops.sigmoid(logits)
targets_float = math_ops.to_float(targets)
default_metrics = (
logistic_regressor.LogisticRegressor.get_default_metrics())
for metric_name, metric_op in default_metrics.items():
result[metric_name] = metric_op(predictions, targets_float)
if metrics:
class_metrics = {}
proba_metrics = {}
for name, metric_op in six.iteritems(metrics):
if isinstance(name, tuple):
if len(name) != 2:
raise ValueError(
"Ignoring metric {}. It returned a tuple with "
"len {}, expected 2.".format(name, len(name)))
else:
if name[1] not in ["classes", "probabilities"]:
raise ValueError(
"Ignoring metric {}. The 2nd element of its "
"name should be either 'classes' or "
"'probabilities'.".format(name))
elif name[1] == "classes":
class_metrics[name[0]] = metric_op
else:
proba_metrics[name[0]] = metric_op
elif isinstance(name, str):
class_metrics[name] = metric_op
else:
raise ValueError(
"Ignoring metric {}. Its name is not in the correct "
"form.".format(name))
if class_metrics:
predictions = self._target_column.logits_to_predictions(
logits, proba=False)
result.update(
self._run_metrics(
predictions, targets, class_metrics,
self._target_column.get_weight_tensor(features)))
if proba_metrics:
predictions = self._target_column.logits_to_predictions(
logits, proba=True)
result.update(
self._run_metrics(
predictions, targets, proba_metrics,
self._target_column.get_weight_tensor(features)))
return result
def _labels_streaming_mean(unused_predictions, labels):
return metrics_lib.streaming_mean(labels)
def _predictions_streaming_mean(predictions, unused_target, weights=None): return metrics_lib.streaming_mean(predictions, weights=weights)
def _labels_streaming_mean(unused_predictions, labels): return metrics_lib.streaming_mean(labels)
import tensorflow as tf
import tensorflow.contrib.metrics as tcm
values = tf.random_uniform(shape=[2], minval=1, maxval=5, dtype=tf.int32)
# 1.Initialization: initializing the metric state.
# Calling streaming_mean creates a pair of state variables that will contain：
# (1) the running sum and (2) the count of the number of samples in the sum.
# tf.local_variables_initializer() sets the sum and count variables to zero.
mean_value, update_op = tcm.streaming_mean(values, weights=[0.2, 0.8])
with tf.Session() as sess:
# The streaming metrics use local variables.
sess.run(tf.local_variables_initializer())
for i in range(10):
# 2.Aggregation: updating the values of the metric state.
# Aggregation is performed by examining the current state of values
# and incrementing the state variables appropriately.
mean_value_result, value_result = sess.run([update_op, values])
print("mean after {}: {} {}".format(i, value_result,
mean_value_result))
# 3.Finalization: computing the final metric value.
final_mean_value_result = mean_value.eval()
print("final mean: {}".format(final_mean_value_result))
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir, config.dataset,
config.datasubset)
init_op, init_feed_dict, image, label = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images, labels = tf.train.batch(
[image, label],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
tfprof.model_analyzer.print_model_analysis(tf.get_default_graph())
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
predictions = tf.argmax(outputs, 1)
metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels)
metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k(
outputs, tf.expand_dims(labels, 1), 5)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
scaffold = tf.train.Scaffold(init_op, init_feed_dict)
hooks = [
training.SummaryAtEndHook(FLAGS.log_dir, summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
# On Tensorflow master fd87896 fixes this, but for now just set a very large number
eval_kwargs['max_number_of_evaluations'] = sys.maxint
eval_fn(checkpoint_path,
scaffold=scaffold,
hooks=hooks,
eval_ops=eval_ops,
**eval_kwargs)
def _targets_streaming_mean(unused_predictions, targets): return metrics_lib.streaming_mean(targets)
def __init__(self, run_context=None, hyper_dict={}):
#inilization of hyper
for i in hyper_dict:
assert i in self.hyper, "Wrong hyper dict '{}'!".format(i)
self.hyper[i] = hyper_dict[i]
# inilize bg noise input
if self.hyper['bg_noise']:
self.bg_noise_input = tf.convert_to_tensor(self.hyper['bg_noise_input'],dtype=tf.float32)
else:
self.bg_noise_input = tf.convert_to_tensor(np.zeros((32,32)),dtype=tf.float32)
# inilization model
print('{} is initliazed!'.format(self.hyper['cnn']))
self.cnn = getattr(model,self.hyper['cnn'])
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard')
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32, shape=(None,) + self.hyper['input_dim'], name='images')
self.labels = tf.placeholder(dtype=tf.int32, shape=(None,) + self.hyper['label_dim'], name='labels')
self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
with tf.name_scope("ramps"):
self.learning_rate, self.cons_coefficient, \
self.adam_beta_1, self.adam_beta_2, \
self.ema_decay = ramp_value(self.global_step,self.hyper)
( self.class_logits_1,
self.class_logits_ema
) = self.inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
student_dropout_probability=self.hyper['student_dropout_probability'],
teacher_dropout_probability=self.hyper['teacher_dropout_probability'],
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'],
translate=self.hyper['translate'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels,sig = self.hyper['sig'])
self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels,sig = self.hyper['sig'])
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels,sig = self.hyper['sig'])
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels,sig = self.hyper['sig'])
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency)
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.class_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'],sig = 'softmax')
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt)
self.cost_to_be_minimized = self.mean_total_cost_mt
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
if self.hyper['optimizer']=='adam':
self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
elif self.hyper['optimizer']=='sgd':
self.train_step_op = nn.sgd_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.hyper['max_learning_rate'])
else:
assert False, 'Wrong optimizer input!'
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema": streaming_mean(self.class_costs_ema),
})
metric_variables = slim.get_local_variables(scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables() if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.session = tf.Session(config=config)
self.run(self.init_init_op)
self.save_tensorboard_graph()
def __init__(self, run_context=None):
self.name = "Tweet Data Class"
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard')
with tf.name_scope("placeholders"):
self.tweets = tf.placeholder(dtype=tf.float32, shape=(None, 500), name='tweets')
self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels')
self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
# Ramp-up and ramp-down has been removed for simplicity
# self.learning_rate = tf.constant(self.hyper['max_learning_rate'], dtype = tf.float32)
# self.adam_beta_1 = tf.constant(self.hyper['adam_beta_1_after_rampdown'], dtype = tf.float32)
# self.cons_coefficient = tf.constant(self.hyper['max_consistency_cost'], dtype = tf.float32)
# self.adam_beta_2 = tf.constant(self.hyper['adam_beta_2_after_rampup'], dtype = tf.float32)
# self.ema_decay = tf.constant(self.hyper['ema_decay_after_rampup'], dtype = tf.float32)
# self.learning_rate =self.DEFAULT_HYPERPARAMS['max_learning_rate']
# self.adam_beta_1 = self.DEFAULT_HYPERPARAMS['adam_beta_1_after_rampdown']
# self.cons_coefficient = self.DEFAULT_HYPERPARAMS['max_consistency_cost']
# self.adam_beta_2 = self.DEFAULT_HYPERPARAMS['adam_beta_2_after_rampup']
# self.ema_decay = self.DEFAULT_HYPERPARAMS['ema_decay_after_rampup']
sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(self.global_step,
self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(sigmoid_rampup_value,
self.hyper['max_consistency_cost'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length'])
self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
# below is where the interesting stuff happens, mostly.
# Inference is a function which creates the towers and sets up the different logits for the two models
(
(self.class_logits_1, self.cons_logits_1),
(self.class_logits_2, self.cons_logits_2),
(self.class_logits_ema, self.cons_logits_ema)
) = inference(
self.tweets,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
hidden_dims = self.DEFAULT_HYPERPARAMS['hidden_dims'],
student_dropout_probability=self.hyper['student_dropout_probability'],
teacher_dropout_probability=self.hyper['teacher_dropout_probability'],
num_logits=self.hyper['num_logits'])
with tf.name_scope("objectives"):
# something weird is done with errors for unlabeled examples.
# I think errors are only calculated for labeled, but you don't calculate it for unlabeled, so it is NaN for unlabeled
self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels)
self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels)
# where we calculate classification costs.
# the cost_1 should be for student and ema is for teacher
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency)
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask)
def l2_norms(matrix):
l2s = tf.reduce_sum(matrix ** 2, axis=1)
mean_l2 = tf.reduce_mean(l2s)
return mean_l2, l2s
self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1)
self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema)
# mean total cost is what you are optimizng.
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt)
assert_shape(self.total_costs_mt, [2])
self.cost_to_be_minimized = self.mean_total_cost_mt
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = adam_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
# TODO do we really need this?
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
# NOTE these should not need training, since we don't do ramp-up and ramp-down
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/mt": self.mean_total_cost_mt,
}
# TODO not sure what streaming mean does?
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema": streaming_mean(self.class_costs_ema),
})
metric_variables = slim.get_local_variables(scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
# TODO string utils just formats dictionary results in a nice way for logging, not needed?
self.result_formatter = string_utils.DictFormatter(
order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables() if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
print("Train init variables:")
for var in train_init_variables:
print(var)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def __init__(self, run_context=None):
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir,
'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir,
'tensorboard')
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32,
shape=(None, 32, 32, 3),
name='images')
self.labels = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name='labels')
self.is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
sigmoid_rampup_value = sigmoid_rampup(self.global_step,
self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(
self.global_step, self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value *
sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(
sigmoid_rampdown_value *
self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) *
self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(
sigmoid_rampup_value,
self.hyper['max_consistency_cost'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step,
self.hyper['rampup_length'])
self.adam_beta_2 = tf.add(
(1 - step_rampup_value) *
self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add(
(1 - step_rampup_value) *
self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
((self.class_logits_1, self.cons_logits_1), (self.class_logits_2,
self.cons_logits_2),
(self.class_logits_ema, self.cons_logits_ema)) = inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
student_dropout_probability=self.
hyper['student_dropout_probability'],
teacher_dropout_probability=self.
hyper['teacher_dropout_probability'],
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'],
translate=self.hyper['translate'],
num_logits=self.hyper['num_logits'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.class_logits_1,
self.labels)
self.mean_error_ema, self.errors_ema = errors(
self.class_logits_ema, self.labels)
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1),
labeled_consistency)
self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs(
self.cons_logits_1, self.class_logits_2, self.cons_coefficient,
consistency_mask, self.hyper['consistency_trust'])
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.cons_logits_1, self.class_logits_ema,
self.cons_coefficient, consistency_mask,
self.hyper['consistency_trust'])
def l2_norms(matrix):
l2s = tf.reduce_sum(matrix**2, axis=1)
mean_l2 = tf.reduce_mean(l2s)
return mean_l2, l2s
self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 -
self.cons_logits_1)
self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(
self.class_logits_ema - self.cons_logits_ema)
self.res_costs_1 = self.hyper[
'logit_distance_cost'] * self.res_l2s_1
self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1)
self.res_costs_ema = self.hyper[
'logit_distance_cost'] * self.res_l2s_ema
self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema)
self.mean_total_cost_pi, self.total_costs_pi = total_costs(
self.class_costs_1, self.cons_costs_pi, self.res_costs_1)
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt, self.res_costs_1)
assert_shape(self.total_costs_pi, [3])
assert_shape(self.total_costs_mt, [3])
self.cost_to_be_minimized = tf.cond(
self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt,
lambda: self.mean_total_cost_pi)
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = nn.adam_optimizer(
self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/pi": self.mean_cons_cost_pi,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/res_cost/1": self.mean_res_cost_1,
"train/res_cost/ema": self.mean_res_cost_ema,
"train/total_cost/pi": self.mean_total_cost_pi,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map(
{
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema":
streaming_mean(self.class_costs_ema),
"eval/res_cost/1": streaming_mean(self.res_costs_1),
"eval/res_cost/ema": streaming_mean(self.res_costs_ema),
})
metric_variables = slim.get_local_variables(
scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=[
"eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"
],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables()
if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def model_fn(features, mode, config, embedding_matrix, vocab_tables):
if mode == tf.estimator.ModeKeys.PREDICT:
base_words, src_words, tgt_words, inserted_words, deleted_words = features
else:
src_words, tgt_words, inserted_words, deleted_words = features
base_words = src_words
if mode != tf.estimator.ModeKeys.TRAIN:
config.put('editor.enable_dropout', False)
config.put('editor.dropout_keep', 1.0)
vocab_s2i = vocab_tables[vocab.STR_TO_INT]
vocab_i2s = vocab_tables[vocab.INT_TO_STR]
vocab.init_embeddings(embedding_matrix)
train_decoder_output, infer_decoder_output, \
gold_dec_out, gold_dec_out_len = editor.editor_train(
base_words, src_words, tgt_words, inserted_words, deleted_words, embedding_matrix, vocab_s2i,
config.editor.hidden_dim, config.editor.agenda_dim, config.editor.edit_dim,
config.editor.encoder_layers, config.editor.decoder_layers, config.editor.attention_dim,
config.editor.beam_width,
config.editor.max_sent_length, config.editor.dropout_keep, config.editor.lamb_reg,
config.editor.norm_eps, config.editor.norm_max, config.editor.kill_edit,
config.editor.draw_edit, config.editor.use_swap_memory,
config.get('editor.use_beam_decoder', False), config.get('editor.enable_dropout', False),
config.get('editor.no_insert_delete_attn', False), config.get('editor.enable_vae', True)
)
loss = optimizer.loss(train_decoder_output, gold_dec_out, gold_dec_out_len)
train_op, gradients_norm = optimizer.train(
loss, config.optim.learning_rate, config.optim.max_norm_observe_steps)
if mode == tf.estimator.ModeKeys.TRAIN:
tf.summary.scalar('grad_norm', gradients_norm)
ops = add_extra_summary(config, vocab_i2s, train_decoder_output,
src_words, tgt_words, inserted_words,
deleted_words, ['extra'])
hooks = [
get_train_extra_summary_writer(config),
get_extra_summary_logger(ops, config),
]
if config.get('logger.enable_profiler', False):
hooks.append(get_profiler_hook(config))
return tf.estimator.EstimatorSpec(mode,
train_op=train_op,
loss=loss,
training_hooks=hooks)
elif mode == tf.estimator.ModeKeys.EVAL:
ops = add_extra_summary(config, vocab_i2s, train_decoder_output,
src_words, tgt_words, inserted_words,
deleted_words, ['extra'])
return tf.estimator.EstimatorSpec(
mode,
loss=loss,
evaluation_hooks=[get_extra_summary_logger(ops, config)],
eval_metric_ops={'bleu': tf_metrics.streaming_mean(ops[ES_BLEU])})
elif mode == tf.estimator.ModeKeys.PREDICT:
lengths = decoder.seq_length(infer_decoder_output)
tokens = decoder.str_tokens(infer_decoder_output, vocab_i2s)
preds = {
'str_tokens': tokens,
'sample_id': decoder.sample_id(infer_decoder_output),
'lengths': lengths,
'joined': metrics.join_tokens(tokens, lengths),
}
return tf.estimator.EstimatorSpec(mode, predictions=preds)
def _target_streaming_mean(unused_predictions, target, weights=None): return metrics_lib.streaming_mean(target, weights=weights)
def _get_eval_ops(self, features, targets, metrics):
"""Method that builds model graph and returns evaluation ops.
Expected to be overriden by sub-classes that require custom support.
This implementation uses `model_fn` passed as parameter to constructor to
build model.
Args:
features: `Tensor` or `dict` of `Tensor` objects.
targets: `Tensor` or `dict` of `Tensor` objects.
metrics: Dict of metric ops to run. If None, the default metric functions
are used; if {}, no metrics are used. If model has one output (i.e.,
returning single predction), keys are `str`, e.g. `'accuracy'` - just a
name of the metric that will show up in the logs / summaries.
Otherwise, keys are tuple of two `str`, e.g. `('accuracy', 'classes')`
- name of the metric and name of `Tensor` in the predictions to run
this metric on. Metric ops should support streaming, e.g., returning
update_op and value tensors. See more details in
../../../../metrics/python/metrics/ops/streaming_metrics.py.
Returns:
metrics: `dict` of `Tensor` objects.
Raises:
ValueError: if `metrics` don't match `targets`.
"""
predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL)
result = {'loss': metrics_lib.streaming_mean(loss)}
weights = self._get_weight_tensor(features)
metrics = metrics or {}
if isinstance(targets, dict) and len(targets) == 1:
# Unpack single target into just tensor.
targets = targets[list(targets.keys())[0]]
for name, metric in six.iteritems(metrics):
if isinstance(name, tuple):
# Multi-head metrics.
if not isinstance(predictions, dict):
raise ValueError(
'Metrics passed provide (name, prediction), '
'but predictions are not dict. '
'Metrics: %s, Predictions: %s.' % (metrics, predictions))
# Here are two options: targets are single Tensor or a dict.
if isinstance(targets, dict) and name[1] in targets:
# If targets are dict and the prediction name is in it, apply metric.
result[name[0]] = metrics_lib.run_metric(
metric, predictions[name[1]], targets[name[1]], weights)
else:
# Otherwise pass the targets to the metric.
result[name[0]] = metrics_lib.run_metric(
metric, predictions[name[1]], targets, weights)
else:
# Single head metrics.
if isinstance(predictions, dict):
raise ValueError(
'Metrics passed provide only name, no prediction, '
'but predictions are dict. '
'Metrics: %s, Targets: %s.' % (metrics, targets))
result[name] = metrics_lib.run_metric(
metric, predictions, targets, weights)
return result
def model_fn(features, mode, config, embedding_matrix, vocab_tables):
if mode == tf.estimator.ModeKeys.PREDICT:
base_words, extended_base_words, \
_, _, \
src_words, tgt_words, \
inserted_words, deleted_words, \
oov = features
output_words = extended_output_words = tgt_words
else:
base_words, extended_base_words, \
output_words, extended_output_words, \
src_words, tgt_words, \
inserted_words, deleted_words, \
oov = features
if mode != tf.estimator.ModeKeys.TRAIN:
config.put('editor.enable_dropout', False)
config.put('editor.dropout_keep', 1.0)
vocab_i2s = vocab_tables[vocab.INT_TO_STR]
vocab.init_embeddings(embedding_matrix)
vocab_size = len(vocab_tables[vocab.RAW_WORD2ID])
train_decoder_output, infer_decoder_output, \
gold_dec_out, gold_dec_out_len = editor.editor_train(
base_words, extended_base_words, output_words, extended_output_words,
src_words, tgt_words, inserted_words, deleted_words, oov,
vocab_size,
config.editor.hidden_dim, config.editor.agenda_dim, config.editor.edit_dim,
config.editor.edit_enc.micro_ev_dim, config.editor.edit_enc.num_heads,
config.editor.encoder_layers, config.editor.decoder_layers, config.editor.attention_dim,
config.editor.beam_width,
config.editor.edit_enc.ctx_hidden_dim, config.editor.edit_enc.ctx_hidden_layer,
config.editor.edit_enc.wa_hidden_dim, config.editor.edit_enc.wa_hidden_layer,
config.editor.edit_enc.meve_hidden_dim, config.editor.edit_enc.meve_hidden_layer,
config.editor.max_sent_length, config.editor.dropout_keep, config.editor.lamb_reg,
config.editor.norm_eps, config.editor.norm_max, config.editor.kill_edit,
config.editor.draw_edit, config.editor.use_swap_memory,
config.get('editor.use_beam_decoder', False), config.get('editor.enable_dropout', False),
config.get('editor.no_insert_delete_attn', False), config.get('editor.enable_vae', True)
)
loss = optimizer.loss(train_decoder_output, gold_dec_out, gold_dec_out_len)
train_op, gradients_norm = optimizer.train(
loss, config.optim.learning_rate, config.optim.max_norm_observe_steps)
tf.logging.info("Trainable variable")
for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
tf.logging.info(str(i))
if mode == tf.estimator.ModeKeys.TRAIN:
tf.summary.scalar('grad_norm', gradients_norm)
ops = add_extra_summary(config,
vocab_i2s,
train_decoder_output,
base_words,
output_words,
src_words,
tgt_words,
inserted_words,
deleted_words,
oov,
vocab_size,
collections=['extra'])
hooks = [
get_train_extra_summary_writer(config),
get_extra_summary_logger(ops, config),
]
if config.get('logger.enable_profiler', False):
hooks.append(get_profiler_hook(config))
return tf.estimator.EstimatorSpec(mode,
train_op=train_op,
loss=loss,
training_hooks=hooks)
elif mode == tf.estimator.ModeKeys.EVAL:
ops = add_extra_summary(config,
vocab_i2s,
train_decoder_output,
base_words,
output_words,
src_words,
tgt_words,
inserted_words,
deleted_words,
oov,
vocab_size,
collections=['extra'])
return tf.estimator.EstimatorSpec(
mode,
loss=loss,
evaluation_hooks=[get_extra_summary_logger(ops, config)],
eval_metric_ops={'bleu': tf_metrics.streaming_mean(ops[ES_BLEU])})
elif mode == tf.estimator.ModeKeys.PREDICT:
lengths = decoder.seq_length(infer_decoder_output)
tokens = decoder.str_tokens(infer_decoder_output, vocab_i2s,
vocab_size, oov)
preds = {
'str_tokens': tokens,
'sample_id': decoder.sample_id(infer_decoder_output),
'lengths': lengths,
'joined': metrics.join_tokens(tokens, lengths),
}
return tf.estimator.EstimatorSpec(mode, predictions=preds)
def _maybe_add_streaming_mean(result, key, value):
if key in result:
logging.warning('Metrics already contains %s, skipping.', key)
return
result[key] = metrics_lib.streaming_mean(value)
def _predictions_streaming_mean(predictions, unused_labels, weights=None): return metrics_lib.streaming_mean(predictions, weights=weights)
def model_fn(features, mode, config, embedding_matrix, vocab_tables):
if mode == tf.estimator.ModeKeys.PREDICT:
base_words, _, src_words, tgt_words, inserted_words, commong_words = features
output_words = tgt_words
else:
base_words, output_words, src_words, tgt_words, inserted_words, commong_words = features
is_training = mode == tf.estimator.ModeKeys.TRAIN
tf.add_to_collection('is_training', is_training)
if mode != tf.estimator.ModeKeys.TRAIN:
config.put('editor.enable_dropout', False)
config.put('editor.dropout_keep', 1.0)
config.put('editor.dropout', 0.0)
config.put('editor.transformer.enable_dropout', False)
config.put('editor.transformer.layer_postprocess_dropout', 0.0)
config.put('editor.transformer.attention_dropout', 0.0)
config.put('editor.transformer.relu_dropout', 0.0)
vocab.init_embeddings(embedding_matrix)
EmbeddingSharedWeights.init_from_embedding_matrix()
editor_model = Editor(config)
logits, beam_prediction = editor_model(base_words, src_words, tgt_words,
inserted_words, commong_words,
output_words)
targets = decoder.prepare_decoder_output(
output_words, sequence.length_pre_embedding(output_words))
target_lengths = sequence.length_pre_embedding(targets)
vocab_size = embedding_matrix.shape[0]
loss, weights = optimizer.padded_cross_entropy_loss(
logits, targets, target_lengths, config.optim.label_smoothing,
vocab_size)
train_op = optimizer.get_train_op(loss, config)
tf.logging.info("Trainable variable")
for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
tf.logging.info(str(i))
tf.logging.info("Num of Trainable parameters")
tf.logging.info(
np.sum([
np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()
]))
if mode == tf.estimator.ModeKeys.TRAIN:
decoded_ids = decoder.logits_to_decoded_ids(logits)
ops = add_extra_summary(config,
decoded_ids,
target_lengths,
base_words,
output_words,
src_words,
tgt_words,
inserted_words,
commong_words,
collections=['extra'])
hooks = [
get_train_extra_summary_writer(config),
get_extra_summary_logger(ops, config),
]
if config.get('logger.enable_profiler', False):
hooks.append(get_profiler_hook(config))
return tf.estimator.EstimatorSpec(mode,
train_op=train_op,
loss=loss,
training_hooks=hooks)
elif mode == tf.estimator.ModeKeys.EVAL:
decoded_ids = decoder.logits_to_decoded_ids(logits)
ops = add_extra_summary(config,
decoded_ids,
target_lengths,
base_words,
output_words,
src_words,
tgt_words,
inserted_words,
commong_words,
collections=['extra'])
return tf.estimator.EstimatorSpec(
mode,
loss=loss,
evaluation_hooks=[get_extra_summary_logger(ops, config)],
eval_metric_ops={'bleu': tf_metrics.streaming_mean(ops[ES_BLEU])})
elif mode == tf.estimator.ModeKeys.PREDICT:
decoded_ids, decoded_lengths, scores = beam_prediction
tokens = decoder.str_tokens(decoded_ids)
preds = {
'str_tokens': tf.transpose(tokens, [0, 2, 1]),
'decoded_ids': tf.transpose(decoded_ids, [0, 2, 1]),
'lengths': decoded_lengths,
'joined': metrics.join_tokens(tokens, decoded_lengths)
}
tmee_attentions = tf.get_collection(
'TransformerMicroEditExtractor_Attentions')
if len(tmee_attentions) > 0:
preds.update({
'tmee_attentions_st_enc_self': tmee_attentions[0][0],
'tmee_attentions_st_dec_self': tmee_attentions[0][1],
'tmee_attentions_st_dec_enc': tmee_attentions[0][2],
'tmee_attentions_ts_enc_self': tmee_attentions[1][0],
'tmee_attentions_ts_dec_self': tmee_attentions[1][1],
'tmee_attentions_ts_dec_enc': tmee_attentions[1][2],
'src_words': src_words,
'tgt_words': tgt_words,
'base_words': base_words,
'output_words': output_words
})
return tf.estimator.EstimatorSpec(mode, predictions=preds)
