def _make_logistic_eval_metric_ops(labels, predictions, thresholds): """Returns a dictionary of evaluation metric ops for logistic regression. Args: labels: The labels `Tensor`, or a dict with only one `Tensor` keyed by name. predictions: The predictions `Tensor`. thresholds: List of floating point thresholds to use for accuracy, precision, and recall metrics. Returns: A dict of metric results keyed by name. """ # If labels is a dict with a single key, unpack into a single tensor. labels_tensor = labels if isinstance(labels, dict) and len(labels) == 1: labels_tensor = labels.values()[0] metrics = {} metrics[metric_key.MetricKey.PREDICTION_MEAN] = metrics_lib.streaming_mean( predictions) metrics[metric_key.MetricKey.LABEL_MEAN] = metrics_lib.streaming_mean( labels_tensor) # Also include the streaming mean of the label as an accuracy baseline, as # a reminder to users. metrics[metric_key.MetricKey.ACCURACY_BASELINE] = metrics_lib.streaming_mean( labels_tensor) metrics[metric_key.MetricKey.AUC] = metrics_lib.streaming_auc( labels=labels_tensor, predictions=predictions) for threshold in thresholds: predictions_at_threshold = math_ops.cast( math_ops.greater_equal(predictions, threshold), dtypes.float32, name='predictions_at_threshold_%f' % threshold) metrics[metric_key.MetricKey.ACCURACY_MEAN % threshold] = ( metrics_lib.streaming_accuracy(labels=labels_tensor, predictions=predictions_at_threshold)) # Precision for positive examples. metrics[metric_key.MetricKey.PRECISION_MEAN % threshold] = ( metrics_lib.streaming_precision(labels=labels_tensor, predictions=predictions_at_threshold)) # Recall for positive examples. metrics[metric_key.MetricKey.RECALL_MEAN % threshold] = ( metrics_lib.streaming_recall(labels=labels_tensor, predictions=predictions_at_threshold)) return metrics
def _get_eval_ops(self, features, targets, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. targets: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metrics to run. If None, the default metric functions are used; if {}, no metrics are used. Otherwise, `metrics` should map friendly names for the metric to a `MetricSpec` object defining which model outputs to evaluate against which targets with which metric function. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in `../../../../metrics/python/metrics/ops/streaming_metrics.py` and `../metric_spec.py`. Returns: metrics: `dict` of `Tensor` objects. Raises: ValueError: if `metrics` don't match `targets`. """ predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL) result = {'loss': metrics_lib.streaming_mean(loss)} result.update(_make_metrics_ops(metrics, features, targets, predictions)) return result
def _get_eval_ops(self, features, targets, metrics=None): logits = self._model.build_model( features, self._feature_columns, is_training=False) model_fn_ops = self._head.head_ops(features, targets, tf.contrib.learn.ModeKeys.TRAIN, _noop_training_fn, logits=logits) return {'loss': metrics_lib.streaming_mean(model_fn_ops.loss)}
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = {"loss": metrics_lib.streaming_mean(self._loss( logits, targets, features))} # Adds default metrics. if metrics is None: # TODO(b/29366811): This currently results in both an "accuracy" and an # "accuracy/threshold_0.500000_mean" metric for binary classification. metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy} # Adds additional useful metrics for the special case of binary # classification. # TODO(zakaria): Move LogisticRegressor.get_default_metrics to metrics # and handle eval metric from targetcolumn. if self._target_column.num_label_columns == 1: predictions = math_ops.sigmoid(logits) targets_float = math_ops.to_float(targets) default_metrics = ( logistic_regressor.LogisticRegressor.get_default_metrics()) for metric_name, metric_op in default_metrics.items(): result[metric_name] = metric_op(predictions, targets_float) if metrics: class_metrics = {} proba_metrics = {} for name, metric_op in six.iteritems(metrics): if isinstance(name, tuple): if len(name) != 2: raise ValueError("Ignoring metric {}. It returned a tuple with " "len {}, expected 2.".format(name, len(name))) else: if name[1] not in ["classes", "probabilities"]: raise ValueError("Ignoring metric {}. The 2nd element of its " "name should be either 'classes' or " "'probabilities'.".format(name)) elif name[1] == "classes": class_metrics[name[0]] = metric_op else: proba_metrics[name[0]] = metric_op elif isinstance(name, str): class_metrics[name] = metric_op else: raise ValueError("Ignoring metric {}. Its name is not in the correct " "form.".format(name)) if class_metrics: predictions = self._target_column.logits_to_predictions(logits, proba=False) result.update(self._run_metrics(predictions, targets, class_metrics, self._target_column.get_weight_tensor( features))) if proba_metrics: predictions = self._target_column.logits_to_predictions(logits, proba=True) result.update(self._run_metrics(predictions, targets, proba_metrics, self._target_column.get_weight_tensor( features))) return result
def _get_eval_ops(self, features, targets, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. targets: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metric ops to run. If None, the default metric functions are used; if {}, no metrics are used. If model has one output (i.e., returning single predction), keys are `str`, e.g. `'accuracy'` - just a name of the metric that will show up in the logs / summaries. Otherwise, keys are tuple of two `str`, e.g. `('accuracy', 'classes')` - name of the metric and name of `Tensor` in the predictions to run this metric on. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in ../../../../metrics/python/metrics/ops/streaming_metrics.py. Returns: metrics: `dict` of `Tensor` objects. Raises: ValueError: if `metrics` don't match `targets`. """ predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL) result = {"loss": metrics_lib.streaming_mean(loss)} weights = self._get_weight_tensor(features) metrics = metrics or {} if isinstance(targets, dict) and len(targets) == 1: # Unpack single target into just tensor. targets = targets[list(targets.keys())[0]] for name, metric in six.iteritems(metrics): if isinstance(name, tuple): # Multi-head metrics. if not isinstance(predictions, dict): raise ValueError( "Metrics passed provide (name, prediction), " "but predictions are not dict. " "Metrics: %s, Predictions: %s." % (metrics, predictions) ) # Here are two options: targets are single Tensor or a dict. if isinstance(targets, dict) and name[1] in targets: # If targets are dict and the prediction name is in it, apply metric. result[name[0]] = metrics_lib.run_metric(metric, predictions[name[1]], targets[name[1]], weights) else: # Otherwise pass the targets to the metric. result[name[0]] = metrics_lib.run_metric(metric, predictions[name[1]], targets, weights) else: # Single head metrics. if isinstance(predictions, dict): raise ValueError( "Metrics passed provide only name, no prediction, " "but predictions are dict. " "Metrics: %s, Targets: %s." % (metrics, targets) ) result[name] = metrics_lib.run_metric(metric, predictions, targets, weights) return result
def get_eval_ops(self, features, logits, targets, metrics=None): loss = self.loss(logits, targets, features) result = {"loss": metrics_lib.streaming_mean(loss)} if metrics: predictions = self.logits_to_predictions(logits, proba=False) result.update(_run_metrics(predictions, targets, metrics, self.get_weight_tensor(features))) return result
def _streaming_weighted_average_loss(predictions, target, weights=None): loss_unweighted = loss_fn(predictions, target) if weights is not None: weights = math_ops.to_float(weights) _, weighted_average_loss = _loss(loss_unweighted, weights, name="eval_loss") return metrics_lib.streaming_mean(weighted_average_loss)
def _class_predictions_streaming_mean( predictions, labels, weights=None, class_id=None): del labels return metrics_lib.streaming_mean( array_ops.where( math_ops.equal( math_ops.to_int32(class_id), math_ops.to_int32(predictions)), array_ops.ones_like(predictions), array_ops.zeros_like(predictions)), weights=weights)
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = {"loss": metrics_lib.streaming_mean(self._loss(logits, targets, features))} if metrics: predictions = self._target_column.logits_to_predictions(logits, proba=False) result.update( self._run_metrics(predictions, targets, metrics, self._target_column.get_weight_tensor(features)) ) return result
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = {"loss": metrics_lib.streaming_mean(self._loss( logits, targets, weight_tensor=self._get_weight_tensor(features)))} # Adding default metrics if metrics is None: metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy} if self._n_classes == 2: predictions = math_ops.sigmoid(logits) result["auc"] = metrics_lib.streaming_auc(predictions, targets) if metrics: class_metrics = {} proba_metrics = {} for name, metric_op in six.iteritems(metrics): if isinstance(name, tuple): if len(name) != 2: raise ValueError("Ignoring metric {}. It returned a tuple with " "len {}, expected 2.".format(name, len(name))) else: if name[1] not in ["classes", "probabilities"]: raise ValueError("Ignoring metric {}. The 2nd element of its " "name should be either 'classes' or " "'probabilities'.".format(name)) elif name[1] == "classes": class_metrics[name[0]] = metric_op else: proba_metrics[name[0]] = metric_op elif isinstance(name, str): class_metrics[name] = metric_op else: raise ValueError("Ignoring metric {}. Its name is not in the correct " "form.".format(name)) if class_metrics: predictions = self._logits_to_predictions(logits, proba=False) result.update(self._run_metrics(predictions, targets, class_metrics, self._get_weight_tensor(features))) if proba_metrics: predictions = self._logits_to_predictions(logits, proba=True) result.update(self._run_metrics(predictions, targets, proba_metrics, self._get_weight_tensor(features))) return result
def get_eval_ops(self, features, logits, targets, metrics=None): loss = self.loss(logits, targets, features) result = {"loss": metrics_lib.streaming_mean(loss)} # Adds default metrics. if metrics is None: # TODO(b/29366811): This currently results in both an "accuracy" and an # "accuracy/threshold_0.500000_mean" metric for binary classification. metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy} predictions = math_ops.sigmoid(logits) targets_float = math_ops.to_float(targets) default_metrics = self._default_eval_metrics() for metric_name, metric_op in default_metrics.items(): result[metric_name] = metric_op(predictions, targets_float) class_metrics = {} proba_metrics = {} for name, metric_op in six.iteritems(metrics): if isinstance(name, tuple): if len(name) != 2: raise ValueError("Ignoring metric {}. It returned a tuple with " "len {}, expected 2.".format(name, len(name))) else: if name[1] not in ["classes", "probabilities"]: raise ValueError("Ignoring metric {}. The 2nd element of its " "name should be either 'classes' or " "'probabilities'.".format(name)) elif name[1] == "classes": class_metrics[name[0]] = metric_op else: proba_metrics[name[0]] = metric_op elif isinstance(name, str): class_metrics[name] = metric_op else: raise ValueError("Ignoring metric {}. Its name is not in the correct " "form.".format(name)) if class_metrics: class_predictions = self.logits_to_predictions(logits, proba=False) result.update(_run_metrics(class_predictions, targets, class_metrics, self.get_weight_tensor(features))) if proba_metrics: predictions = self.logits_to_predictions(logits, proba=True) result.update(_run_metrics(predictions, targets, proba_metrics, self.get_weight_tensor(features))) return result
def _evaluate_model(self, input_fn, hooks=None, checkpoint_path=None, name=''): # Check that model has been trained (if nothing has been set explicitly). if not checkpoint_path: latest_path = saver.latest_checkpoint(self._model_dir) if not latest_path: error_message = "Could not find trained model at {}.".format(self._model_dir) raise EstimatorNotTrainedError(error_message) checkpoint_path = latest_path # Setup output directory. eval_dir = os.path.join(self._model_dir, 'eval' if not name else 'eval_' + name) with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) global_step = training.create_global_step(g) features, labels = input_fn() estimator_spec = self._call_model_fn(features, labels, Modes.EVAL) if MetricKeys.LOSS in estimator_spec.eval_metric_ops: raise ValueError("Metric with name `{}` is not allowed, because Estimator " "already defines a default metric " "with the same name.".format(MetricKeys.LOSS)) estimator_spec.eval_metric_ops[ MetricKeys.LOSS] = metrics_lib.streaming_mean(estimator_spec.loss) update_op, eval_dict = self._extract_metric_update_ops(estimator_spec.eval_metric_ops) if ops.GraphKeys.GLOBAL_STEP in eval_dict: raise ValueError("Metric with name `global_step` is not allowed, because " "Estimator already defines a default metric with the same name.") eval_dict[ops.GraphKeys.GLOBAL_STEP] = global_step eval_results = evaluation._evaluate_once( checkpoint_path=checkpoint_path, master=self._config.evaluation_master, scaffold=estimator_spec.scaffold, eval_ops=update_op, final_ops=eval_dict, hooks=hooks, config=self._session_config) self._write_dict_to_summary( output_dir=eval_dir, dictionary=eval_results, current_global_step=eval_results[ops.GraphKeys.GLOBAL_STEP]) return eval_results
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = {"loss": metrics_lib.streaming_mean(self._loss( logits, targets, weight_tensor=self._get_weight_tensor(features)))} # Adding default metrics if metrics is None: metrics = {"accuracy": metrics_lib.streaming_accuracy} if self._n_classes == 2: predictions = math_ops.sigmoid(logits) result["eval_auc"] = metrics_lib.streaming_auc(predictions, targets) if metrics: predictions = self._logits_to_predictions(logits, proba=False) result.update(self._run_metrics(predictions, targets, metrics, self._get_weight_tensor(features))) return result
def _get_eval_ops(self, features, labels, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. labels: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metrics to run. If None, the default metric functions are used; if {}, no metrics are used. Otherwise, `metrics` should map friendly names for the metric to a `MetricSpec` object defining which model outputs to evaluate against which labels with which metric function. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in `../../../../metrics/python/metrics/ops/streaming_metrics.py` and `../metric_spec.py`. Returns: metrics: `dict` of `Tensor` objects. Raises: ValueError: if `metrics` don't match `labels`. """ model_fn_ops = self._call_model_fn(features, labels, ModeKeys.EVAL) all_metrics = model_fn_ops.default_metrics # Custom metrics should overwrite defaults. if metrics: all_metrics.update(metrics) result = _make_metrics_ops(all_metrics, features, labels, model_fn_ops.predictions) if metric_key.MetricKey.LOSS not in result: result[metric_key.MetricKey.LOSS] = metrics_lib.streaming_mean( model_fn_ops.loss) return result
def _get_eval_ops(self, features, labels, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. labels: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metrics to run. If None, the default metric functions are used; if {}, no metrics are used. Otherwise, `metrics` should map friendly names for the metric to a `MetricSpec` object defining which model outputs to evaluate against which labels with which metric function. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in `../../../../metrics/python/metrics/ops/streaming_metrics.py` and `../metric_spec.py`. Returns: `ModelFnOps` object. Raises: ValueError: if `metrics` don't match `labels`. """ model_fn_ops = self._call_model_fn( features, labels, model_fn_lib.ModeKeys.EVAL) # Custom metrics should overwrite defaults. if metrics: model_fn_ops.eval_metric_ops.update(_make_metrics_ops( metrics, features, labels, model_fn_ops.predictions)) if metric_key.MetricKey.LOSS not in model_fn_ops.eval_metric_ops: model_fn_ops.eval_metric_ops[metric_key.MetricKey.LOSS] = ( metrics_lib.streaming_mean(model_fn_ops.loss)) return model_fn_ops
def _predictions_streaming_mean(predictions, unused_labels): return metrics_lib.streaming_mean(predictions)
def _predictions_streaming_mean(predictions, unused_labels): return metrics_lib.streaming_mean(predictions)
def _target_streaming_mean(unused_predictions, target, weights=None): return metrics_lib.streaming_mean(target, weights=weights)
def _labels_streaming_mean(unused_predictions, labels, weights=None): return metrics_lib.streaming_mean(labels, weights=weights)
def _targets_streaming_mean(unused_predictions, targets): return metrics_lib.streaming_mean(targets)
def _get_eval_ops(self, features, targets, metrics=None): logits = self._model.build_model(features, self._feature_columns, is_training=False) loss = self._target_column.loss(logits, targets, features) return {'loss': metrics_lib.streaming_mean(loss)}
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset, num_folds=config.fold_count, fold=config.fold, holdout=True) init_op, init_feed_dict, image = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images = tf.train.batch([image], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for loss in tf.losses.get_losses(): metrics_map[loss.op.name] = metrics.streaming_mean(loss) for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) total_loss = tf.losses.get_total_loss() metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) images = input_utils.reshape_images(images, config.dataset) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) # Generate the canvases that lead to the final output image with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append( tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) hooks = [ training.SummaryAtEndHook(log_dir=FLAGS.log_dir, summary_op=summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def __init__(self, run_context=None): if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown(self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply(sigmoid_rampup_value, self.hyper['max_consistency_cost'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') ( (self.class_logits_1, self.cons_logits_1), (self.class_logits_2, self.cons_logits_2), (self.class_logits_ema, self.cons_logits_ema) ) = inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], student_dropout_probability=self.hyper['student_dropout_probability'], teacher_dropout_probability=self.hyper['teacher_dropout_probability'], normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally'], translate=self.hyper['translate'], num_logits=self.hyper['num_logits']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs( self.cons_logits_1, self.class_logits_2, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) def l2_norms(matrix): l2s = tf.reduce_sum(matrix ** 2, axis=1) mean_l2 = tf.reduce_mean(l2s) return mean_l2, l2s self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1) self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema) self.res_costs_1 = self.hyper['logit_distance_cost'] * self.res_l2s_1 self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1) self.res_costs_ema = self.hyper['logit_distance_cost'] * self.res_l2s_ema self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema) self.mean_total_cost_pi, self.total_costs_pi = total_costs( self.class_costs_1, self.cons_costs_pi, self.res_costs_1) self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt, self.res_costs_1) assert_shape(self.total_costs_pi, [3]) assert_shape(self.total_costs_mt, [3]) self.cost_to_be_minimized = tf.cond(self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt, lambda: self.mean_total_cost_pi) with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/pi": self.mean_cons_cost_pi, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/res_cost/1": self.mean_res_cost_1, "train/res_cost/ema": self.mean_res_cost_ema, "train/total_cost/pi": self.mean_total_cost_pi, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({ "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), "eval/res_cost/1": streaming_mean(self.res_costs_1), "eval/res_cost/ema": streaming_mean(self.res_costs_ema), }) metric_variables = slim.get_local_variables(scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def _predictions_streaming_mean( predictions, labels, weights=None, class_id=None): del labels if class_id is not None: predictions = predictions[:, class_id] return metrics_lib.streaming_mean(predictions, weights=weights)
def _indicator_labels_streaming_mean( predictions, labels, weights=None, class_id=None): del predictions if class_id is not None: labels = labels[:, class_id] return metrics_lib.streaming_mean(labels, weights=weights)
def create_metric_ops(self, _inputs, labels, predictions): """Creates the metric op""" loss_mask = tf.sequence_mask( lengths=tf.to_int32(labels["target_len"] - 1), maxlen=tf.to_int32(tf.shape(predictions["losses"])[1])) return metrics.streaming_mean(predictions["losses"], loss_mask)
def _streaming_weighted_average_loss(predictions, target, weights=None): loss_unweighted = loss_fn(predictions, target) _, weighted_average_loss = _loss(loss_unweighted, weights, name="eval_loss") return metrics_lib.streaming_mean(weighted_average_loss)
def _get_eval_ops(self, features, targets, metrics=None): logits = self._model.build_model( features, self._feature_columns, is_training=False) loss = self._target_column.loss(logits, targets, features) return {'loss': metrics_lib.streaming_mean(loss)}
def model_fn(features, labels, mode, params): is_training = mode == learn.ModeKeys.TRAIN optimizer_class = find_class_by_name(params.optimizer, [tf.train]) label_loss_fn = find_class_by_name(params.label_loss, [losses])() model = find_class_by_name(params.model, [frame_level_models, video_level_models])() global_step = tf.train.get_or_create_global_step() learning_rate = tf.train.exponential_decay( params.base_learning_rate, global_step * params.batch_size * params.num_towers, params.learning_rate_decay_examples, params.learning_rate_decay, staircase=True, ) tf.summary.scalar('learning_rate', learning_rate) optimizer = optimizer_class(learning_rate) tf.summary.histogram("model/input_raw", features['model_input']) feature_dim = len(features['model_input'].get_shape()) - 1 model_input = tf.nn.l2_normalize(features['model_input'], feature_dim) tower_inputs = tf.split(model_input, params.num_towers) if mode == learn.ModeKeys.INFER: # *** # this is a quick hack so that the existing model_fn code, # taken from train.py, doesn't break in inference (or serving) mode. # Normally, we would write model_fn such that the 'labels' input arg # can be None in inference mode, but this existing model code was not written this # way. See the serving_input_fn() defined below, to see where 'labels_batch' # is added to the features dict, just to make this code work properly labels = features['labels_batch'] tower_labels = tf.split(labels, params.num_towers) tower_num_frames = tf.split(features['num_frames'], params.num_towers) tower_gradients = [] tower_predictions = [] tower_label_losses = [] tower_reg_losses = [] for i in range(params.num_towers): # For some reason these 'with' statements can't be combined onto the same # line. They have to be nested. with tf.device(params.device_string % i): with (tf.variable_scope(("tower"), reuse=True if i > 0 else None)): with (slim.arg_scope([slim.model_variable, slim.variable], device="/cpu:0" if params.num_gpus != 1 else "/gpu:0")): result = model.create_model( tower_inputs[i], num_frames=tower_num_frames[i], vocab_size=params.reader.num_classes, labels=tower_labels[i], is_training=is_training) for variable in slim.get_model_variables(): tf.summary.histogram(variable.op.name, variable) predictions = result["predictions"] tower_predictions.append(predictions) if "loss" in result.keys(): label_loss = result["loss"] else: label_loss = label_loss_fn.calculate_loss( predictions, tower_labels[i]) if "regularization_loss" in result.keys(): reg_loss = result["regularization_loss"] else: reg_loss = tf.constant(0.0) reg_losses = tf.losses.get_regularization_losses() if reg_losses: reg_loss += tf.add_n(reg_losses) tower_reg_losses.append(reg_loss) # Adds update_ops (e.g., moving average updates in batch normalization) as # a dependency to the train_op. update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) if "update_ops" in result.keys(): update_ops += result["update_ops"] if update_ops: with tf.control_dependencies(update_ops): barrier = tf.no_op(name="gradient_barrier") with tf.control_dependencies([barrier]): label_loss = tf.identity(label_loss) tower_label_losses.append(label_loss) final_loss = params.regularization_penalty * reg_loss + label_loss gradients = optimizer.compute_gradients( final_loss, colocate_gradients_with_ops=False) tower_gradients.append(gradients) pred_dict = {} label_loss = tf.reduce_mean(tf.stack(tower_label_losses)) predictions = tf.concat(tower_predictions, 0) pred_dict['predictions'] = predictions tf.summary.scalar("label_loss", label_loss) if params.regularization_penalty != 0: reg_loss = tf.reduce_mean(tf.stack(tower_reg_losses)) tf.summary.scalar("reg_loss", reg_loss) if is_training: # Incorporate the L2 weight penalties, etc. merged_gradients = utils.combine_gradients(tower_gradients) if params.clip_gradient_norm > 0: with tf.name_scope('clip_grads'): merged_gradients = utils.clip_gradient_norms( merged_gradients, params.clip_gradient_norm) train_op = optimizer.apply_gradients(merged_gradients, global_step=global_step) else: train_op = None eval_metric_ops = {} if mode == learn.ModeKeys.EVAL or is_training: eval_metric_ops['hit_at_one'] = metrics.streaming_mean( tf.py_func( lambda x, y: np.float32(eval_util.calculate_hit_at_one(x, y)), [predictions, labels], tf.float32, stateful=False, )) eval_metric_ops['perr'] = metrics.streaming_mean( tf.py_func( lambda x, y: np.float32( eval_util.calculate_precision_at_equal_recall_rate(x, y)), [predictions, labels], tf.float32, stateful=False, )) eval_metric_ops['gap'] = metrics.streaming_mean( tf.py_func( lambda x, y: np.float32(eval_util.calculate_gap(x, y)), [predictions, labels], tf.float32, stateful=False, )) else: pass top_predictions, top_indices = tf.nn.top_k(predictions, _TOP_PREDICTIONS_IN_OUTPUT) pred_dict['top_predictions'] = top_predictions pred_dict['top_indices'] = top_indices #add eval summaries and update ops for training for key, val in eval_metric_ops.items(): tf.summary.scalar(key, val[0]) #create summary for each eval op tf.add_to_collection( tf.GraphKeys.UPDATE_OPS, val[1] ) # add the update op for each eval up to update ops collection, so that it will be run every train_op call # tf.add_to_collection("global_step", global_step) # tf.add_to_collection("loss", label_loss) tf.add_to_collection("predictions", tf.concat(tower_predictions, 0)) # tf.add_to_collection("input_batch_raw", model_input_raw) # tf.add_to_collection("input_batch", model_input) # tf.add_to_collection("num_frames", num_frames) tf.add_to_collection("labels", tf.cast(labels, tf.float32)) # tf.add_to_collection("train_op", train_op) tf.summary.scalar("loss", label_loss) export_outputs = { tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(pred_dict) } return tf.estimator.EstimatorSpec(mode=mode, predictions=pred_dict, loss=label_loss, train_op=train_op, export_outputs=export_outputs, eval_metric_ops=eval_metric_ops)
def _maybe_add_streaming_mean(result, key, value): if key in result: logging.warning('Metrics already contains %s, skipping.', key) return result[key] = metrics_lib.streaming_mean(value)
def perplexity_metric(losses, target_len): """Calculates the mean log perplexity based on losses and target_len""" loss_mask = tf.sequence_mask(lengths=tf.to_int32(target_len - 1), maxlen=tf.to_int32(tf.shape(losses)[1])) return metrics.streaming_mean(losses, loss_mask)
def __init__(self, result_dir): self.checkpoint_dir = os.path.join(result_dir, 'checkpoints') self.summary_dir = os.path.join(result_dir, 'summaries') os.makedirs(self.checkpoint_dir) os.makedirs(self.summary_dir) with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None, ), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown( self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add( sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply( sigmoid_rampup_value, self.hyper['max_consistency_coefficient'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add( (1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add( (1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') self.logits_1, self.logits_2, self.logits_ema = inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors( self.logits_ema, self.labels) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs( self.logits_1, self.logits_2, self.cons_coefficient, consistency_mask) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.logits_1, self.logits_ema, self.cons_coefficient, consistency_mask) self.mean_total_cost_pi, self.total_costs_pi = total_costs( self.class_costs_1, self.cons_costs_pi) self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt) self.cost_to_be_minimized = tf.cond( self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt, lambda: self.mean_total_cost_pi) with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = nn.adam_optimizer( self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/pi": self.mean_cons_cost_pi, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/pi": self.mean_total_cost_pi, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map( { "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), # Note that the evaluation costs are not directly comparable # to the training costs. Training batches contain unlabeled # samples but the evaluation batches do not. Because # classification cost is zero for unlabeled samples, the # training costs are smaller than evaluation costs when # doing semi-supervised learning. "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), "eval/cons_cost/pi": streaming_mean(self.cons_costs_pi), "eval/cons_cost/mt": streaming_mean(self.cons_costs_mt), "eval/total_cost/pi": streaming_mean(self.total_costs_pi), "eval/total_cost/mt": streaming_mean(self.total_costs_mt) }) metric_variables = slim.get_local_variables( scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def streaming_mean(name, value): tf.summary.scalar( name, tfm.streaming_mean(value, name='stream/{}'.format(name))[1])
def _compute_loss(logits, targets, weights=None): return metrics_lib.streaming_mean(self._loss( logits, targets, weight_tensor=weights))
def create_metric_ops(self, _inputs, labels, predictions): """Creates the metric op""" loss_mask = tf.sequence_mask( lengths=tf.to_int32(labels["target_len"] - 1), maxlen=tf.to_int32(tf.shape(predictions["losses"])[1])) return metrics.streaming_mean(predictions["losses"], loss_mask)
def _streaming_weighted_average_loss(predictions, target, weights=None): loss_unweighted = loss_fn(predictions, target) _, weighted_average_loss = _loss(loss_unweighted, weights, name="eval_loss") return metrics_lib.streaming_mean(weighted_average_loss)
def _get_eval_ops(self, features, targets, metrics=None): """See base class.""" logits = self._logits(features) result = { "loss": metrics_lib.streaming_mean(self._loss(logits, targets, features)) } # Adds default metrics. if metrics is None: # TODO(b/29366811): This currently results in both an "accuracy" and an # "accuracy/threshold_0.500000_mean" metric for binary classification. metrics = {("accuracy", "classes"): metrics_lib.streaming_accuracy} # Adds additional useful metrics for the special case of binary # classification. # TODO(zakaria): Move LogisticRegressor.get_default_metrics to metrics # and handle eval metric from targetcolumn. if self._target_column.num_label_columns == 1: predictions = math_ops.sigmoid(logits) targets_float = math_ops.to_float(targets) default_metrics = ( logistic_regressor.LogisticRegressor.get_default_metrics()) for metric_name, metric_op in default_metrics.items(): result[metric_name] = metric_op(predictions, targets_float) if metrics: class_metrics = {} proba_metrics = {} for name, metric_op in six.iteritems(metrics): if isinstance(name, tuple): if len(name) != 2: raise ValueError( "Ignoring metric {}. It returned a tuple with " "len {}, expected 2.".format(name, len(name))) else: if name[1] not in ["classes", "probabilities"]: raise ValueError( "Ignoring metric {}. The 2nd element of its " "name should be either 'classes' or " "'probabilities'.".format(name)) elif name[1] == "classes": class_metrics[name[0]] = metric_op else: proba_metrics[name[0]] = metric_op elif isinstance(name, str): class_metrics[name] = metric_op else: raise ValueError( "Ignoring metric {}. Its name is not in the correct " "form.".format(name)) if class_metrics: predictions = self._target_column.logits_to_predictions( logits, proba=False) result.update( self._run_metrics( predictions, targets, class_metrics, self._target_column.get_weight_tensor(features))) if proba_metrics: predictions = self._target_column.logits_to_predictions( logits, proba=True) result.update( self._run_metrics( predictions, targets, proba_metrics, self._target_column.get_weight_tensor(features))) return result
def _labels_streaming_mean(unused_predictions, labels): return metrics_lib.streaming_mean(labels)
def _predictions_streaming_mean(predictions, unused_target, weights=None): return metrics_lib.streaming_mean(predictions, weights=weights)
def _labels_streaming_mean(unused_predictions, labels): return metrics_lib.streaming_mean(labels)
import tensorflow as tf import tensorflow.contrib.metrics as tcm values = tf.random_uniform(shape=[2], minval=1, maxval=5, dtype=tf.int32) # 1.Initialization: initializing the metric state. # Calling streaming_mean creates a pair of state variables that will contain: # (1) the running sum and (2) the count of the number of samples in the sum. # tf.local_variables_initializer() sets the sum and count variables to zero. mean_value, update_op = tcm.streaming_mean(values, weights=[0.2, 0.8]) with tf.Session() as sess: # The streaming metrics use local variables. sess.run(tf.local_variables_initializer()) for i in range(10): # 2.Aggregation: updating the values of the metric state. # Aggregation is performed by examining the current state of values # and incrementing the state variables appropriately. mean_value_result, value_result = sess.run([update_op, values]) print("mean after {}: {} {}".format(i, value_result, mean_value_result)) # 3.Finalization: computing the final metric value. final_mean_value_result = mean_value.eval() print("final mean: {}".format(final_mean_value_result))
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset) init_op, init_feed_dict, image, label = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images, labels = tf.train.batch( [image, label], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) tfprof.model_analyzer.print_model_analysis(tf.get_default_graph()) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) predictions = tf.argmax(outputs, 1) metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels) metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k( outputs, tf.expand_dims(labels, 1), 5) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) scaffold = tf.train.Scaffold(init_op, init_feed_dict) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') # On Tensorflow master fd87896 fixes this, but for now just set a very large number eval_kwargs['max_number_of_evaluations'] = sys.maxint eval_fn(checkpoint_path, scaffold=scaffold, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def _targets_streaming_mean(unused_predictions, targets): return metrics_lib.streaming_mean(targets)
def __init__(self, run_context=None, hyper_dict={}): #inilization of hyper for i in hyper_dict: assert i in self.hyper, "Wrong hyper dict '{}'!".format(i) self.hyper[i] = hyper_dict[i] # inilize bg noise input if self.hyper['bg_noise']: self.bg_noise_input = tf.convert_to_tensor(self.hyper['bg_noise_input'],dtype=tf.float32) else: self.bg_noise_input = tf.convert_to_tensor(np.zeros((32,32)),dtype=tf.float32) # inilization model print('{} is initliazed!'.format(self.hyper['cnn'])) self.cnn = getattr(model,self.hyper['cnn']) if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None,) + self.hyper['input_dim'], name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None,) + self.hyper['label_dim'], name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) with tf.name_scope("ramps"): self.learning_rate, self.cons_coefficient, \ self.adam_beta_1, self.adam_beta_2, \ self.ema_decay = ramp_value(self.global_step,self.hyper) ( self.class_logits_1, self.class_logits_ema ) = self.inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], student_dropout_probability=self.hyper['student_dropout_probability'], teacher_dropout_probability=self.hyper['teacher_dropout_probability'], normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally'], translate=self.hyper['translate']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels,sig = self.hyper['sig']) self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels,sig = self.hyper['sig']) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels,sig = self.hyper['sig']) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels,sig = self.hyper['sig']) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.class_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'],sig = 'softmax') self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt) self.cost_to_be_minimized = self.mean_total_cost_mt with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): if self.hyper['optimizer']=='adam': self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) elif self.hyper['optimizer']=='sgd': self.train_step_op = nn.sgd_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.hyper['max_learning_rate']) else: assert False, 'Wrong optimizer input!' self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({ "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), }) metric_variables = slim.get_local_variables(scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True self.session = tf.Session(config=config) self.run(self.init_init_op) self.save_tensorboard_graph()
def __init__(self, run_context=None): self.name = "Tweet Data Class" if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.tweets = tf.placeholder(dtype=tf.float32, shape=(None, 500), name='tweets') self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): # Ramp-up and ramp-down has been removed for simplicity # self.learning_rate = tf.constant(self.hyper['max_learning_rate'], dtype = tf.float32) # self.adam_beta_1 = tf.constant(self.hyper['adam_beta_1_after_rampdown'], dtype = tf.float32) # self.cons_coefficient = tf.constant(self.hyper['max_consistency_cost'], dtype = tf.float32) # self.adam_beta_2 = tf.constant(self.hyper['adam_beta_2_after_rampup'], dtype = tf.float32) # self.ema_decay = tf.constant(self.hyper['ema_decay_after_rampup'], dtype = tf.float32) # self.learning_rate =self.DEFAULT_HYPERPARAMS['max_learning_rate'] # self.adam_beta_1 = self.DEFAULT_HYPERPARAMS['adam_beta_1_after_rampdown'] # self.cons_coefficient = self.DEFAULT_HYPERPARAMS['max_consistency_cost'] # self.adam_beta_2 = self.DEFAULT_HYPERPARAMS['adam_beta_2_after_rampup'] # self.ema_decay = self.DEFAULT_HYPERPARAMS['ema_decay_after_rampup'] sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown(self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply(sigmoid_rampup_value, self.hyper['max_consistency_cost'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') # below is where the interesting stuff happens, mostly. # Inference is a function which creates the towers and sets up the different logits for the two models ( (self.class_logits_1, self.cons_logits_1), (self.class_logits_2, self.cons_logits_2), (self.class_logits_ema, self.cons_logits_ema) ) = inference( self.tweets, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], hidden_dims = self.DEFAULT_HYPERPARAMS['hidden_dims'], student_dropout_probability=self.hyper['student_dropout_probability'], teacher_dropout_probability=self.hyper['teacher_dropout_probability'], num_logits=self.hyper['num_logits']) with tf.name_scope("objectives"): # something weird is done with errors for unlabeled examples. # I think errors are only calculated for labeled, but you don't calculate it for unlabeled, so it is NaN for unlabeled self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels) # where we calculate classification costs. # the cost_1 should be for student and ema is for teacher self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask) def l2_norms(matrix): l2s = tf.reduce_sum(matrix ** 2, axis=1) mean_l2 = tf.reduce_mean(l2s) return mean_l2, l2s self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1) self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema) # mean total cost is what you are optimizng. self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt) assert_shape(self.total_costs_mt, [2]) self.cost_to_be_minimized = self.mean_total_cost_mt with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = adam_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) # TODO do we really need this? self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { # NOTE these should not need training, since we don't do ramp-up and ramp-down "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/mt": self.mean_total_cost_mt, } # TODO not sure what streaming mean does? with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({ "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), }) metric_variables = slim.get_local_variables(scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) # TODO string utils just formats dictionary results in a nice way for logging, not needed? self.result_formatter = string_utils.DictFormatter( order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) print("Train init variables:") for var in train_init_variables: print(var) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def __init__(self, run_context=None): if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None, ), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown( self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add( sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply( sigmoid_rampup_value, self.hyper['max_consistency_cost'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add( (1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add( (1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') ((self.class_logits_1, self.cons_logits_1), (self.class_logits_2, self.cons_logits_2), (self.class_logits_ema, self.cons_logits_ema)) = inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], student_dropout_probability=self. hyper['student_dropout_probability'], teacher_dropout_probability=self. hyper['teacher_dropout_probability'], normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally'], translate=self.hyper['translate'], num_logits=self.hyper['num_logits']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors( self.class_logits_ema, self.labels) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs( self.cons_logits_1, self.class_logits_2, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) def l2_norms(matrix): l2s = tf.reduce_sum(matrix**2, axis=1) mean_l2 = tf.reduce_mean(l2s) return mean_l2, l2s self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1) self.mean_res_l2_ema, self.res_l2s_ema = l2_norms( self.class_logits_ema - self.cons_logits_ema) self.res_costs_1 = self.hyper[ 'logit_distance_cost'] * self.res_l2s_1 self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1) self.res_costs_ema = self.hyper[ 'logit_distance_cost'] * self.res_l2s_ema self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema) self.mean_total_cost_pi, self.total_costs_pi = total_costs( self.class_costs_1, self.cons_costs_pi, self.res_costs_1) self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt, self.res_costs_1) assert_shape(self.total_costs_pi, [3]) assert_shape(self.total_costs_mt, [3]) self.cost_to_be_minimized = tf.cond( self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt, lambda: self.mean_total_cost_pi) with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = nn.adam_optimizer( self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/pi": self.mean_cons_cost_pi, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/res_cost/1": self.mean_res_cost_1, "train/res_cost/ema": self.mean_res_cost_ema, "train/total_cost/pi": self.mean_total_cost_pi, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map( { "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), "eval/res_cost/1": streaming_mean(self.res_costs_1), "eval/res_cost/ema": streaming_mean(self.res_costs_ema), }) metric_variables = slim.get_local_variables( scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=[ "eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt" ], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def model_fn(features, mode, config, embedding_matrix, vocab_tables): if mode == tf.estimator.ModeKeys.PREDICT: base_words, src_words, tgt_words, inserted_words, deleted_words = features else: src_words, tgt_words, inserted_words, deleted_words = features base_words = src_words if mode != tf.estimator.ModeKeys.TRAIN: config.put('editor.enable_dropout', False) config.put('editor.dropout_keep', 1.0) vocab_s2i = vocab_tables[vocab.STR_TO_INT] vocab_i2s = vocab_tables[vocab.INT_TO_STR] vocab.init_embeddings(embedding_matrix) train_decoder_output, infer_decoder_output, \ gold_dec_out, gold_dec_out_len = editor.editor_train( base_words, src_words, tgt_words, inserted_words, deleted_words, embedding_matrix, vocab_s2i, config.editor.hidden_dim, config.editor.agenda_dim, config.editor.edit_dim, config.editor.encoder_layers, config.editor.decoder_layers, config.editor.attention_dim, config.editor.beam_width, config.editor.max_sent_length, config.editor.dropout_keep, config.editor.lamb_reg, config.editor.norm_eps, config.editor.norm_max, config.editor.kill_edit, config.editor.draw_edit, config.editor.use_swap_memory, config.get('editor.use_beam_decoder', False), config.get('editor.enable_dropout', False), config.get('editor.no_insert_delete_attn', False), config.get('editor.enable_vae', True) ) loss = optimizer.loss(train_decoder_output, gold_dec_out, gold_dec_out_len) train_op, gradients_norm = optimizer.train( loss, config.optim.learning_rate, config.optim.max_norm_observe_steps) if mode == tf.estimator.ModeKeys.TRAIN: tf.summary.scalar('grad_norm', gradients_norm) ops = add_extra_summary(config, vocab_i2s, train_decoder_output, src_words, tgt_words, inserted_words, deleted_words, ['extra']) hooks = [ get_train_extra_summary_writer(config), get_extra_summary_logger(ops, config), ] if config.get('logger.enable_profiler', False): hooks.append(get_profiler_hook(config)) return tf.estimator.EstimatorSpec(mode, train_op=train_op, loss=loss, training_hooks=hooks) elif mode == tf.estimator.ModeKeys.EVAL: ops = add_extra_summary(config, vocab_i2s, train_decoder_output, src_words, tgt_words, inserted_words, deleted_words, ['extra']) return tf.estimator.EstimatorSpec( mode, loss=loss, evaluation_hooks=[get_extra_summary_logger(ops, config)], eval_metric_ops={'bleu': tf_metrics.streaming_mean(ops[ES_BLEU])}) elif mode == tf.estimator.ModeKeys.PREDICT: lengths = decoder.seq_length(infer_decoder_output) tokens = decoder.str_tokens(infer_decoder_output, vocab_i2s) preds = { 'str_tokens': tokens, 'sample_id': decoder.sample_id(infer_decoder_output), 'lengths': lengths, 'joined': metrics.join_tokens(tokens, lengths), } return tf.estimator.EstimatorSpec(mode, predictions=preds)
def _target_streaming_mean(unused_predictions, target, weights=None): return metrics_lib.streaming_mean(target, weights=weights)
def _get_eval_ops(self, features, targets, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. targets: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metric ops to run. If None, the default metric functions are used; if {}, no metrics are used. If model has one output (i.e., returning single predction), keys are `str`, e.g. `'accuracy'` - just a name of the metric that will show up in the logs / summaries. Otherwise, keys are tuple of two `str`, e.g. `('accuracy', 'classes')` - name of the metric and name of `Tensor` in the predictions to run this metric on. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in ../../../../metrics/python/metrics/ops/streaming_metrics.py. Returns: metrics: `dict` of `Tensor` objects. Raises: ValueError: if `metrics` don't match `targets`. """ predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL) result = {'loss': metrics_lib.streaming_mean(loss)} weights = self._get_weight_tensor(features) metrics = metrics or {} if isinstance(targets, dict) and len(targets) == 1: # Unpack single target into just tensor. targets = targets[list(targets.keys())[0]] for name, metric in six.iteritems(metrics): if isinstance(name, tuple): # Multi-head metrics. if not isinstance(predictions, dict): raise ValueError( 'Metrics passed provide (name, prediction), ' 'but predictions are not dict. ' 'Metrics: %s, Predictions: %s.' % (metrics, predictions)) # Here are two options: targets are single Tensor or a dict. if isinstance(targets, dict) and name[1] in targets: # If targets are dict and the prediction name is in it, apply metric. result[name[0]] = metrics_lib.run_metric( metric, predictions[name[1]], targets[name[1]], weights) else: # Otherwise pass the targets to the metric. result[name[0]] = metrics_lib.run_metric( metric, predictions[name[1]], targets, weights) else: # Single head metrics. if isinstance(predictions, dict): raise ValueError( 'Metrics passed provide only name, no prediction, ' 'but predictions are dict. ' 'Metrics: %s, Targets: %s.' % (metrics, targets)) result[name] = metrics_lib.run_metric( metric, predictions, targets, weights) return result
def model_fn(features, mode, config, embedding_matrix, vocab_tables): if mode == tf.estimator.ModeKeys.PREDICT: base_words, extended_base_words, \ _, _, \ src_words, tgt_words, \ inserted_words, deleted_words, \ oov = features output_words = extended_output_words = tgt_words else: base_words, extended_base_words, \ output_words, extended_output_words, \ src_words, tgt_words, \ inserted_words, deleted_words, \ oov = features if mode != tf.estimator.ModeKeys.TRAIN: config.put('editor.enable_dropout', False) config.put('editor.dropout_keep', 1.0) vocab_i2s = vocab_tables[vocab.INT_TO_STR] vocab.init_embeddings(embedding_matrix) vocab_size = len(vocab_tables[vocab.RAW_WORD2ID]) train_decoder_output, infer_decoder_output, \ gold_dec_out, gold_dec_out_len = editor.editor_train( base_words, extended_base_words, output_words, extended_output_words, src_words, tgt_words, inserted_words, deleted_words, oov, vocab_size, config.editor.hidden_dim, config.editor.agenda_dim, config.editor.edit_dim, config.editor.edit_enc.micro_ev_dim, config.editor.edit_enc.num_heads, config.editor.encoder_layers, config.editor.decoder_layers, config.editor.attention_dim, config.editor.beam_width, config.editor.edit_enc.ctx_hidden_dim, config.editor.edit_enc.ctx_hidden_layer, config.editor.edit_enc.wa_hidden_dim, config.editor.edit_enc.wa_hidden_layer, config.editor.edit_enc.meve_hidden_dim, config.editor.edit_enc.meve_hidden_layer, config.editor.max_sent_length, config.editor.dropout_keep, config.editor.lamb_reg, config.editor.norm_eps, config.editor.norm_max, config.editor.kill_edit, config.editor.draw_edit, config.editor.use_swap_memory, config.get('editor.use_beam_decoder', False), config.get('editor.enable_dropout', False), config.get('editor.no_insert_delete_attn', False), config.get('editor.enable_vae', True) ) loss = optimizer.loss(train_decoder_output, gold_dec_out, gold_dec_out_len) train_op, gradients_norm = optimizer.train( loss, config.optim.learning_rate, config.optim.max_norm_observe_steps) tf.logging.info("Trainable variable") for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): tf.logging.info(str(i)) if mode == tf.estimator.ModeKeys.TRAIN: tf.summary.scalar('grad_norm', gradients_norm) ops = add_extra_summary(config, vocab_i2s, train_decoder_output, base_words, output_words, src_words, tgt_words, inserted_words, deleted_words, oov, vocab_size, collections=['extra']) hooks = [ get_train_extra_summary_writer(config), get_extra_summary_logger(ops, config), ] if config.get('logger.enable_profiler', False): hooks.append(get_profiler_hook(config)) return tf.estimator.EstimatorSpec(mode, train_op=train_op, loss=loss, training_hooks=hooks) elif mode == tf.estimator.ModeKeys.EVAL: ops = add_extra_summary(config, vocab_i2s, train_decoder_output, base_words, output_words, src_words, tgt_words, inserted_words, deleted_words, oov, vocab_size, collections=['extra']) return tf.estimator.EstimatorSpec( mode, loss=loss, evaluation_hooks=[get_extra_summary_logger(ops, config)], eval_metric_ops={'bleu': tf_metrics.streaming_mean(ops[ES_BLEU])}) elif mode == tf.estimator.ModeKeys.PREDICT: lengths = decoder.seq_length(infer_decoder_output) tokens = decoder.str_tokens(infer_decoder_output, vocab_i2s, vocab_size, oov) preds = { 'str_tokens': tokens, 'sample_id': decoder.sample_id(infer_decoder_output), 'lengths': lengths, 'joined': metrics.join_tokens(tokens, lengths), } return tf.estimator.EstimatorSpec(mode, predictions=preds)
def _maybe_add_streaming_mean(result, key, value): if key in result: logging.warning('Metrics already contains %s, skipping.', key) return result[key] = metrics_lib.streaming_mean(value)
def _predictions_streaming_mean(predictions, unused_labels, weights=None): return metrics_lib.streaming_mean(predictions, weights=weights)
def model_fn(features, mode, config, embedding_matrix, vocab_tables): if mode == tf.estimator.ModeKeys.PREDICT: base_words, _, src_words, tgt_words, inserted_words, commong_words = features output_words = tgt_words else: base_words, output_words, src_words, tgt_words, inserted_words, commong_words = features is_training = mode == tf.estimator.ModeKeys.TRAIN tf.add_to_collection('is_training', is_training) if mode != tf.estimator.ModeKeys.TRAIN: config.put('editor.enable_dropout', False) config.put('editor.dropout_keep', 1.0) config.put('editor.dropout', 0.0) config.put('editor.transformer.enable_dropout', False) config.put('editor.transformer.layer_postprocess_dropout', 0.0) config.put('editor.transformer.attention_dropout', 0.0) config.put('editor.transformer.relu_dropout', 0.0) vocab.init_embeddings(embedding_matrix) EmbeddingSharedWeights.init_from_embedding_matrix() editor_model = Editor(config) logits, beam_prediction = editor_model(base_words, src_words, tgt_words, inserted_words, commong_words, output_words) targets = decoder.prepare_decoder_output( output_words, sequence.length_pre_embedding(output_words)) target_lengths = sequence.length_pre_embedding(targets) vocab_size = embedding_matrix.shape[0] loss, weights = optimizer.padded_cross_entropy_loss( logits, targets, target_lengths, config.optim.label_smoothing, vocab_size) train_op = optimizer.get_train_op(loss, config) tf.logging.info("Trainable variable") for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): tf.logging.info(str(i)) tf.logging.info("Num of Trainable parameters") tf.logging.info( np.sum([ np.prod(v.get_shape().as_list()) for v in tf.trainable_variables() ])) if mode == tf.estimator.ModeKeys.TRAIN: decoded_ids = decoder.logits_to_decoded_ids(logits) ops = add_extra_summary(config, decoded_ids, target_lengths, base_words, output_words, src_words, tgt_words, inserted_words, commong_words, collections=['extra']) hooks = [ get_train_extra_summary_writer(config), get_extra_summary_logger(ops, config), ] if config.get('logger.enable_profiler', False): hooks.append(get_profiler_hook(config)) return tf.estimator.EstimatorSpec(mode, train_op=train_op, loss=loss, training_hooks=hooks) elif mode == tf.estimator.ModeKeys.EVAL: decoded_ids = decoder.logits_to_decoded_ids(logits) ops = add_extra_summary(config, decoded_ids, target_lengths, base_words, output_words, src_words, tgt_words, inserted_words, commong_words, collections=['extra']) return tf.estimator.EstimatorSpec( mode, loss=loss, evaluation_hooks=[get_extra_summary_logger(ops, config)], eval_metric_ops={'bleu': tf_metrics.streaming_mean(ops[ES_BLEU])}) elif mode == tf.estimator.ModeKeys.PREDICT: decoded_ids, decoded_lengths, scores = beam_prediction tokens = decoder.str_tokens(decoded_ids) preds = { 'str_tokens': tf.transpose(tokens, [0, 2, 1]), 'decoded_ids': tf.transpose(decoded_ids, [0, 2, 1]), 'lengths': decoded_lengths, 'joined': metrics.join_tokens(tokens, decoded_lengths) } tmee_attentions = tf.get_collection( 'TransformerMicroEditExtractor_Attentions') if len(tmee_attentions) > 0: preds.update({ 'tmee_attentions_st_enc_self': tmee_attentions[0][0], 'tmee_attentions_st_dec_self': tmee_attentions[0][1], 'tmee_attentions_st_dec_enc': tmee_attentions[0][2], 'tmee_attentions_ts_enc_self': tmee_attentions[1][0], 'tmee_attentions_ts_dec_self': tmee_attentions[1][1], 'tmee_attentions_ts_dec_enc': tmee_attentions[1][2], 'src_words': src_words, 'tgt_words': tgt_words, 'base_words': base_words, 'output_words': output_words }) return tf.estimator.EstimatorSpec(mode, predictions=preds)