def _create_modalities(self, problem_hparams, hparams): """Construct modalities in problem_hparams.""" input_modality_overrides = {} for override_str in hparams.input_modalities.split(";"): if override_str != "default": parts = override_str.split(":") feature_name = parts[0] modality_name = ":".join(parts[1:]) input_modality_overrides[feature_name] = modality_name target_modality_name = None if hparams.target_modality and hparams.target_modality != "default": target_modality_name = hparams.target_modality input_modality = {} for f, modality_spec in six.iteritems(problem_hparams.input_modality): if f in input_modality_overrides: _warn_changed_modality_type(input_modality_overrides[f], modality_spec[0], f) modality_spec = (input_modality_overrides[f], modality_spec[1]) input_modality[f] = registry.create_modality(modality_spec, hparams) problem_hparams.input_modality = input_modality target_modality_spec = problem_hparams.target_modality if target_modality_name: _warn_changed_modality_type(target_modality_name, target_modality_spec[0], "target") target_modality_spec = (target_modality_name, target_modality_spec[1]) target_modality = registry.create_modality(target_modality_spec, hparams) problem_hparams.target_modality = target_modality
def _create_tpu_eval_metrics_fn(problem, hparams): """Create the metrics_fn that TPUEstimatorSpec expects.""" tm = problem.get_hparams().target_modality if isinstance(tm, tuple): tm = registry.create_modality(tm, hparams) weights_fn = tm.targets_weights_fn def make_metric_fn(metric_fn): def wrapped_metric_fn(logits, labels): num, den = metric_fn(logits, labels, weights_fn=weights_fn) return tf.metrics.mean(num, den) return wrapped_metric_fn metric_fns = [] eval_metrics = problem.eval_metrics() for metric in eval_metrics: if metric in TPU_METRIC_BLACKLIST: tf.logging.warn("Skipping eval metric %s in TPU_METRIC_BLACKLIST", metric) continue name = "metrics-%s/%s" % (problem.name, metric) metric_fns.append((name, make_metric_fn(metrics.METRICS_FNS[metric]))) def all_metrics_fn(logits, labels): metrics_dict = {} for name, fn in metric_fns: metrics_dict[name] = fn(logits, labels) return metrics_dict return all_metrics_fn
def create_eager_metrics_for_problem(problem, model_hparams=None): """See create_eager_metrics.""" metric_names = problem.eval_metrics() tm = problem.get_hparams().target_modality if isinstance(tm, tuple): assert model_hparams is not None tm = registry.create_modality(tm, model_hparams) return create_eager_metrics(metric_names, weights_fn=tm.targets_weights_fn)
def _create_modalities(self, problem_hparams, hparams): """Construct modalities in problem_hparams.""" input_modality_overrides = {} for override_str in hparams.input_modalities.split(";"): parts = override_str.split(":") feature_name = parts[0] modality_name = ":".join(parts[1:]) input_modality_overrides[feature_name] = modality_name target_modality_name = None if hparams.target_modality: target_modality_name = hparams.target_modality input_modality = {} for f, modality_spec in six.iteritems(problem_hparams.input_modality): if isinstance(modality_spec, modality.Modality): # This function has been previously run (e.g. for training and now is # being called for eval) and the modalities have already been # constructed. Return. return if f in input_modality_overrides: _warn_changed_modality_type(input_modality_overrides[f], modality_spec[0], f) modality_spec = (input_modality_overrides[f], modality_spec[1]) input_modality[f] = registry.create_modality( modality_spec, hparams) problem_hparams.input_modality = input_modality target_modality_spec = problem_hparams.target_modality if isinstance(target_modality_spec, modality.Modality): return if target_modality_name: _warn_changed_modality_type(target_modality_name, target_modality_spec[0], "target") target_modality_spec = (target_modality_name, target_modality_spec[1]) target_modality = registry.create_modality(target_modality_spec, hparams) problem_hparams.target_modality = target_modality
def _create_modalities(self, problem_hparams, hparams): """Construct modalities in problem_hparams.""" input_modality_overrides = {} for override_str in hparams.input_modalities.split(";"): if override_str != "default": parts = override_str.split(":") feature_name = parts[0] modality_name = ":".join(parts[1:]) input_modality_overrides[feature_name] = modality_name target_modality_name = None if hparams.target_modality and hparams.target_modality != "default": target_modality_name = hparams.target_modality input_modality = {} for f, modality_spec in six.iteritems(problem_hparams.input_modality): if f in input_modality_overrides: _warn_changed_modality_type(input_modality_overrides[f], modality_spec[0], f) modality_spec = (input_modality_overrides[f], modality_spec[1]) input_modality[f] = registry.create_modality(modality_spec, hparams) problem_hparams.input_modality = input_modality if isinstance(problem_hparams.target_modality, dict): target_modality = {} for f, modality_spec in six.iteritems(problem_hparams.target_modality): if target_modality_name: _warn_changed_modality_type(target_modality_name, modality_spec[0], "target_modality/%s" % f) modality_spec = (target_modality_name, modality_spec[1]) target_modality[f] = registry.create_modality(modality_spec, hparams) else: target_modality_spec = problem_hparams.target_modality if target_modality_name: _warn_changed_modality_type(target_modality_name, target_modality_spec[0], "target") target_modality_spec = (target_modality_name, target_modality_spec[1]) target_modality = registry.create_modality(target_modality_spec, hparams) problem_hparams.target_modality = target_modality
def _create_tpu_eval_metrics_fn(problem, hparams): """Create the metrics_fn that TPUEstimatorSpec expects.""" tm = problem.get_hparams().target_modality if isinstance(tm, tuple): tm = registry.create_modality(tm, hparams) weights_fn = tm.targets_weights_fn def make_metric_fn(metric_fn): def wrapped_metric_fn(logits, labels): num, den = metric_fn(logits, labels, weights_fn=weights_fn) return tf.metrics.mean(num, den) return wrapped_metric_fn metric_fns = [] eval_metrics = problem.eval_metrics() for metric in eval_metrics: if metric in TPU_METRIC_BLACKLIST: log_warn("Skipping eval metric %s in TPU_METRIC_BLACKLIST", metric) continue name = "metrics-%s/%s" % (problem.name, metric) metric_fns.append((name, make_metric_fn(metrics.METRICS_FNS[metric]))) def all_metrics_fn(logits=None, labels=None, **kwargs): """Construct metrics dictionary.""" metrics_dict = {} if logits is None: logits = kwargs for name, fn in metric_fns: if isinstance(logits, dict): for k, v in logits.iteritems(): metrics_dict["%s/%s" % (name, k)] = fn(v, labels) else: metrics_dict[name] = fn(logits, labels) return metrics_dict return all_metrics_fn
def create_evaluation_metrics(problems, model_hparams): """Creates the evaluation metrics for the model. Args: problems: List of Problem instances. model_hparams: a set of hparams. Returns: dict<metric name, metric function>. The metric functions have signature (Tensor predictions, features) -> (metric Tensor, update op), where features is a dict with keys {targets}. Raises: ValueError: if the metrics specified by a problem are not recognized (i.e. are not defined in the Metrics enum. """ def reduce_dimensions(predictions, labels): """Reduce dimensions for high-dimensional predictions and labels.""" # We will treat first dimensions as batch. One example are video frames. if len(predictions.get_shape()) > 5: predictions_shape = common_layers.shape_list(predictions) predictions = tf.reshape(predictions, [ predictions_shape[0], predictions_shape[1], -1, predictions_shape[-1] ]) labels_shape = common_layers.shape_list(labels) labels = tf.reshape(labels, [labels_shape[0], labels_shape[1], -1]) return predictions, labels def make_problem_specific_metric_fn(metric_fn, weights_fn): """Create a metric fn.""" def problem_metric_fn(predictions, features, labels): """Metric fn.""" # Send along the entire features dict if the metric fn has the kwarg # "features". kwargs = {} args, _, keywords, _ = inspect.getargspec(metric_fn) if ("features" in args) or keywords: kwargs["features"] = features predictions, labels = reduce_dimensions(predictions, labels) scores, weights = metric_fn(predictions, labels, weights_fn=weights_fn, **kwargs) return tf.metrics.mean(scores, weights) return problem_metric_fn def make_image_wrapped_metric_fn(metric_fn): """Metric fn without tf.metrics.mean.""" def image_wrapped_metric_fn(predictions, features, labels, weights_fn=common_layers.weights_all): del weights_fn del features predictions, labels = reduce_dimensions(predictions, labels) return metric_fn(predictions, labels, model_hparams) return image_wrapped_metric_fn def weights_fn_for_mp(problem_task_id): return lambda x: common_layers.weights_multi_problem( x, problem_task_id) eval_metrics = dict() for problem_instance in problems: problem_name = problem_instance.name metrics = problem_instance.eval_metrics() if hasattr(model_hparams.problem, "task_list"): metrics = model_hparams.problem.eval_metrics() if not all([m in METRICS_FNS for m in metrics]): error_str = ("Unrecognized metric. Problem %s specified metrics " "%s. Recognized metrics are %s.") raise ValueError(error_str % (problem_name, metrics, list(METRICS_FNS.keys()))) tm = problem_instance.get_hparams().target_modality if not isinstance(tm, dict): tm = {"targets": tm} for target_name, modality in six.iteritems(tm): if isinstance(modality, tuple): modality = registry.create_modality(modality, model_hparams) weights_fn = modality.targets_weights_fn if hasattr(model_hparams.problem, "task_list"): ptid = problem_instance.task_id # pylint: disable=cell-var-from-loop weights_fn = weights_fn_for_mp(ptid) for metric in metrics: metric_fn = METRICS_FNS[metric] metric_name = "metrics-%s/%s/%s" % (problem_name, target_name, metric) if metric == Metrics.IMAGE_SUMMARY: eval_metrics[metric_name] = make_image_wrapped_metric_fn( metric_fn) else: eval_metrics[ metric_name] = make_problem_specific_metric_fn( metric_fn, weights_fn) return eval_metrics
def create_evaluation_metrics(problems, model_hparams): """Creates the evaluation metrics for the model. Args: problems: List of Problem instances. model_hparams: a set of hparams. Returns: dict<metric name, metric function>. The metric functions have signature (Tensor predictions, features) -> (metric Tensor, update op), where features is a dict with keys {targets, problem_choice}. Raises: ValueError: if the metrics specified by a problem are not recognized (i.e. are not defined in the Metrics enum. """ def make_problem_specific_metric_fn(metric_fn, problem_idx, weights_fn): """Create a metric fn conditioned on problem_idx.""" def problem_metric_fn(predictions, features): """Metric fn.""" labels = features.get("targets", None) problem_choice = features.get("problem_choice", 0) # Send along the entire features dict if the metric fn has the kwarg # "features". kwargs = {} args, _, keywords, _ = inspect.getargspec(metric_fn) if ("features" in args) or keywords: kwargs["features"] = features def wrapped_metric_fn(): return metric_fn(predictions, labels, weights_fn=weights_fn, **kwargs) (scores, weights) = tf.cond( tf.equal(problem_idx, problem_choice), wrapped_metric_fn, lambda: (tf.constant(0.0), tf.constant(0.0))) # The tf.metrics.mean function assures correct aggregation. return tf.metrics.mean(scores, weights) return problem_metric_fn eval_metrics = dict() for problem_idx, problem_instance in enumerate(problems): problem_name = problem_instance.name metrics = problem_instance.eval_metrics() if not all([m in METRICS_FNS for m in metrics]): error_str = ("Unrecognized metric. Problem %s specified metrics " "%s. Recognized metrics are %s.") raise ValueError(error_str % (problem_name, metrics, list(METRICS_FNS.keys()))) def image_wrapped_metric_fn(predictions, labels, weights_fn=common_layers.weights_nonzero): _, _ = labels, weights_fn return metric_fn(predictions, model_hparams) tm = problem_instance.get_hparams().target_modality if isinstance(tm, tuple): tm = registry.create_modality(tm, model_hparams) weights_fn = tm.targets_weights_fn for metric in metrics: metric_fn = METRICS_FNS[metric] metric_name = "metrics-%s/%s" % (problem_name, metric) if metric == Metrics.IMAGE_SUMMARY: eval_metrics[metric_name] = image_wrapped_metric_fn else: problem_metric_fn = make_problem_specific_metric_fn( metric_fn, problem_idx, weights_fn) eval_metrics[metric_name] = problem_metric_fn return eval_metrics
def create_evaluation_metrics(problems, model_hparams): """Creates the evaluation metrics for the model. Args: problems: List of Problem instances. model_hparams: a set of hparams. Returns: dict<metric name, metric function>. The metric functions have signature (Tensor predictions, features) -> (metric Tensor, update op), where features is a dict with keys {targets}. Raises: ValueError: if the metrics specified by a problem are not recognized (i.e. are not defined in the Metrics enum. """ def reduce_dimensions(predictions, labels): """Reduce dimensions for high-dimensional predictions and labels.""" # We will treat first dimensions as batch. One example are video frames. if len(predictions.get_shape()) > 5: predictions = tf.reshape( predictions, [-1] + common_layers.shape_list(predictions)[-4:]) if len(labels.get_shape()) > 4: labels = tf.reshape(labels, [-1] + common_layers.shape_list(labels)[-3:]) return predictions, labels def make_problem_specific_metric_fn(metric_fn, weights_fn): """Create a metric fn.""" def problem_metric_fn(predictions, features, labels): """Metric fn.""" # Send along the entire features dict if the metric fn has the kwarg # "features". kwargs = {} args, _, keywords, _ = inspect.getargspec(metric_fn) if ("features" in args) or keywords: kwargs["features"] = features predictions, labels = reduce_dimensions(predictions, labels) scores, weights = metric_fn(predictions, labels, weights_fn=weights_fn, **kwargs) return tf.metrics.mean(scores, weights) return problem_metric_fn eval_metrics = dict() for problem_instance in problems: problem_name = problem_instance.name metrics = problem_instance.eval_metrics() if not all([m in METRICS_FNS for m in metrics]): error_str = ("Unrecognized metric. Problem %s specified metrics " "%s. Recognized metrics are %s.") raise ValueError(error_str % (problem_name, metrics, list(METRICS_FNS.keys()))) def image_wrapped_metric_fn(predictions, features, labels, weights_fn=common_layers.weights_all): del weights_fn del features predictions, labels = reduce_dimensions(predictions, labels) return metric_fn(predictions, labels, model_hparams) tm = problem_instance.get_hparams().target_modality if isinstance(tm, dict): for k, v in six.iteritems(tm): if isinstance(v, tuple): v = registry.create_modality(v, model_hparams) weights_fn = v.targets_weights_fn for metric in metrics: metric_fn = METRICS_FNS[metric] metric_name = "metrics-%s/%s/%s" % (problem_name, k, metric) if metric == Metrics.IMAGE_SUMMARY: eval_metrics[metric_name] = image_wrapped_metric_fn else: problem_metric_fn = make_problem_specific_metric_fn( metric_fn, weights_fn) eval_metrics[metric_name] = problem_metric_fn else: if isinstance(tm, tuple): tm = registry.create_modality(tm, model_hparams) weights_fn = tm.targets_weights_fn for metric in metrics: metric_fn = METRICS_FNS[metric] metric_name = "metrics-%s/%s" % (problem_name, metric) if metric == Metrics.IMAGE_SUMMARY: eval_metrics[metric_name] = image_wrapped_metric_fn else: problem_metric_fn = make_problem_specific_metric_fn( metric_fn, weights_fn) eval_metrics[metric_name] = problem_metric_fn return eval_metrics
def _create_tpu_eval_metrics_fn(problem, hparams): """Create the metrics_fn that TPUEstimatorSpec expects.""" metric_fns = [] eval_metrics = problem.eval_metrics() tm = problem.get_hparams().target_modality if isinstance(tm, dict): for k, v in six.iteritems(tm): if isinstance(v, tuple): v = registry.create_modality(v, hparams) weights_fn = v.targets_weights_fn def make_metric_fn(metric_fn): def wrapped_metric_fn(logits, labels, weights_fn=weights_fn): num, den = metric_fn(logits, labels, weights_fn=weights_fn) return tf.metrics.mean(num, den) return wrapped_metric_fn for metric in eval_metrics: if metric in TPU_METRIC_BLACKLIST: log_warn("Skipping eval metric %s in TPU_METRIC_BLACKLIST", metric) continue name = "%s/metrics-%s/%s" % (k, problem.name, metric) metric_fns.append((name, make_metric_fn(metrics.METRICS_FNS[metric]))) else: if isinstance(tm, tuple): tm = registry.create_modality(tm, hparams) weights_fn = tm.targets_weights_fn def make_metric_fn(metric_fn): def wrapped_metric_fn(logits, labels): num, den = metric_fn(logits, labels, weights_fn=weights_fn) return tf.metrics.mean(num, den) return wrapped_metric_fn for metric in eval_metrics: if metric in TPU_METRIC_BLACKLIST: log_warn("Skipping eval metric %s in TPU_METRIC_BLACKLIST", metric) continue name = "metrics-%s/%s" % (problem.name, metric) metric_fns.append((name, make_metric_fn(metrics.METRICS_FNS[metric]))) def all_metrics_fn(logits=None, labels=None, **kwargs): """Construct metrics dictionary.""" metrics_dict = {} if logits is None: logits = kwargs for name, fn in metric_fns: if isinstance(logits, dict): for k, v in six.iteritems(logits): if isinstance(labels, dict): metrics_dict["%s/%s" % (name, k)] = fn(v, labels[k]) else: metrics_dict["%s/%s" % (name, k)] = fn(v, labels) else: metrics_dict[name] = fn(logits, labels) return metrics_dict return all_metrics_fn
def create_evaluation_metrics(problems, model_hparams): """Creates the evaluation metrics for the model. Args: problems: List of Problem instances. model_hparams: a set of hparams. Returns: dict<metric name, metric function>. The metric functions have signature (Tensor predictions, features) -> (metric Tensor, update op), where features is a dict with keys {targets, problem_choice}. Raises: ValueError: if the metrics specified by a problem are not recognized (i.e. are not defined in the Metrics enum. """ def make_problem_specific_metric_fn(metric_fn, problem_idx, weights_fn): """Create a metric fn conditioned on problem_idx.""" def problem_metric_fn(predictions, features): """Metric fn.""" labels = features.get("targets", None) problem_choice = features.get("problem_choice", 0) # Send along the entire features dict if the metric fn has the kwarg # "features". kwargs = {} args, _, keywords, _ = inspect.getargspec(metric_fn) if ("features" in args) or keywords: kwargs["features"] = features def wrapped_metric_fn(): return metric_fn(predictions, labels, weights_fn=weights_fn, **kwargs) (scores, weights) = tf.cond(tf.equal(problem_idx, problem_choice), wrapped_metric_fn, lambda: (tf.constant(0.0), tf.constant(0.0))) # The tf.metrics.mean function assures correct aggregation. return tf.metrics.mean(scores, weights) return problem_metric_fn eval_metrics = dict() for problem_idx, problem_instance in enumerate(problems): problem_name = problem_instance.name metrics = problem_instance.eval_metrics() if not all([m in METRICS_FNS for m in metrics]): error_str = ("Unrecognized metric. Problem %s specified metrics " "%s. Recognized metrics are %s.") raise ValueError(error_str % (problem_name, metrics, list(METRICS_FNS.keys()))) def image_wrapped_metric_fn(predictions, labels, weights_fn=common_layers.weights_nonzero): _, _ = labels, weights_fn return metric_fn(predictions, model_hparams) tm = problem_instance.get_hparams().target_modality if isinstance(tm, tuple): tm = registry.create_modality(tm, model_hparams) weights_fn = tm.targets_weights_fn for metric in metrics: metric_fn = METRICS_FNS[metric] metric_name = "metrics-%s/%s" % (problem_name, metric) if metric == Metrics.IMAGE_SUMMARY: eval_metrics[metric_name] = image_wrapped_metric_fn else: problem_metric_fn = make_problem_specific_metric_fn( metric_fn, problem_idx, weights_fn) eval_metrics[metric_name] = problem_metric_fn return eval_metrics