def add_input( self, accumulator: _CompilableMetricsAccumulator, element: metric_types.StandardMetricInputs ) -> _CompilableMetricsAccumulator: for i, output_name in enumerate(self._output_names): # The use of class_weights means that micro averaging is being used. When # micro averaging is being used, flatten should be set to True so that # each class is treated as though it was an independent example. for label, prediction, example_weight in ( metric_util.to_label_prediction_example_weight( element, eval_config=self._eval_config, model_name=self._model_name, output_name=output_name, # Skip top_k processing and let keras perform top_k calculations sub_key=self._sub_key if not self._is_top_k() else None, class_weights=self._class_weights, flatten=self._class_weights is not None)): # Keras requires non-sparse keys for top_k calcuations. if self._is_top_k() and label.shape != prediction.shape: label = metric_util.one_hot(label, prediction) accumulator.add_input(i, label, prediction, example_weight) if (accumulator.len_inputs() >= self._batch_size or accumulator.total_input_byte_size >= self._TOTAL_INPUT_BYTE_SIZE_THRESHOLD): self._process_batch(accumulator) return accumulator
def add_input( self, accumulator: tf_metric_accumulators.TFCompilableMetricsAccumulator, element: metric_types.StandardMetricInputs ) -> tf_metric_accumulators.TFCompilableMetricsAccumulator: for i, output_name in enumerate(self._output_names): # When micro averaging is being used, flatten should be set to True so # that each class is treated as though it was an independent example. micro_average = ( self._aggregation_type and self._aggregation_type.micro_average) for label, prediction, example_weight in ( metric_util.to_label_prediction_example_weight( element, eval_config=self._eval_config, model_name=self._model_name, output_name=output_name, # Skip sub_key processing if part of the keras config sub_key=self._sub_key if not self._sub_key_in_config else None, aggregation_type=self._aggregation_type, class_weights=self._class_weights, flatten=micro_average)): # Keras requires non-sparse keys for its calcuations. if self._sub_key_in_config and label.shape != prediction.shape: label = metric_util.one_hot(label, prediction) accumulator.add_input(i, label, prediction, example_weight) if accumulator.should_flush(): self._process_batch(accumulator) return accumulator
def add_input(self, accumulator: _Matrices, element: metric_types.StandardMetricInputs) -> _Matrices: labels, predictions, example_weight = next( metric_util.to_label_prediction_example_weight( element, eval_config=self._eval_config, model_name=self._key.model_name, output_name=self._key.output_name, flatten=False, require_single_example_weight=True)) if not labels.shape: raise ValueError( 'Labels missing from example: StandardMetricInputs={}'.format( element)) if predictions.shape in ((), (1, )): raise ValueError( 'Predictions shape must be > 1 for multi-label confusion matrix: ' 'shape={}, StandardMetricInputs={}'.format( predictions.shape, element)) # If the label and prediction shapes are different then assume the labels # are sparse and convert them to dense. if (len(labels.shape) != len(predictions.shape) or labels.shape[-1] != predictions.shape[-1]): labels = metric_util.one_hot(labels, predictions) example_weight = float(example_weight) for threshold in self._thresholds: if threshold not in accumulator: accumulator[threshold] = {} for actual_class_id, label in enumerate(labels): if not label: continue for class_id, prediction in enumerate(predictions): matrix_key = _MatrixEntryKey(actual_class_id, class_id) fn = (labels[class_id] and prediction <= threshold) * example_weight fp = (not labels[class_id] and prediction > threshold) * example_weight tn = ((not labels[class_id] and prediction <= threshold) * example_weight) tp = (labels[class_id] and prediction > threshold) * example_weight if matrix_key in accumulator[threshold]: accumulator[threshold][ matrix_key].false_negatives += fn accumulator[threshold][matrix_key].true_negatives += tn accumulator[threshold][ matrix_key].false_positives += fp accumulator[threshold][matrix_key].true_positives += tp else: matrix = _ConfusionMatrix() matrix.false_negatives = fn matrix.true_negatives = tn matrix.false_positives = fp matrix.true_positives = tp accumulator[threshold][matrix_key] = matrix return accumulator