def _add_input(
self, accumulator: tf_metric_accumulators.TFMetricsAccumulator,
element: metric_types.StandardMetricInputs
) -> tf_metric_accumulators.TFMetricsAccumulator:
for i, output_name in enumerate(self._output_names):
if not output_name and len(self._output_names) > 1:
# The first output_name for multi-output models is '' and is used to
# store combined metric weights for all outputs, but is not for labels
# and example weights.
labels, example_weights = None, None
else:
labels, _, example_weights = next(
metric_util.to_label_prediction_example_weight(
element,
self._eval_config,
self._model_name,
output_name,
flatten=False,
example_weighted=True))
if i == 0:
if element.transformed_features:
features = {}
features.update(element.features)
features.update(element.transformed_features)
else:
features = element.features
else:
features = None
accumulator.add_input(i, features, labels, example_weights)
return accumulator
def _update_state(
self, accumulator: tf_metric_accumulators.TFMetricsAccumulator):
features = {}
labels = {}
example_weights = {}
for i, output_name in enumerate(self._output_names):
f, l, w = accumulator.get_inputs(i)
if i == 0:
features = util.merge_extracts(f)
if not output_name and len(self._output_names) > 1:
# The empty output_name for multi-output models is not used for inputs.
continue
labels[output_name] = np.array(l)
weights = np.array(w)
# TFv1 will not squeeze the weights, so must do manually
if weights.shape[-1] == 1:
weights = weights.squeeze(axis=-1)
example_weights[output_name] = weights
if len(self._output_names) == 1:
# Single-output models don't use dicts.
labels = next(iter(labels.values()))
example_weights = next(iter(example_weights.values()))
input_specs = model_util.get_input_specs(self._model,
signature_name=None)
inputs = model_util.get_inputs(features, input_specs)
if inputs is None:
raise ValueError('unable to prepare inputs for evaluation: '
f'input_specs={input_specs}, features={features}')
self._model.evaluate(x=inputs,
y=labels,
batch_size=util.batch_size(features),
verbose=0,
sample_weight=example_weights)
def add_input(
self, accumulator: tf_metric_accumulators.TFMetricsAccumulator,
element: metric_types.StandardMetricInputs
) -> tf_metric_accumulators.TFMetricsAccumulator:
accumulator = self._add_input(accumulator, element)
if accumulator.should_flush():
self._process_batch(accumulator)
return accumulator
def extract_output(
self, accumulator: tf_metric_accumulators.TFMetricsAccumulator
) -> Dict[metric_types.MetricKey, np.ndarray]:
# Finish processing last batch
self._process_batch(accumulator)
result = {}
for metric_index, metric in enumerate(self._metrics()):
key = self._keys[metric_index]
weights = accumulator.get_weights(0, metric_index)
if weights is not None:
metric.set_weights(weights)
else:
metric.reset_states()
result[key] = metric.result().numpy()
return result
def _process_batch(
self, accumulator: tf_metric_accumulators.TFMetricsAccumulator):
if accumulator.len_inputs() == 0:
return
self._batch_size_beam_metric_dist.update(accumulator.len_inputs())
self._total_input_byte_size_beam_metric_dist.update(
accumulator.get_size_estimate())
for metric_index, metric in enumerate(self._metrics()):
metric.reset_states()
self._update_state(accumulator)
# For metrics stored with the model, the outputs get encoded in the
# metric names so we will use a single output for the weights and parse the
# names at the end to separate metrics by output.
for metric_index, metric in enumerate(self._metrics()):
accumulator.add_weights(0, metric_index, metric.get_weights())
accumulator.clear_inputs()
def _update_state(
self, accumulator: tf_metric_accumulators.TFMetricsAccumulator):
serialized_examples = None
labels = {}
example_weights = {}
for i, output_name in enumerate(self._output_names):
e, l, w = accumulator.get_inputs(i)
if i == 0:
serialized_examples = e
if not output_name and len(self._output_names) > 1:
# The empty output_name for multi-output models is not used for inputs.
continue
labels[output_name] = np.array(l)
weights = np.array(w)
# TFv1 will not squeeze the weights, so must do manually
if weights.shape[-1] == 1:
weights = weights.squeeze(axis=-1)
example_weights[output_name] = weights
if len(self._output_names) == 1:
# Single-output models don't use dicts.
labels = next(iter(labels.values()))
example_weights = next(iter(example_weights.values()))
record_batch = self._decoder.DecodeBatch(serialized_examples)
input_specs = model_util.get_input_specs(self._model,
signature_name=None)
inputs = model_util.get_inputs(record_batch, input_specs,
self._tensor_adapter)
if inputs is None:
raise ValueError('unable to prepare inputs for evaluation: '
'input_specs={}, record_batch={}'.format(
input_specs, record_batch))
self._model.evaluate(x=inputs,
y=labels,
batch_size=record_batch.num_rows,
verbose=0,
sample_weight=example_weights)