def preprocess(
self, features,
labels, mode
):
"""The function which preprocesses the features and labels per example.
Note, this function performs the boilerplate packing and flattening and
verification of the features and labels according to our spec. The actual
preprocessing is performed by _preprocess_fn.
Args:
features: The features of a single example.
labels: (Optional None) The labels of a single example.
mode: (ModeKeys) Specifies if this is training, evaluation or prediction.
Returns:
features_preprocessed: The preprocessed and flattened features
verified to fulfill our output specs.
labels_preprocessed: (Optional None) The preprocessed and flattened labels
verified to fulfill our output specs.
"""
# First, we verify that the input features and labels fulfill our spec.
# We further pack the flattened features and labels to our (hierarchical)
# specification.:
features = tensorspec_utils.validate_and_pack(
expected_spec=self.get_in_feature_specification(mode),
actual_tensors_or_spec=features,
ignore_batch=True)
if labels is not None:
labels = tensorspec_utils.validate_and_pack(
expected_spec=self.get_in_label_specification(mode),
actual_tensors_or_spec=labels,
ignore_batch=True)
features_preprocessed, labels_preprocessed = self._preprocess_fn(
features=features, labels=labels, mode=mode)
features_preprocessed = tensorspec_utils.validate_and_flatten(
expected_spec=self.get_out_feature_specification(mode),
actual_tensors_or_spec=features_preprocessed,
ignore_batch=True)
if labels_preprocessed:
labels_preprocessed = tensorspec_utils.validate_and_flatten(
expected_spec=self.get_out_label_specification(mode),
actual_tensors_or_spec=labels_preprocessed,
ignore_batch=True)
return features_preprocessed, labels_preprocessed
def test_validate_flatten_and_pack(self):
# An example data pipeline.
# Some input generator creates input features according to some spec.
input_features = utils.make_placeholders(mock_nested_spec)
# Assume a preprocessor has altered these input_features and we want
# to pass the data on to the next stage, then we simply assure that
# our output is according to our spec and flatten.
flat_input_features = utils.validate_and_flatten(
mock_nested_optional_spec, input_features, ignore_batch=True)
utils.assert_required(
mock_nested_optional_spec, input_features, ignore_batch=True)
# Then e.g. the model_fn receives the flat_input_spec and validates
# that it is according to it's requirements and packs it back into the
# spec structure.
output_features = utils.validate_and_pack(
mock_nested_subset_spec, flat_input_features, ignore_batch=True)
utils.assert_required(
mock_nested_subset_spec, output_features, ignore_batch=True)