def test_pack_flat_sequence_to_spec_structure(self): subset_placeholders = utils.make_placeholders(mock_nested_subset_spec) flattened_subset_placeholders = utils.flatten_spec_structure( subset_placeholders) packed_subset_placeholders = utils.pack_flat_sequence_to_spec_structure( mock_nested_subset_spec, flattened_subset_placeholders) utils.assert_equal(subset_placeholders, packed_subset_placeholders) utils.assert_equal(mock_nested_subset_spec, packed_subset_placeholders, ignore_batch=True) placeholders = utils.make_placeholders(mock_nested_spec) flattened_placeholders = utils.flatten_spec_structure(placeholders) packed_placeholders = utils.pack_flat_sequence_to_spec_structure( mock_nested_subset_spec, flattened_placeholders) # We only subselect what we need in pack_flat_sequence_to_spec_structure, # hence, we should recover what we wanted. utils.assert_equal(mock_nested_subset_spec, packed_placeholders, ignore_batch=True) utils.assert_equal(subset_placeholders, packed_placeholders) packed_optional_placeholders = utils.pack_flat_sequence_to_spec_structure( mock_nested_optional_spec, flattened_placeholders) # Although mock_nested_optional_spec would like more tensors # flattened_placeholders cannot provide them, fortunately they are optional. utils.assert_required(packed_optional_placeholders, placeholders) utils.assert_required(mock_nested_spec, packed_optional_placeholders, ignore_batch=True)
def test_validate_flatten_and_pack(self): # An example data pipeline. # Some input generator creates input features according to some spec. input_features = utils.make_placeholders(mock_nested_spec) # Assume a preprocessor has altered these input_features and we want # to pass the data on to the next stage, then we simply assure that # our output is according to our spec and flatten. flat_input_features = utils.validate_and_flatten( mock_nested_optional_spec, input_features, ignore_batch=True) utils.assert_required( mock_nested_optional_spec, input_features, ignore_batch=True) # Then e.g. the model_fn receives the flat_input_spec and validates # that it is according to it's requirements and packs it back into the # spec structure. output_features = utils.validate_and_pack( mock_nested_subset_spec, flat_input_features, ignore_batch=True) utils.assert_required( mock_nested_subset_spec, output_features, ignore_batch=True)
def test_assert_required(self): # A subset of the required spec is asked for, that should work. utils.assert_required(mock_nested_subset_spec, mock_nested_spec) # We would like to have more, but all additional specs are optional. utils.assert_required(mock_nested_optional_spec, mock_nested_spec) # We ask for more required tensor_specs than we actually have. with self.assertRaises(ValueError): utils.assert_required(mock_nested_spec, mock_nested_subset_spec)