def test_with_two_level_tuple(self):
dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes(
collections.OrderedDict([
('a', tf.bool),
('b',
collections.OrderedDict([
('c', tf.float32),
('d', (tf.int32, [20])),
])),
('e', ()),
]))
test.assert_nested_struct_eq(dtypes, {
'a': tf.bool,
'b': {
'c': tf.float32,
'd': tf.int32
},
'e': (),
})
test.assert_nested_struct_eq(
shapes, {
'a': tf.TensorShape([]),
'b': {
'c': tf.TensorShape([]),
'd': tf.TensorShape([20])
},
'e': (),
})
def test_with_two_level_tuple(self):
type_signature = computation_types.StructWithPythonType([
('a', tf.bool),
('b',
computation_types.StructWithPythonType([
('c', computation_types.TensorType(tf.float32)),
('d', computation_types.TensorType(tf.int32, [20])),
], collections.OrderedDict)),
('e', computation_types.StructType([])),
], collections.OrderedDict)
dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes(
type_signature)
test.assert_nested_struct_eq(dtypes, {
'a': tf.bool,
'b': {
'c': tf.float32,
'd': tf.int32
},
'e': (),
})
test.assert_nested_struct_eq(
shapes, {
'a': tf.TensorShape([]),
'b': {
'c': tf.TensorShape([]),
'd': tf.TensorShape([20])
},
'e': (),
})
def test_stamp_parameter_in_graph_with_int_vector_sequence(self):
with tf.Graph().as_default():
x = self._checked_stamp_parameter(
'foo', computation_types.SequenceType((tf.int32, [50])))
self.assertIsInstance(x, graph_utils.DATASET_REPRESENTATION_TYPES)
test.assert_nested_struct_eq(x.output_types, tf.int32)
test.assert_nested_struct_eq(x.output_shapes, tf.TensorShape([50]))
def test_stamp_parameter_in_graph_with_bool_sequence(self):
with tf.Graph().as_default():
x = self._checked_stamp_parameter(
'foo', computation_types.SequenceType(tf.bool))
self.assertIsInstance(x, graph_utils.DATASET_REPRESENTATION_TYPES)
test.assert_nested_struct_eq(tf.compat.v1.data.get_output_types(x),
tf.bool)
test.assert_nested_struct_eq(
tf.compat.v1.data.get_output_shapes(x), tf.TensorShape([]))
def test_with_tensor_triple(self):
tensor_specs = type_conversions.type_to_tf_tensor_specs(
collections.OrderedDict([('a', (tf.int32, [5])), ('b', tf.bool),
('c', (tf.float32, [3]))]))
test.assert_nested_struct_eq(
tensor_specs, {
'a': tf.TensorSpec([5], tf.int32),
'b': tf.TensorSpec([], tf.bool),
'c': tf.TensorSpec([3], tf.float32)
})
def _assert_binding_matches_type_and_value(self, binding, type_spec, val,
graph):
"""Asserts that 'bindings' matches the given type, value, and graph."""
self.assertIsInstance(binding, pb.TensorFlow.Binding)
self.assertIsInstance(type_spec, computation_types.Type)
binding_oneof = binding.WhichOneof('binding')
if binding_oneof == 'tensor':
self.assertTrue(tf.contrib.framework.is_tensor(val))
if not isinstance(val, tf.Variable):
# We insert a read_value() op for Variables, which produces
# a name we don't control. Otherwise, names should match:
self.assertEqual(binding.tensor.tensor_name, val.name)
self.assertIsInstance(type_spec, computation_types.TensorType)
self.assertEqual(type_spec.dtype, val.dtype.base_dtype)
self.assertEqual(repr(type_spec.shape), repr(val.shape))
elif binding_oneof == 'sequence':
self.assertIsInstance(val,
graph_utils.DATASET_REPRESENTATION_TYPES)
sequence_oneof = binding.sequence.WhichOneof('binding')
if sequence_oneof == 'iterator_string_handle_name':
# TODO(b/129956296): Eventually delete this deprecated code path.
handle = graph.get_tensor_by_name(
binding.sequence.iterator_string_handle_name)
self.assertIn(str(handle.op.type),
['Placeholder', 'IteratorToStringHandle'])
self.assertEqual(handle.dtype, tf.string)
else:
self.assertEqual(sequence_oneof, 'variant_tensor_name')
variant_tensor = graph.get_tensor_by_name(
binding.sequence.variant_tensor_name)
op = str(variant_tensor.op.type)
self.assertTrue((op == 'Placeholder') or ('Dataset' in op))
self.assertEqual(variant_tensor.dtype, tf.variant)
self.assertIsInstance(type_spec, computation_types.SequenceType)
output_dtypes, output_shapes = (
type_utils.type_to_tf_dtypes_and_shapes(type_spec.element))
test.assert_nested_struct_eq(
tf.compat.v1.data.get_output_types(val), output_dtypes)
test.assert_nested_struct_eq(
tf.compat.v1.data.get_output_shapes(val), output_shapes)
elif binding_oneof == 'tuple':
self.assertIsInstance(type_spec, computation_types.NamedTupleType)
if not isinstance(val,
(list, tuple, anonymous_tuple.AnonymousTuple)):
self.assertIsInstance(val, dict)
if isinstance(val, collections.OrderedDict):
val = list(val.values())
else:
val = [v for _, v in sorted(six.iteritems(val))]
for idx, e in enumerate(anonymous_tuple.to_elements(type_spec)):
self._assert_binding_matches_type_and_value(
binding.tuple.element[idx], e[1], val[idx], graph)
else:
self.fail('Unknown binding.')
def test_type_to_tf_structure_without_names(self):
type_spec = computation_types.to_type((tf.bool, tf.int32))
dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes(type_spec)
structure = type_utils.type_to_tf_structure(type_spec)
with tf.Graph().as_default():
ds = tf.data.experimental.from_variant(
tf.placeholder(tf.variant, shape=[]), structure=structure)
ds_dtypes = tf.compat.v1.data.get_output_types(ds)
ds_shapes = tf.compat.v1.data.get_output_shapes(ds)
test.assert_nested_struct_eq(ds_dtypes, dtypes)
test.assert_nested_struct_eq(ds_shapes, shapes)
def test_with_tensor_triple(self):
type_signature = computation_types.StructWithPythonType([
('a', computation_types.TensorType(tf.int32, [5])),
('b', computation_types.TensorType(tf.bool)),
('c', computation_types.TensorType(tf.float32, [3])),
], collections.OrderedDict)
tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature)
test.assert_nested_struct_eq(
tensor_specs, {
'a': tf.TensorSpec([5], tf.int32),
'b': tf.TensorSpec([], tf.bool),
'c': tf.TensorSpec([3], tf.float32)
})
def test_with_tensor_triple(self):
dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes(
collections.OrderedDict([('a', (tf.int32, [5])), ('b', tf.bool),
('c', (tf.float32, [3]))]))
test.assert_nested_struct_eq(dtypes, {
'a': tf.int32,
'b': tf.bool,
'c': tf.float32
})
test.assert_nested_struct_eq(shapes, {
'a': tf.TensorShape([5]),
'b': tf.TensorShape([]),
'c': tf.TensorShape([3])
})
def test_type_to_tf_dtypes_and_shapes_with_tensor_triple(self):
dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes([
('a', (tf.int32, [5])), ('b', tf.bool), ('c', (tf.float32, [3]))
])
test.assert_nested_struct_eq(dtypes, {
'a': tf.int32,
'b': tf.bool,
'c': tf.float32
})
test.assert_nested_struct_eq(
shapes, {
'a': tf.TensorShape([5]),
'b': tf.TensorShape([]),
'c': tf.TensorShape([3])
})
def test_stamp_parameter_in_graph_with_tensor_pair_sequence(self):
with tf.Graph().as_default():
x = self._checked_stamp_parameter(
'foo',
computation_types.SequenceType([('A', (tf.float32, [3, 4, 5])),
('B', (tf.int32, [1]))]))
self.assertIsInstance(x, graph_utils.DATASET_REPRESENTATION_TYPES)
test.assert_nested_struct_eq(x.output_types, {
'A': tf.float32,
'B': tf.int32
})
test.assert_nested_struct_eq(x.output_shapes, {
'A': tf.TensorShape([3, 4, 5]),
'B': tf.TensorShape([1])
})
def test_stamp_parameter_in_graph_with_tensor_ordered_dict_sequence(self):
with tf.Graph().as_default():
x = self._checked_stamp_parameter(
'foo',
computation_types.SequenceType(
collections.OrderedDict([('A', (tf.float32, [3, 4, 5])),
('B', (tf.int32, [1]))])))
self.assertIsInstance(x, tensorflow_utils.DATASET_REPRESENTATION_TYPES)
test.assert_nested_struct_eq(
tf.compat.v1.data.get_output_types(x), {
'A': tf.float32,
'B': tf.int32
})
test.assert_nested_struct_eq(
tf.compat.v1.data.get_output_shapes(x), {
'A': tf.TensorShape([3, 4, 5]),
'B': tf.TensorShape([1])
})
def test_with_tensor_triple(self):
type_signature = computation_types.NamedTupleTypeWithPyContainerType([
('a', computation_types.TensorType(tf.int32, [5])),
('b', computation_types.TensorType(tf.bool)),
('c', computation_types.TensorType(tf.float32, [3])),
], collections.OrderedDict)
dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes(
type_signature)
test.assert_nested_struct_eq(dtypes, {
'a': tf.int32,
'b': tf.bool,
'c': tf.float32
})
test.assert_nested_struct_eq(shapes, {
'a': tf.TensorShape([5]),
'b': tf.TensorShape([]),
'c': tf.TensorShape([3])
})
def test_type_to_tf_dtypes_and_shapes_with_two_level_tuple(self):
dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes([
('a', tf.bool), ('b', [('c', tf.float32), ('d', (tf.int32, [20]))])
])
test.assert_nested_struct_eq(dtypes, {
'a': tf.bool,
'b': {
'c': tf.float32,
'd': tf.int32
}
})
test.assert_nested_struct_eq(
shapes, {
'a': tf.TensorShape([]),
'b': {
'c': tf.TensorShape([]),
'd': tf.TensorShape([20])
}
})
def test_type_to_tf_structure_with_names(self):
type_spec = computation_types.to_type(
collections.OrderedDict([
('a', tf.bool),
('b',
collections.OrderedDict([
('c', tf.float32),
('d', (tf.int32, [20])),
])),
]))
dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes(type_spec)
structure = type_utils.type_to_tf_structure(type_spec)
with tf.Graph().as_default():
ds = tf.data.experimental.from_variant(
tf.placeholder(tf.variant, shape=[]), structure=structure)
ds_dtypes = tf.compat.v1.data.get_output_types(ds)
ds_shapes = tf.compat.v1.data.get_output_shapes(ds)
test.assert_nested_struct_eq(ds_dtypes, dtypes)
test.assert_nested_struct_eq(ds_shapes, shapes)
def test_with_two_level_tuple(self):
type_signature = computation_types.NamedTupleTypeWithPyContainerType([
('a', tf.bool),
('b',
computation_types.NamedTupleTypeWithPyContainerType([
('c', computation_types.TensorType(tf.float32)),
('d', computation_types.TensorType(tf.int32, [20])),
], collections.OrderedDict)),
('e', computation_types.NamedTupleType([])),
], collections.OrderedDict)
tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature)
test.assert_nested_struct_eq(
tensor_specs, {
'a': tf.TensorSpec([], tf.bool),
'b': {
'c': tf.TensorSpec([], tf.float32),
'd': tf.TensorSpec([20], tf.int32)
},
'e': (),
})
def test_with_two_level_tuple(self):
tensor_specs = type_conversions.type_to_tf_tensor_specs(
collections.OrderedDict([
('a', tf.bool),
('b',
collections.OrderedDict([
('c', tf.float32),
('d', (tf.int32, [20])),
])),
('e', ()),
]))
test.assert_nested_struct_eq(
tensor_specs, {
'a': tf.TensorSpec([], tf.bool),
'b': {
'c': tf.TensorSpec([], tf.float32),
'd': tf.TensorSpec([20], tf.int32)
},
'e': (),
})
def test_with_unnamed_element(self):
type_signature = computation_types.StructType([tf.int32])
tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature)
test.assert_nested_struct_eq(tensor_specs,
(tf.TensorSpec([], tf.int32), ))
def test_with_int_vector(self):
type_signature = computation_types.TensorType(tf.int32, [10])
tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature)
test.assert_nested_struct_eq(tensor_specs, tf.TensorSpec([10],
tf.int32))
def test_with_int_scalar(self):
dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes(
tf.int32)
test.assert_nested_struct_eq(dtypes, tf.int32)
test.assert_nested_struct_eq(shapes, tf.TensorShape([]))
def test_with_int_vector(self):
type_signature = computation_types.TensorType(tf.int32, [10])
dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes(
type_signature)
test.assert_nested_struct_eq(dtypes, tf.int32)
test.assert_nested_struct_eq(shapes, tf.TensorShape([10]))
def test_with_int_scalar(self):
tensor_specs = type_conversions.type_to_tf_tensor_specs(tf.int32)
test.assert_nested_struct_eq(tensor_specs, tf.TensorSpec([], tf.int32))
def test_with_int_vector(self):
tensor_specs = type_conversions.type_to_tf_tensor_specs(
(tf.int32, [10]))
test.assert_nested_struct_eq(tensor_specs, tf.TensorSpec([10],
tf.int32))
def test_nested_structures_are_same_where_they_are_same(self):
test.assert_nested_struct_eq({'a': 10}, {'a': 10})
def test_type_to_tf_dtypes_and_shapes_with_int_vector(self):
dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes(
(tf.int32, [10]))
test.assert_nested_struct_eq(dtypes, tf.int32)
test.assert_nested_struct_eq(shapes, tf.TensorShape([10]))
def test_nested_structures_are_same_where_nesting_differs(self):
with self.assertRaises(ValueError):
test.assert_nested_struct_eq({'a': 10}, 10)
def test_nested_structures_are_same_where_values_differ(self):
with self.assertRaises(ValueError):
test.assert_nested_struct_eq({'a': 10}, {'a': False})
def test_nested_structures_are_same_where_values_differ(self):
with self.assertRaises(ValueError): # pylint: disable=g-error-prone-assert-raises
test.assert_nested_struct_eq({'a': 10}, {'a': False})
def test_with_unnamed_element(self):
tensor_specs = type_conversions.type_to_tf_tensor_specs(
computation_types.NamedTupleType([tf.int32]))
test.assert_nested_struct_eq(tensor_specs,
(tf.TensorSpec([], tf.int32), ))
