def test_with_two_level_tuple(self): dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes( collections.OrderedDict([ ('a', tf.bool), ('b', collections.OrderedDict([ ('c', tf.float32), ('d', (tf.int32, [20])), ])), ('e', ()), ])) test.assert_nested_struct_eq(dtypes, { 'a': tf.bool, 'b': { 'c': tf.float32, 'd': tf.int32 }, 'e': (), }) test.assert_nested_struct_eq( shapes, { 'a': tf.TensorShape([]), 'b': { 'c': tf.TensorShape([]), 'd': tf.TensorShape([20]) }, 'e': (), })
def test_with_two_level_tuple(self): type_signature = computation_types.StructWithPythonType([ ('a', tf.bool), ('b', computation_types.StructWithPythonType([ ('c', computation_types.TensorType(tf.float32)), ('d', computation_types.TensorType(tf.int32, [20])), ], collections.OrderedDict)), ('e', computation_types.StructType([])), ], collections.OrderedDict) dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes( type_signature) test.assert_nested_struct_eq(dtypes, { 'a': tf.bool, 'b': { 'c': tf.float32, 'd': tf.int32 }, 'e': (), }) test.assert_nested_struct_eq( shapes, { 'a': tf.TensorShape([]), 'b': { 'c': tf.TensorShape([]), 'd': tf.TensorShape([20]) }, 'e': (), })
def test_stamp_parameter_in_graph_with_int_vector_sequence(self): with tf.Graph().as_default(): x = self._checked_stamp_parameter( 'foo', computation_types.SequenceType((tf.int32, [50]))) self.assertIsInstance(x, graph_utils.DATASET_REPRESENTATION_TYPES) test.assert_nested_struct_eq(x.output_types, tf.int32) test.assert_nested_struct_eq(x.output_shapes, tf.TensorShape([50]))
def test_stamp_parameter_in_graph_with_bool_sequence(self): with tf.Graph().as_default(): x = self._checked_stamp_parameter( 'foo', computation_types.SequenceType(tf.bool)) self.assertIsInstance(x, graph_utils.DATASET_REPRESENTATION_TYPES) test.assert_nested_struct_eq(tf.compat.v1.data.get_output_types(x), tf.bool) test.assert_nested_struct_eq( tf.compat.v1.data.get_output_shapes(x), tf.TensorShape([]))
def test_with_tensor_triple(self): tensor_specs = type_conversions.type_to_tf_tensor_specs( collections.OrderedDict([('a', (tf.int32, [5])), ('b', tf.bool), ('c', (tf.float32, [3]))])) test.assert_nested_struct_eq( tensor_specs, { 'a': tf.TensorSpec([5], tf.int32), 'b': tf.TensorSpec([], tf.bool), 'c': tf.TensorSpec([3], tf.float32) })
def _assert_binding_matches_type_and_value(self, binding, type_spec, val, graph): """Asserts that 'bindings' matches the given type, value, and graph.""" self.assertIsInstance(binding, pb.TensorFlow.Binding) self.assertIsInstance(type_spec, computation_types.Type) binding_oneof = binding.WhichOneof('binding') if binding_oneof == 'tensor': self.assertTrue(tf.contrib.framework.is_tensor(val)) if not isinstance(val, tf.Variable): # We insert a read_value() op for Variables, which produces # a name we don't control. Otherwise, names should match: self.assertEqual(binding.tensor.tensor_name, val.name) self.assertIsInstance(type_spec, computation_types.TensorType) self.assertEqual(type_spec.dtype, val.dtype.base_dtype) self.assertEqual(repr(type_spec.shape), repr(val.shape)) elif binding_oneof == 'sequence': self.assertIsInstance(val, graph_utils.DATASET_REPRESENTATION_TYPES) sequence_oneof = binding.sequence.WhichOneof('binding') if sequence_oneof == 'iterator_string_handle_name': # TODO(b/129956296): Eventually delete this deprecated code path. handle = graph.get_tensor_by_name( binding.sequence.iterator_string_handle_name) self.assertIn(str(handle.op.type), ['Placeholder', 'IteratorToStringHandle']) self.assertEqual(handle.dtype, tf.string) else: self.assertEqual(sequence_oneof, 'variant_tensor_name') variant_tensor = graph.get_tensor_by_name( binding.sequence.variant_tensor_name) op = str(variant_tensor.op.type) self.assertTrue((op == 'Placeholder') or ('Dataset' in op)) self.assertEqual(variant_tensor.dtype, tf.variant) self.assertIsInstance(type_spec, computation_types.SequenceType) output_dtypes, output_shapes = ( type_utils.type_to_tf_dtypes_and_shapes(type_spec.element)) test.assert_nested_struct_eq( tf.compat.v1.data.get_output_types(val), output_dtypes) test.assert_nested_struct_eq( tf.compat.v1.data.get_output_shapes(val), output_shapes) elif binding_oneof == 'tuple': self.assertIsInstance(type_spec, computation_types.NamedTupleType) if not isinstance(val, (list, tuple, anonymous_tuple.AnonymousTuple)): self.assertIsInstance(val, dict) if isinstance(val, collections.OrderedDict): val = list(val.values()) else: val = [v for _, v in sorted(six.iteritems(val))] for idx, e in enumerate(anonymous_tuple.to_elements(type_spec)): self._assert_binding_matches_type_and_value( binding.tuple.element[idx], e[1], val[idx], graph) else: self.fail('Unknown binding.')
def test_type_to_tf_structure_without_names(self): type_spec = computation_types.to_type((tf.bool, tf.int32)) dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes(type_spec) structure = type_utils.type_to_tf_structure(type_spec) with tf.Graph().as_default(): ds = tf.data.experimental.from_variant( tf.placeholder(tf.variant, shape=[]), structure=structure) ds_dtypes = tf.compat.v1.data.get_output_types(ds) ds_shapes = tf.compat.v1.data.get_output_shapes(ds) test.assert_nested_struct_eq(ds_dtypes, dtypes) test.assert_nested_struct_eq(ds_shapes, shapes)
def test_with_tensor_triple(self): type_signature = computation_types.StructWithPythonType([ ('a', computation_types.TensorType(tf.int32, [5])), ('b', computation_types.TensorType(tf.bool)), ('c', computation_types.TensorType(tf.float32, [3])), ], collections.OrderedDict) tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature) test.assert_nested_struct_eq( tensor_specs, { 'a': tf.TensorSpec([5], tf.int32), 'b': tf.TensorSpec([], tf.bool), 'c': tf.TensorSpec([3], tf.float32) })
def test_with_tensor_triple(self): dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes( collections.OrderedDict([('a', (tf.int32, [5])), ('b', tf.bool), ('c', (tf.float32, [3]))])) test.assert_nested_struct_eq(dtypes, { 'a': tf.int32, 'b': tf.bool, 'c': tf.float32 }) test.assert_nested_struct_eq(shapes, { 'a': tf.TensorShape([5]), 'b': tf.TensorShape([]), 'c': tf.TensorShape([3]) })
def test_type_to_tf_dtypes_and_shapes_with_tensor_triple(self): dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes([ ('a', (tf.int32, [5])), ('b', tf.bool), ('c', (tf.float32, [3])) ]) test.assert_nested_struct_eq(dtypes, { 'a': tf.int32, 'b': tf.bool, 'c': tf.float32 }) test.assert_nested_struct_eq( shapes, { 'a': tf.TensorShape([5]), 'b': tf.TensorShape([]), 'c': tf.TensorShape([3]) })
def test_stamp_parameter_in_graph_with_tensor_pair_sequence(self): with tf.Graph().as_default(): x = self._checked_stamp_parameter( 'foo', computation_types.SequenceType([('A', (tf.float32, [3, 4, 5])), ('B', (tf.int32, [1]))])) self.assertIsInstance(x, graph_utils.DATASET_REPRESENTATION_TYPES) test.assert_nested_struct_eq(x.output_types, { 'A': tf.float32, 'B': tf.int32 }) test.assert_nested_struct_eq(x.output_shapes, { 'A': tf.TensorShape([3, 4, 5]), 'B': tf.TensorShape([1]) })
def test_stamp_parameter_in_graph_with_tensor_ordered_dict_sequence(self): with tf.Graph().as_default(): x = self._checked_stamp_parameter( 'foo', computation_types.SequenceType( collections.OrderedDict([('A', (tf.float32, [3, 4, 5])), ('B', (tf.int32, [1]))]))) self.assertIsInstance(x, tensorflow_utils.DATASET_REPRESENTATION_TYPES) test.assert_nested_struct_eq( tf.compat.v1.data.get_output_types(x), { 'A': tf.float32, 'B': tf.int32 }) test.assert_nested_struct_eq( tf.compat.v1.data.get_output_shapes(x), { 'A': tf.TensorShape([3, 4, 5]), 'B': tf.TensorShape([1]) })
def test_with_tensor_triple(self): type_signature = computation_types.NamedTupleTypeWithPyContainerType([ ('a', computation_types.TensorType(tf.int32, [5])), ('b', computation_types.TensorType(tf.bool)), ('c', computation_types.TensorType(tf.float32, [3])), ], collections.OrderedDict) dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes( type_signature) test.assert_nested_struct_eq(dtypes, { 'a': tf.int32, 'b': tf.bool, 'c': tf.float32 }) test.assert_nested_struct_eq(shapes, { 'a': tf.TensorShape([5]), 'b': tf.TensorShape([]), 'c': tf.TensorShape([3]) })
def test_type_to_tf_dtypes_and_shapes_with_two_level_tuple(self): dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes([ ('a', tf.bool), ('b', [('c', tf.float32), ('d', (tf.int32, [20]))]) ]) test.assert_nested_struct_eq(dtypes, { 'a': tf.bool, 'b': { 'c': tf.float32, 'd': tf.int32 } }) test.assert_nested_struct_eq( shapes, { 'a': tf.TensorShape([]), 'b': { 'c': tf.TensorShape([]), 'd': tf.TensorShape([20]) } })
def test_type_to_tf_structure_with_names(self): type_spec = computation_types.to_type( collections.OrderedDict([ ('a', tf.bool), ('b', collections.OrderedDict([ ('c', tf.float32), ('d', (tf.int32, [20])), ])), ])) dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes(type_spec) structure = type_utils.type_to_tf_structure(type_spec) with tf.Graph().as_default(): ds = tf.data.experimental.from_variant( tf.placeholder(tf.variant, shape=[]), structure=structure) ds_dtypes = tf.compat.v1.data.get_output_types(ds) ds_shapes = tf.compat.v1.data.get_output_shapes(ds) test.assert_nested_struct_eq(ds_dtypes, dtypes) test.assert_nested_struct_eq(ds_shapes, shapes)
def test_with_two_level_tuple(self): type_signature = computation_types.NamedTupleTypeWithPyContainerType([ ('a', tf.bool), ('b', computation_types.NamedTupleTypeWithPyContainerType([ ('c', computation_types.TensorType(tf.float32)), ('d', computation_types.TensorType(tf.int32, [20])), ], collections.OrderedDict)), ('e', computation_types.NamedTupleType([])), ], collections.OrderedDict) tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature) test.assert_nested_struct_eq( tensor_specs, { 'a': tf.TensorSpec([], tf.bool), 'b': { 'c': tf.TensorSpec([], tf.float32), 'd': tf.TensorSpec([20], tf.int32) }, 'e': (), })
def test_with_two_level_tuple(self): tensor_specs = type_conversions.type_to_tf_tensor_specs( collections.OrderedDict([ ('a', tf.bool), ('b', collections.OrderedDict([ ('c', tf.float32), ('d', (tf.int32, [20])), ])), ('e', ()), ])) test.assert_nested_struct_eq( tensor_specs, { 'a': tf.TensorSpec([], tf.bool), 'b': { 'c': tf.TensorSpec([], tf.float32), 'd': tf.TensorSpec([20], tf.int32) }, 'e': (), })
def test_with_unnamed_element(self): type_signature = computation_types.StructType([tf.int32]) tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature) test.assert_nested_struct_eq(tensor_specs, (tf.TensorSpec([], tf.int32), ))
def test_with_int_vector(self): type_signature = computation_types.TensorType(tf.int32, [10]) tensor_specs = type_conversions.type_to_tf_tensor_specs(type_signature) test.assert_nested_struct_eq(tensor_specs, tf.TensorSpec([10], tf.int32))
def test_with_int_scalar(self): dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes( tf.int32) test.assert_nested_struct_eq(dtypes, tf.int32) test.assert_nested_struct_eq(shapes, tf.TensorShape([]))
def test_with_int_vector(self): type_signature = computation_types.TensorType(tf.int32, [10]) dtypes, shapes = type_conversions.type_to_tf_dtypes_and_shapes( type_signature) test.assert_nested_struct_eq(dtypes, tf.int32) test.assert_nested_struct_eq(shapes, tf.TensorShape([10]))
def test_with_int_scalar(self): tensor_specs = type_conversions.type_to_tf_tensor_specs(tf.int32) test.assert_nested_struct_eq(tensor_specs, tf.TensorSpec([], tf.int32))
def test_with_int_vector(self): tensor_specs = type_conversions.type_to_tf_tensor_specs( (tf.int32, [10])) test.assert_nested_struct_eq(tensor_specs, tf.TensorSpec([10], tf.int32))
def test_nested_structures_are_same_where_they_are_same(self): test.assert_nested_struct_eq({'a': 10}, {'a': 10})
def test_type_to_tf_dtypes_and_shapes_with_int_vector(self): dtypes, shapes = type_utils.type_to_tf_dtypes_and_shapes( (tf.int32, [10])) test.assert_nested_struct_eq(dtypes, tf.int32) test.assert_nested_struct_eq(shapes, tf.TensorShape([10]))
def test_nested_structures_are_same_where_nesting_differs(self): with self.assertRaises(ValueError): test.assert_nested_struct_eq({'a': 10}, 10)
def test_nested_structures_are_same_where_values_differ(self): with self.assertRaises(ValueError): test.assert_nested_struct_eq({'a': 10}, {'a': False})
def test_nested_structures_are_same_where_values_differ(self): with self.assertRaises(ValueError): # pylint: disable=g-error-prone-assert-raises test.assert_nested_struct_eq({'a': 10}, {'a': False})
def test_with_unnamed_element(self): tensor_specs = type_conversions.type_to_tf_tensor_specs( computation_types.NamedTupleType([tf.int32])) test.assert_nested_struct_eq(tensor_specs, (tf.TensorSpec([], tf.int32), ))