def test_make_data_set_from_elements_with_wrong_elements(self): with self.assertRaises(TypeError): graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [{ 'a': 1 }, { 'a': 2 }], tf.int32)
def test_make_data_set_from_elements_with_just_one_batch(self): graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [np.array([1])], computation_types.TensorType(tf.int32, tf.TensorShape([None]))) graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [{ 'x': np.array([1]) }], [('x', computation_types.TensorType(tf.int32, tf.TensorShape([None])))])
def test_make_data_set_from_elements_with_empty_list(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [], tf.float32) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual( tf.compat.v1.Session().run(ds.reduce(1.0, lambda x, y: x + y)), 1.0)
def test_make_data_set_from_elements_with_empty_list_definite_tuple(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [], [ computation_types.TensorType(tf.float32, [None, 10]), computation_types.TensorType(tf.float32, [None, 5]) ]) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual(tf.compat.v1.data.get_output_shapes(ds), ([0, 10], [0, 5]))
def test_make_data_set_from_elements_with_list_of_lists(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [ [[1], [2]], [[3], [4]], ], [[tf.int32], [tf.int32]]) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual( tf.compat.v1.Session().run( ds.reduce(0, lambda x, y: x + tf.reduce_sum(y))), 10)
def test_make_data_set_from_elements_with_odd_all_batches(self): graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [ np.array([1, 2]), np.array([3]), np.array([4, 5, 6]), np.array([7, 8]) ], computation_types.TensorType(tf.int32, tf.TensorShape([None]))) graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [{ 'x': np.array([1, 2]) }, { 'x': np.array([3]) }, { 'x': np.array([4, 5, 6]) }, { 'x': np.array([7, 8]) }], [('x', computation_types.TensorType(tf.int32, tf.TensorShape([None])))])
def test_make_data_set_from_elements_with_empty_list_definite_tensor(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [], computation_types.TensorType(tf.float32, [None, 10])) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual( tf.compat.v1.data.get_output_shapes(ds).as_list(), tf.TensorShape([0, 10]).as_list()) self.assertEqual( tf.compat.v1.Session().run(ds.reduce(1.0, lambda x, y: x + y)), 1.0)
def test_make_data_set_from_elements_with_list_of_dicts(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [{ 'a': 1, 'b': 2, }, { 'a': 3, 'b': 4, }], [('a', tf.int32), ('b', tf.int32)]) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual( tf.compat.v1.Session().run( ds.reduce(0, lambda x, y: x + y['a'] + y['b'])), 10)
def test_make_data_set_from_elements_with_list_of_anonymous_tuples(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [ anonymous_tuple.AnonymousTuple([ ('a', 1), ('b', 2), ]), anonymous_tuple.AnonymousTuple([ ('a', 3), ('b', 4), ]), ], [('a', tf.int32), ('b', tf.int32)]) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual( tf.compat.v1.Session().run( ds.reduce(0, lambda x, y: x + y['a'] + y['b'])), 10)
def test_make_data_set_from_elements_with_list_of_ordered_dicts(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [ collections.OrderedDict([ ('a', 1), ('b', 2), ]), collections.OrderedDict([ ('a', 3), ('b', 4), ]), ], [('a', tf.int32), ('b', tf.int32)]) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) self.assertEqual( tf.compat.v1.Session().run( ds.reduce(0, lambda x, y: x + y['a'] + y['b'])), 10)
def test_make_data_set_from_elements_with_list_of_dicts_with_lists(self): ds = graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), [{ 'a': [1], 'b': [2], }, { 'a': [3], 'b': [4], }], [('a', [tf.int32]), ('b', [tf.int32])]) self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES) def reduce_fn(x, y): return x + tf.reduce_sum(y['a']) + tf.reduce_sum(y['b']) self.assertEqual(tf.compat.v1.Session().run(ds.reduce(0, reduce_fn)), 10)
def test_make_data_set_from_elements_in_eager_context(self): ds = graph_utils.make_data_set_from_elements(None, [10, 20], tf.int32) self.assertCountEqual([x.numpy() for x in iter(ds)], [10, 20])
def _create_dataset_from_elements(): return graph_utils.make_data_set_from_elements( tf.compat.v1.get_default_graph(), elements, element_type)
def to_representation_for_type(value, type_spec=None, device=None): """Verifies or converts the `value` to an eager objct matching `type_spec`. WARNING: This function is only partially implemented. It does not support data sets at this point. The output of this function is always an eager tensor, eager dataset, a representation of a TensorFlow computtion, or a nested structure of those that matches `type_spec`, and when `device` has been specified, everything is placed on that device on a best-effort basis. TensorFlow computations are represented here as zero- or one-argument Python callables that accept their entire argument bundle as a single Python object. Args: value: The raw representation of a value to compare against `type_spec` and potentially to be converted. type_spec: An instance of `tff.Type`, can be `None` for values that derive from `typed_object.TypedObject`. device: The optional device to place the value on (for tensor-level values). Returns: Either `value` itself, or a modified version of it. Raises: TypeError: If the `value` is not compatible with `type_spec`. """ if device is not None: py_typecheck.check_type(device, str) with tf.device(device): return to_representation_for_type(value, type_spec=type_spec, device=None) type_spec = type_utils.reconcile_value_with_type_spec(value, type_spec) if isinstance(value, EagerValue): return value.internal_representation if isinstance(value, executor_value_base.ExecutorValue): raise TypeError( 'Cannot accept a value embedded within a non-eager executor.') if isinstance(value, computation_base.Computation): return to_representation_for_type( computation_impl.ComputationImpl.get_proto(value), type_spec, device) if isinstance(value, pb.Computation): return embed_tensorflow_computation(value, type_spec, device) if isinstance(type_spec, computation_types.TensorType): if not isinstance(value, tf.Tensor): if isinstance(value, np.ndarray): value = tf.constant(value, dtype=type_spec.dtype) else: value = tf.constant(value, dtype=type_spec.dtype, shape=type_spec.shape) value_type = (computation_types.TensorType(value.dtype.base_dtype, value.shape)) if not type_utils.is_assignable_from(type_spec, value_type): raise TypeError( 'The apparent type {} of a tensor {} does not match the expected ' 'type {}.'.format(str(value_type), str(value), str(type_spec))) return value elif isinstance(type_spec, computation_types.NamedTupleType): type_elem = anonymous_tuple.to_elements(type_spec) value_elem = (anonymous_tuple.to_elements( anonymous_tuple.from_container(value))) result_elem = [] if len(type_elem) != len(value_elem): raise TypeError( 'Expected a {}-element tuple, found {} elements.'.format( str(len(type_elem)), str(len(value_elem)))) for (t_name, el_type), (v_name, el_val) in zip(type_elem, value_elem): if t_name != v_name: raise TypeError( 'Mismatching element names in type vs. value: {} vs. {}.'. format(t_name, v_name)) el_repr = to_representation_for_type(el_val, el_type, device) result_elem.append((t_name, el_repr)) return anonymous_tuple.AnonymousTuple(result_elem) elif isinstance(type_spec, computation_types.SequenceType): if isinstance(value, list): value = graph_utils.make_data_set_from_elements( None, value, type_spec.element) py_typecheck.check_type(value, (tf.data.Dataset, tf.compat.v1.data.Dataset, tf.compat.v2.data.Dataset)) element_type = type_utils.tf_dtypes_and_shapes_to_type( tf.compat.v1.data.get_output_types(value), tf.compat.v1.data.get_output_shapes(value)) value_type = computation_types.SequenceType(element_type) type_utils.check_assignable_from(type_spec, value_type) return value else: raise TypeError('Unexpected type {}.'.format(str(type_spec)))