def test_make_data_set_from_elements_with_wrong_elements(self):
with self.assertRaises(TypeError):
graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [{
'a': 1
}, {
'a': 2
}], tf.int32)
def test_make_data_set_from_elements_with_just_one_batch(self):
graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [np.array([1])],
computation_types.TensorType(tf.int32, tf.TensorShape([None])))
graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [{
'x': np.array([1])
}],
[('x',
computation_types.TensorType(tf.int32, tf.TensorShape([None])))])
def test_make_data_set_from_elements_with_empty_list(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [], tf.float32)
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(
tf.compat.v1.Session().run(ds.reduce(1.0, lambda x, y: x + y)),
1.0)
def test_make_data_set_from_elements_with_empty_list_definite_tuple(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [], [
computation_types.TensorType(tf.float32, [None, 10]),
computation_types.TensorType(tf.float32, [None, 5])
])
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(tf.compat.v1.data.get_output_shapes(ds),
([0, 10], [0, 5]))
def test_make_data_set_from_elements_with_list_of_lists(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [
[[1], [2]],
[[3], [4]],
], [[tf.int32], [tf.int32]])
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(
tf.compat.v1.Session().run(
ds.reduce(0, lambda x, y: x + tf.reduce_sum(y))), 10)
def test_make_data_set_from_elements_with_odd_all_batches(self):
graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [
np.array([1, 2]),
np.array([3]),
np.array([4, 5, 6]),
np.array([7, 8])
], computation_types.TensorType(tf.int32, tf.TensorShape([None])))
graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [{
'x': np.array([1, 2])
}, {
'x': np.array([3])
}, {
'x': np.array([4, 5, 6])
}, {
'x': np.array([7, 8])
}],
[('x',
computation_types.TensorType(tf.int32, tf.TensorShape([None])))])
def test_make_data_set_from_elements_with_empty_list_definite_tensor(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [],
computation_types.TensorType(tf.float32, [None, 10]))
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(
tf.compat.v1.data.get_output_shapes(ds).as_list(),
tf.TensorShape([0, 10]).as_list())
self.assertEqual(
tf.compat.v1.Session().run(ds.reduce(1.0, lambda x, y: x + y)),
1.0)
def test_make_data_set_from_elements_with_list_of_dicts(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [{
'a': 1,
'b': 2,
}, {
'a': 3,
'b': 4,
}], [('a', tf.int32), ('b', tf.int32)])
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(
tf.compat.v1.Session().run(
ds.reduce(0, lambda x, y: x + y['a'] + y['b'])), 10)
def test_make_data_set_from_elements_with_list_of_anonymous_tuples(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [
anonymous_tuple.AnonymousTuple([
('a', 1),
('b', 2),
]),
anonymous_tuple.AnonymousTuple([
('a', 3),
('b', 4),
]),
], [('a', tf.int32), ('b', tf.int32)])
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(
tf.compat.v1.Session().run(
ds.reduce(0, lambda x, y: x + y['a'] + y['b'])), 10)
def test_make_data_set_from_elements_with_list_of_ordered_dicts(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [
collections.OrderedDict([
('a', 1),
('b', 2),
]),
collections.OrderedDict([
('a', 3),
('b', 4),
]),
], [('a', tf.int32), ('b', tf.int32)])
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
self.assertEqual(
tf.compat.v1.Session().run(
ds.reduce(0, lambda x, y: x + y['a'] + y['b'])), 10)
def test_make_data_set_from_elements_with_list_of_dicts_with_lists(self):
ds = graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), [{
'a': [1],
'b': [2],
}, {
'a': [3],
'b': [4],
}], [('a', [tf.int32]), ('b', [tf.int32])])
self.assertIsInstance(ds, graph_utils.DATASET_REPRESENTATION_TYPES)
def reduce_fn(x, y):
return x + tf.reduce_sum(y['a']) + tf.reduce_sum(y['b'])
self.assertEqual(tf.compat.v1.Session().run(ds.reduce(0, reduce_fn)),
10)
def test_make_data_set_from_elements_in_eager_context(self):
ds = graph_utils.make_data_set_from_elements(None, [10, 20], tf.int32)
self.assertCountEqual([x.numpy() for x in iter(ds)], [10, 20])
def _create_dataset_from_elements():
return graph_utils.make_data_set_from_elements(
tf.compat.v1.get_default_graph(), elements, element_type)
def to_representation_for_type(value, type_spec=None, device=None):
"""Verifies or converts the `value` to an eager objct matching `type_spec`.
WARNING: This function is only partially implemented. It does not support
data sets at this point.
The output of this function is always an eager tensor, eager dataset, a
representation of a TensorFlow computtion, or a nested structure of those
that matches `type_spec`, and when `device` has been specified, everything
is placed on that device on a best-effort basis.
TensorFlow computations are represented here as zero- or one-argument Python
callables that accept their entire argument bundle as a single Python object.
Args:
value: The raw representation of a value to compare against `type_spec` and
potentially to be converted.
type_spec: An instance of `tff.Type`, can be `None` for values that derive
from `typed_object.TypedObject`.
device: The optional device to place the value on (for tensor-level values).
Returns:
Either `value` itself, or a modified version of it.
Raises:
TypeError: If the `value` is not compatible with `type_spec`.
"""
if device is not None:
py_typecheck.check_type(device, str)
with tf.device(device):
return to_representation_for_type(value,
type_spec=type_spec,
device=None)
type_spec = type_utils.reconcile_value_with_type_spec(value, type_spec)
if isinstance(value, EagerValue):
return value.internal_representation
if isinstance(value, executor_value_base.ExecutorValue):
raise TypeError(
'Cannot accept a value embedded within a non-eager executor.')
if isinstance(value, computation_base.Computation):
return to_representation_for_type(
computation_impl.ComputationImpl.get_proto(value), type_spec,
device)
if isinstance(value, pb.Computation):
return embed_tensorflow_computation(value, type_spec, device)
if isinstance(type_spec, computation_types.TensorType):
if not isinstance(value, tf.Tensor):
if isinstance(value, np.ndarray):
value = tf.constant(value, dtype=type_spec.dtype)
else:
value = tf.constant(value,
dtype=type_spec.dtype,
shape=type_spec.shape)
value_type = (computation_types.TensorType(value.dtype.base_dtype,
value.shape))
if not type_utils.is_assignable_from(type_spec, value_type):
raise TypeError(
'The apparent type {} of a tensor {} does not match the expected '
'type {}.'.format(str(value_type), str(value), str(type_spec)))
return value
elif isinstance(type_spec, computation_types.NamedTupleType):
type_elem = anonymous_tuple.to_elements(type_spec)
value_elem = (anonymous_tuple.to_elements(
anonymous_tuple.from_container(value)))
result_elem = []
if len(type_elem) != len(value_elem):
raise TypeError(
'Expected a {}-element tuple, found {} elements.'.format(
str(len(type_elem)), str(len(value_elem))))
for (t_name, el_type), (v_name, el_val) in zip(type_elem, value_elem):
if t_name != v_name:
raise TypeError(
'Mismatching element names in type vs. value: {} vs. {}.'.
format(t_name, v_name))
el_repr = to_representation_for_type(el_val, el_type, device)
result_elem.append((t_name, el_repr))
return anonymous_tuple.AnonymousTuple(result_elem)
elif isinstance(type_spec, computation_types.SequenceType):
if isinstance(value, list):
value = graph_utils.make_data_set_from_elements(
None, value, type_spec.element)
py_typecheck.check_type(value,
(tf.data.Dataset, tf.compat.v1.data.Dataset,
tf.compat.v2.data.Dataset))
element_type = type_utils.tf_dtypes_and_shapes_to_type(
tf.compat.v1.data.get_output_types(value),
tf.compat.v1.data.get_output_shapes(value))
value_type = computation_types.SequenceType(element_type)
type_utils.check_assignable_from(type_spec, value_type)
return value
else:
raise TypeError('Unexpected type {}.'.format(str(type_spec)))
