def test_from_container_with_namedtuple_of_odict_recursive(self):
x = anonymous_tuple.from_container(collections.namedtuple('_', 'x y')(
collections.OrderedDict([('a', 10), ('b', 20)]),
collections.OrderedDict([('c', 30), ('d', 40)])),
recursive=True)
self.assertEqual(str(x), '<x=<a=10,b=20>,y=<c=30,d=40>>')
async def create_value(self, value, type_spec=None):
type_spec = computation_types.to_type(type_spec)
py_typecheck.check_type(type_spec, computation_types.Type)
if isinstance(value, intrinsic_defs.IntrinsicDef):
if not type_utils.is_concrete_instance_of(type_spec,
value.type_signature): # pytype: disable=attribute-error
raise TypeError('Incompatible type {} used with intrinsic {}.'.format(
type_spec, value.uri)) # pytype: disable=attribute-error
else:
return CompositeValue(value, type_spec)
elif isinstance(value, pb.Computation):
which_computation = value.WhichOneof('computation')
if which_computation in ['tensorflow', 'lambda']:
return CompositeValue(value, type_spec)
elif which_computation == 'intrinsic':
intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri)
if intr is None:
raise ValueError('Encountered an unrecognized intrinsic "{}".'.format(
value.intrinsic.uri))
py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef)
return await self.create_value(intr, type_spec)
else:
raise NotImplementedError(
'Unimplemented computation type {}.'.format(which_computation))
elif isinstance(type_spec, computation_types.NamedTupleType):
value_tuple = anonymous_tuple.from_container(value)
items = await asyncio.gather(
*[self.create_value(v, t) for v, t in zip(value_tuple, type_spec)])
type_elemnents_iter = anonymous_tuple.iter_elements(type_spec)
return self.create_tuple(
anonymous_tuple.AnonymousTuple(
(k, i) for (k, _), i in zip(type_elemnents_iter, items)))
elif isinstance(type_spec, computation_types.FederatedType):
if type_spec.placement == placement_literals.SERVER:
if type_spec.all_equal:
return CompositeValue(
await self._parent_executor.create_value(value, type_spec.member),
type_spec)
else:
raise ValueError('A non-all_equal value on the server is unexpected.')
elif type_spec.placement == placement_literals.CLIENTS:
if type_spec.all_equal:
return CompositeValue(
await asyncio.gather(*[
c.create_value(value, type_spec)
for c in self._child_executors
]), type_spec)
else:
py_typecheck.check_type(value, list)
cardinalities = await self._get_cardinalities()
py_typecheck.check_len(cardinalities, len(self._child_executors))
count = sum(cardinalities)
py_typecheck.check_len(value, count)
result = []
offset = 0
for c, n in zip(self._child_executors, cardinalities):
new_offset = offset + n
# The slice opporator is not supported on all the types `value`
# supports.
# pytype: disable=unsupported-operands
result.append(c.create_value(value[offset:new_offset], type_spec))
# pytype: enable=unsupported-operands
offset = new_offset
return CompositeValue(await asyncio.gather(*result), type_spec)
else:
raise ValueError('Unexpected placement {}.'.format(type_spec.placement))
else:
return CompositeValue(
await self._parent_executor.create_value(value, type_spec), type_spec)
def test_from_container_with_namedtuple(self):
x = anonymous_tuple.from_container(
collections.namedtuple('_', 'x y')(1, 2))
self.assertIsInstance(x, anonymous_tuple.AnonymousTuple)
self.assertEqual(str(x), '<x=1,y=2>')
def test_from_container_with_anonymous_tuple(self):
x = anonymous_tuple.from_container(
anonymous_tuple.AnonymousTuple([('a', 10), ('b', 20)]))
self.assertIs(x, x)
def test_from_container_with_dict(self):
x = anonymous_tuple.from_container({'z': 10, 'y': 20, 'a': 30})
self.assertIsInstance(x, anonymous_tuple.AnonymousTuple)
self.assertEqual(str(x), '<a=30,y=20,z=10>')
def test_from_container_with_ordered_dict(self):
x = anonymous_tuple.from_container(
collections.OrderedDict([('z', 10), ('y', 20), ('a', 30)]))
self.assertIsInstance(x, anonymous_tuple.AnonymousTuple)
self.assertEqual(str(x), '<z=10,y=20,a=30>')
def test_from_container_with_int(self):
with self.assertRaises(TypeError):
anonymous_tuple.from_container(10)
def test_from_container_with_tuple(self):
x = anonymous_tuple.from_container(tuple([10, 20]))
self.assertIsInstance(x, anonymous_tuple.AnonymousTuple)
self.assertEqual(str(x), '<10,20>')
def to_representation_for_type(value, type_spec, callable_handler=None):
"""Verifies or converts the `value` representation to match `type_spec`.
This method first tries to determine whether `value` is a valid representation
of TFF type `type_spec`. If so, it is returned unchanged. If not, but if it
can be converted into a valid representation, it is converted to such, and the
valid representation is returned. If no conversion to a valid representation
is possible, TypeError is raised.
The accepted forms of `value` for various TFF types are as follows:
* For TFF tensor types listed in
`tensorflow_utils.TENSOR_REPRESENTATION_TYPES`.
* For TFF named tuple types, instances of `anonymous_tuple.AnonymousTuple`.
* For TFF sequences, Python lists.
* For TFF functional types, Python callables that accept a single argument
that is an instance of `ComputedValue` (if the function has a parameter)
or `None` (otherwise), and return a `ComputedValue` instance as a result.
This function only verifies that `value` is a callable.
* For TFF abstract types, there is no valid representation. The reference
executor requires all types in an executable computation to be concrete.
* For TFF placement types, the valid representations are the placement
literals (currently only `tff.SERVER` and `tff.CLIENTS`).
* For TFF federated types with `all_equal` set to `True`, the representation
is the same as the representation of the member constituent (thus, e.g.,
a valid representation of `int32@SERVER` is the same as that of `int32`).
For those types that have `all_equal_` set to `False`, the representation
is a Python list of member constituents.
NOTE: This function does not attempt at validating that the sizes of lists
that represent federated values match the corresponding placemenets. The
cardinality analysis is a separate step, handled by the reference executor
at a different point. As long as values can be packed into a Python list,
they are accepted as they are.
Args:
value: The raw representation of a value to compare against `type_spec` and
potentially to be converted into a canonical form for the given TFF type.
type_spec: The TFF type, an instance of `tff.Type` or something convertible
to it that determines what the valid representation should be.
callable_handler: The function to invoke to handle TFF functional types. If
this is `None`, functional types are not supported. The function must
accept `value` and `type_spec` as arguments and return the converted valid
representation, just as `to_representation_for_type`.
Returns:
Either `value` itself, or the `value` converted into a valid representation
for `type_spec`.
Raises:
TypeError: If `value` is not a valid representation for given `type_spec`.
NotImplementedError: If verification for `type_spec` is not supported.
"""
type_spec = computation_types.to_type(type_spec)
py_typecheck.check_type(type_spec, computation_types.Type)
if callable_handler is not None:
py_typecheck.check_callable(callable_handler)
# NOTE: We do not simply call `type_utils.infer_type()` on `value`, as the
# representations of values in the reference executor are only a subset of
# the Python types recognized by that helper function.
if isinstance(type_spec, computation_types.TensorType):
if tf.executing_eagerly() and isinstance(value, (tf.Tensor, tf.Variable)):
value = value.numpy()
py_typecheck.check_type(value, tensorflow_utils.TENSOR_REPRESENTATION_TYPES)
inferred_type_spec = type_utils.infer_type(value)
if not type_utils.is_assignable_from(type_spec, inferred_type_spec):
raise TypeError(
'The tensor type {} of the value representation does not match '
'the type spec {}.'.format(inferred_type_spec, type_spec))
return value
elif isinstance(type_spec, computation_types.NamedTupleType):
type_spec_elements = anonymous_tuple.to_elements(type_spec)
# Special-casing unodered dictionaries to allow their elements to be fed in
# the order in which they're defined in the named tuple type.
if (isinstance(value, dict) and
(set(value.keys()) == set(k for k, _ in type_spec_elements))):
value = collections.OrderedDict([
(k, value[k]) for k, _ in type_spec_elements
])
value = anonymous_tuple.from_container(value)
value_elements = anonymous_tuple.to_elements(value)
if len(value_elements) != len(type_spec_elements):
raise TypeError(
'The number of elements {} in the value tuple {} does not match the '
'number of elements {} in the type spec {}.'.format(
len(value_elements), value, len(type_spec_elements), type_spec))
result_elements = []
for index, (type_elem_name, type_elem) in enumerate(type_spec_elements):
value_elem_name, value_elem = value_elements[index]
if value_elem_name not in [type_elem_name, None]:
raise TypeError(
'Found element named `{}` where `{}` was expected at position {} '
'in the value tuple. Value: {}. Type: {}'.format(
value_elem_name, type_elem_name, index, value, type_spec))
converted_value_elem = to_representation_for_type(value_elem, type_elem,
callable_handler)
result_elements.append((type_elem_name, converted_value_elem))
return anonymous_tuple.AnonymousTuple(result_elements)
elif isinstance(type_spec, computation_types.SequenceType):
if isinstance(value, tf.data.Dataset):
inferred_type_spec = computation_types.SequenceType(
computation_types.to_type(tf.data.experimental.get_structure(value)))
if not type_utils.is_assignable_from(type_spec, inferred_type_spec):
raise TypeError(
'Value of type {!s} not assignable to expected type {!s}'.format(
inferred_type_spec, type_spec))
if tf.executing_eagerly():
return [
to_representation_for_type(v, type_spec.element, callable_handler)
for v in value
]
else:
raise ValueError(
'Processing `tf.data.Datasets` outside of eager mode is not '
'currently supported.')
return [
to_representation_for_type(v, type_spec.element, callable_handler)
for v in value
]
elif isinstance(type_spec, computation_types.FunctionType):
if callable_handler is not None:
return callable_handler(value, type_spec)
else:
raise TypeError(
'Values that are callables have been explicitly disallowed '
'in this context. If you would like to supply here a function '
'as a parameter, please construct a computation that contains '
'this call.')
elif isinstance(type_spec, computation_types.AbstractType):
raise TypeError(
'Abstract types are not supported by the reference executor.')
elif isinstance(type_spec, computation_types.PlacementType):
py_typecheck.check_type(value, placement_literals.PlacementLiteral)
return value
elif isinstance(type_spec, computation_types.FederatedType):
if type_spec.all_equal:
return to_representation_for_type(value, type_spec.member,
callable_handler)
elif type_spec.placement is not placements.CLIENTS:
raise TypeError(
'Unable to determine a valid value representation for a federated '
'type with non-equal members placed at {}.'.format(
type_spec.placement))
elif not isinstance(value, (list, tuple)):
raise ValueError('Please pass a list or tuple to any function that'
' expects a federated type placed at {};'
' you passed {}'.format(type_spec.placement, value))
else:
return [
to_representation_for_type(v, type_spec.member, callable_handler)
for v in value
]
else:
raise NotImplementedError(
'Unable to determine valid value representation for {} for what '
'is currently an unsupported TFF type {}.'.format(value, type_spec))
async def create_tuple(self, elements):
return ReferenceResolvingExecutorValue(
anonymous_tuple.from_container(elements))
async def create_tuple(self, elements):
return LambdaExecutorValue(anonymous_tuple.from_container(elements))
def to_representation_for_type(value, type_spec=None, device=None):
"""Verifies or converts the `value` to an eager objct matching `type_spec`.
WARNING: This function is only partially implemented. It does not support
data sets at this point.
The output of this function is always an eager tensor, eager dataset, a
representation of a TensorFlow computtion, or a nested structure of those
that matches `type_spec`, and when `device` has been specified, everything
is placed on that device on a best-effort basis.
TensorFlow computations are represented here as zero- or one-argument Python
callables that accept their entire argument bundle as a single Python object.
Args:
value: The raw representation of a value to compare against `type_spec` and
potentially to be converted.
type_spec: An instance of `tff.Type`, can be `None` for values that derive
from `typed_object.TypedObject`.
device: The optional device to place the value on (for tensor-level values).
Returns:
Either `value` itself, or a modified version of it.
Raises:
TypeError: If the `value` is not compatible with `type_spec`.
"""
if device is not None:
py_typecheck.check_type(device, six.string_types)
with tf.device(device):
return to_representation_for_type(value,
type_spec=type_spec,
device=None)
type_spec = computation_types.to_type(type_spec)
if isinstance(value, typed_object.TypedObject):
if type_spec is not None:
if not type_utils.are_equivalent_types(value.type_signature,
type_spec):
raise TypeError(
'Expected a value of type {}, found {}.'.format(
str(type_spec), str(value.type_signature)))
else:
type_spec = value.type_signature
if type_spec is None:
raise ValueError(
'Cannot derive an eager representation for a value of an unknown type.'
)
if isinstance(value, EagerValue):
return value.internal_representation
if isinstance(value, executor_value_base.ExecutorValue):
raise TypeError(
'Cannot accept a value embedded within a non-eager executor.')
if isinstance(value, computation_base.Computation):
return to_representation_for_type(
computation_impl.ComputationImpl.get_proto(value), type_spec,
device)
if isinstance(value, pb.Computation):
return embed_tensorflow_computation(value, type_spec, device)
if isinstance(type_spec, computation_types.TensorType):
if not isinstance(value, tf.Tensor):
value = tf.constant(value, type_spec.dtype, type_spec.shape)
value_type = (computation_types.TensorType(value.dtype.base_dtype,
value.shape))
if not type_utils.is_assignable_from(type_spec, value_type):
raise TypeError(
'The apparent type {} of a tensor {} does not match the expected '
'type {}.'.format(str(value_type), str(value), str(type_spec)))
return value
elif isinstance(type_spec, computation_types.NamedTupleType):
type_elem = anonymous_tuple.to_elements(type_spec)
value_elem = (anonymous_tuple.to_elements(
anonymous_tuple.from_container(value)))
result_elem = []
if len(type_elem) != len(value_elem):
raise TypeError(
'Expected a {}-element tuple, found {} elements.'.format(
str(len(type_elem)), str(len(value_elem))))
for (t_name, el_type), (v_name, el_val) in zip(type_elem, value_elem):
if t_name != v_name:
raise TypeError(
'Mismatching element names in type vs. value: {} vs. {}.'.
format(t_name, v_name))
el_repr = to_representation_for_type(el_val, el_type, device)
result_elem.append((t_name, el_repr))
return anonymous_tuple.AnonymousTuple(result_elem)
else:
raise TypeError('Unexpected type {}.'.format(str(type_spec)))
def serialize_value(value, type_spec=None):
"""Serializes a value into `executor_pb2.Value`.
Args:
value: A value to be serialized.
type_spec: Optional type spec, a `tff.Type` or something convertible to it.
Returns:
A tuple `(value_proto, ret_type_spec)` where `value_proto` is an instance
of `executor_pb2.Value` with the serialized content of `value`, and the
returned `ret_type_spec` is an instance of `tff.Type` that represents the
TFF type of the serialized value.
Raises:
TypeError: If the arguments are of the wrong types.
ValueError: If the value is malformed.
"""
type_spec = computation_types.to_type(type_spec)
if isinstance(value, computation_pb2.Computation):
type_spec = type_utils.reconcile_value_type_with_type_spec(
type_serialization.deserialize_type(value.type), type_spec)
return executor_pb2.Value(computation=value), type_spec
elif isinstance(value, computation_impl.ComputationImpl):
return serialize_value(
computation_impl.ComputationImpl.get_proto(value),
type_utils.reconcile_value_with_type_spec(value, type_spec))
elif isinstance(type_spec, computation_types.TensorType):
return serialize_tensor_value(value, type_spec)
elif isinstance(type_spec, computation_types.NamedTupleType):
type_elements = anonymous_tuple.to_elements(type_spec)
val_elements = anonymous_tuple.to_elements(
anonymous_tuple.from_container(value))
tup_elems = []
for (e_name, e_type), (_, e_val) in zip(type_elements, val_elements):
e_proto, _ = serialize_value(e_val, e_type)
tup_elems.append(
executor_pb2.Value.Tuple.Element(
name=e_name if e_name else None, value=e_proto))
result_proto = (executor_pb2.Value(tuple=executor_pb2.Value.Tuple(
element=tup_elems)))
return result_proto, type_spec
elif isinstance(type_spec, computation_types.SequenceType):
if not isinstance(value,
type_conversions.TF_DATASET_REPRESENTATION_TYPES):
raise TypeError(
'Cannot serialize Python type {!s} as TFF type {!s}.'.format(
py_typecheck.type_string(type(value)),
type_spec if type_spec is not None else 'unknown'))
value_type = computation_types.SequenceType(
computation_types.to_type(value.element_spec))
if not type_analysis.is_assignable_from(type_spec, value_type):
raise TypeError(
'Cannot serialize dataset with elements of type {!s} as TFF type {!s}.'
.format(value_type,
type_spec if type_spec is not None else 'unknown'))
return serialize_sequence_value(value), type_spec
elif isinstance(type_spec, computation_types.FederatedType):
if type_spec.all_equal:
value = [value]
else:
py_typecheck.check_type(value, list)
items = []
for v in value:
it, it_type = serialize_value(v, type_spec.member)
type_analysis.check_assignable_from(type_spec.member, it_type)
items.append(it)
result_proto = executor_pb2.Value(
federated=executor_pb2.Value.Federated(
type=type_serialization.serialize_type(type_spec).federated,
value=items))
return result_proto, type_spec
else:
raise ValueError(
'Unable to serialize value with Python type {} and {} TFF type.'.
format(str(py_typecheck.type_string(type(value))),
str(type_spec) if type_spec is not None else 'unknown'))
def to_representation_for_type(value,
tf_function_cache,
type_spec=None,
device=None):
"""Verifies or converts the `value` to an eager object matching `type_spec`.
WARNING: This function is only partially implemented. It does not support
data sets at this point.
The output of this function is always an eager tensor, eager dataset, a
representation of a TensorFlow computation, or a nested structure of those
that matches `type_spec`, and when `device` has been specified, everything
is placed on that device on a best-effort basis.
TensorFlow computations are represented here as zero- or one-argument Python
callables that accept their entire argument bundle as a single Python object.
Args:
value: The raw representation of a value to compare against `type_spec` and
potentially to be converted.
tf_function_cache: A cache obeying `dict` semantics that can be used to look
up previously embedded TensorFlow functions.
type_spec: An instance of `tff.Type`, can be `None` for values that derive
from `typed_object.TypedObject`.
device: The optional device to place the value on (for tensor-level values).
Returns:
Either `value` itself, or a modified version of it.
Raises:
TypeError: If the `value` is not compatible with `type_spec`.
"""
type_spec = type_utils.reconcile_value_with_type_spec(value, type_spec)
if isinstance(value, computation_base.Computation):
return to_representation_for_type(
computation_impl.ComputationImpl.get_proto(value), tf_function_cache,
type_spec, device)
elif isinstance(value, pb.Computation):
key = (value.SerializeToString(), str(type_spec), device)
cached_fn = tf_function_cache.get(key)
if cached_fn:
return cached_fn
embedded_fn = embed_tensorflow_computation(value, type_spec, device)
tf_function_cache[key] = embedded_fn
return embedded_fn
elif isinstance(type_spec, computation_types.NamedTupleType):
type_elem = anonymous_tuple.to_elements(type_spec)
value_elem = (
anonymous_tuple.to_elements(anonymous_tuple.from_container(value)))
result_elem = []
if len(type_elem) != len(value_elem):
raise TypeError('Expected a {}-element tuple, found {} elements.'.format(
len(type_elem), len(value_elem)))
for (t_name, el_type), (v_name, el_val) in zip(type_elem, value_elem):
if t_name != v_name:
raise TypeError(
'Mismatching element names in type vs. value: {} vs. {}.'.format(
t_name, v_name))
el_repr = to_representation_for_type(el_val, tf_function_cache, el_type,
device)
result_elem.append((t_name, el_repr))
return anonymous_tuple.AnonymousTuple(result_elem)
elif device is not None:
py_typecheck.check_type(device, str)
with tf.device(device):
return to_representation_for_type(
value, tf_function_cache, type_spec=type_spec, device=None)
elif isinstance(value, EagerValue):
return value.internal_representation
elif isinstance(value, executor_value_base.ExecutorValue):
raise TypeError(
'Cannot accept a value embedded within a non-eager executor.')
elif isinstance(type_spec, computation_types.TensorType):
if not tf.is_tensor(value):
value = tf.convert_to_tensor(value, dtype=type_spec.dtype)
elif hasattr(value, 'read_value'):
# a tf.Variable-like result, get a proper tensor.
value = value.read_value()
value_type = (
computation_types.TensorType(value.dtype.base_dtype, value.shape))
if not type_utils.is_assignable_from(type_spec, value_type):
raise TypeError(
'The apparent type {} of a tensor {} does not match the expected '
'type {}.'.format(value_type, value, type_spec))
return value
elif isinstance(type_spec, computation_types.SequenceType):
if isinstance(value, list):
value = tensorflow_utils.make_data_set_from_elements(
None, value, type_spec.element)
py_typecheck.check_type(value,
type_conversions.TF_DATASET_REPRESENTATION_TYPES)
element_type = computation_types.to_type(value.element_spec)
value_type = computation_types.SequenceType(element_type)
type_utils.check_assignable_from(type_spec, value_type)
return value
else:
raise TypeError('Unexpected type {}.'.format(type_spec))
def _ensure_anonymous_tuple(obj): return anonymous_tuple.from_container(obj, True)
