def test_from_container_with_namedtuple_of_odict_recursive(self): x = anonymous_tuple.from_container(collections.namedtuple('_', 'x y')( collections.OrderedDict([('a', 10), ('b', 20)]), collections.OrderedDict([('c', 30), ('d', 40)])), recursive=True) self.assertEqual(str(x), '<x=<a=10,b=20>,y=<c=30,d=40>>')
async def create_value(self, value, type_spec=None): type_spec = computation_types.to_type(type_spec) py_typecheck.check_type(type_spec, computation_types.Type) if isinstance(value, intrinsic_defs.IntrinsicDef): if not type_utils.is_concrete_instance_of(type_spec, value.type_signature): # pytype: disable=attribute-error raise TypeError('Incompatible type {} used with intrinsic {}.'.format( type_spec, value.uri)) # pytype: disable=attribute-error else: return CompositeValue(value, type_spec) elif isinstance(value, pb.Computation): which_computation = value.WhichOneof('computation') if which_computation in ['tensorflow', 'lambda']: return CompositeValue(value, type_spec) elif which_computation == 'intrinsic': intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri) if intr is None: raise ValueError('Encountered an unrecognized intrinsic "{}".'.format( value.intrinsic.uri)) py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef) return await self.create_value(intr, type_spec) else: raise NotImplementedError( 'Unimplemented computation type {}.'.format(which_computation)) elif isinstance(type_spec, computation_types.NamedTupleType): value_tuple = anonymous_tuple.from_container(value) items = await asyncio.gather( *[self.create_value(v, t) for v, t in zip(value_tuple, type_spec)]) type_elemnents_iter = anonymous_tuple.iter_elements(type_spec) return self.create_tuple( anonymous_tuple.AnonymousTuple( (k, i) for (k, _), i in zip(type_elemnents_iter, items))) elif isinstance(type_spec, computation_types.FederatedType): if type_spec.placement == placement_literals.SERVER: if type_spec.all_equal: return CompositeValue( await self._parent_executor.create_value(value, type_spec.member), type_spec) else: raise ValueError('A non-all_equal value on the server is unexpected.') elif type_spec.placement == placement_literals.CLIENTS: if type_spec.all_equal: return CompositeValue( await asyncio.gather(*[ c.create_value(value, type_spec) for c in self._child_executors ]), type_spec) else: py_typecheck.check_type(value, list) cardinalities = await self._get_cardinalities() py_typecheck.check_len(cardinalities, len(self._child_executors)) count = sum(cardinalities) py_typecheck.check_len(value, count) result = [] offset = 0 for c, n in zip(self._child_executors, cardinalities): new_offset = offset + n # The slice opporator is not supported on all the types `value` # supports. # pytype: disable=unsupported-operands result.append(c.create_value(value[offset:new_offset], type_spec)) # pytype: enable=unsupported-operands offset = new_offset return CompositeValue(await asyncio.gather(*result), type_spec) else: raise ValueError('Unexpected placement {}.'.format(type_spec.placement)) else: return CompositeValue( await self._parent_executor.create_value(value, type_spec), type_spec)
def test_from_container_with_namedtuple(self): x = anonymous_tuple.from_container( collections.namedtuple('_', 'x y')(1, 2)) self.assertIsInstance(x, anonymous_tuple.AnonymousTuple) self.assertEqual(str(x), '<x=1,y=2>')
def test_from_container_with_anonymous_tuple(self): x = anonymous_tuple.from_container( anonymous_tuple.AnonymousTuple([('a', 10), ('b', 20)])) self.assertIs(x, x)
def test_from_container_with_dict(self): x = anonymous_tuple.from_container({'z': 10, 'y': 20, 'a': 30}) self.assertIsInstance(x, anonymous_tuple.AnonymousTuple) self.assertEqual(str(x), '<a=30,y=20,z=10>')
def test_from_container_with_ordered_dict(self): x = anonymous_tuple.from_container( collections.OrderedDict([('z', 10), ('y', 20), ('a', 30)])) self.assertIsInstance(x, anonymous_tuple.AnonymousTuple) self.assertEqual(str(x), '<z=10,y=20,a=30>')
def test_from_container_with_int(self): with self.assertRaises(TypeError): anonymous_tuple.from_container(10)
def test_from_container_with_tuple(self): x = anonymous_tuple.from_container(tuple([10, 20])) self.assertIsInstance(x, anonymous_tuple.AnonymousTuple) self.assertEqual(str(x), '<10,20>')
def to_representation_for_type(value, type_spec, callable_handler=None): """Verifies or converts the `value` representation to match `type_spec`. This method first tries to determine whether `value` is a valid representation of TFF type `type_spec`. If so, it is returned unchanged. If not, but if it can be converted into a valid representation, it is converted to such, and the valid representation is returned. If no conversion to a valid representation is possible, TypeError is raised. The accepted forms of `value` for various TFF types are as follows: * For TFF tensor types listed in `tensorflow_utils.TENSOR_REPRESENTATION_TYPES`. * For TFF named tuple types, instances of `anonymous_tuple.AnonymousTuple`. * For TFF sequences, Python lists. * For TFF functional types, Python callables that accept a single argument that is an instance of `ComputedValue` (if the function has a parameter) or `None` (otherwise), and return a `ComputedValue` instance as a result. This function only verifies that `value` is a callable. * For TFF abstract types, there is no valid representation. The reference executor requires all types in an executable computation to be concrete. * For TFF placement types, the valid representations are the placement literals (currently only `tff.SERVER` and `tff.CLIENTS`). * For TFF federated types with `all_equal` set to `True`, the representation is the same as the representation of the member constituent (thus, e.g., a valid representation of `int32@SERVER` is the same as that of `int32`). For those types that have `all_equal_` set to `False`, the representation is a Python list of member constituents. NOTE: This function does not attempt at validating that the sizes of lists that represent federated values match the corresponding placemenets. The cardinality analysis is a separate step, handled by the reference executor at a different point. As long as values can be packed into a Python list, they are accepted as they are. Args: value: The raw representation of a value to compare against `type_spec` and potentially to be converted into a canonical form for the given TFF type. type_spec: The TFF type, an instance of `tff.Type` or something convertible to it that determines what the valid representation should be. callable_handler: The function to invoke to handle TFF functional types. If this is `None`, functional types are not supported. The function must accept `value` and `type_spec` as arguments and return the converted valid representation, just as `to_representation_for_type`. Returns: Either `value` itself, or the `value` converted into a valid representation for `type_spec`. Raises: TypeError: If `value` is not a valid representation for given `type_spec`. NotImplementedError: If verification for `type_spec` is not supported. """ type_spec = computation_types.to_type(type_spec) py_typecheck.check_type(type_spec, computation_types.Type) if callable_handler is not None: py_typecheck.check_callable(callable_handler) # NOTE: We do not simply call `type_utils.infer_type()` on `value`, as the # representations of values in the reference executor are only a subset of # the Python types recognized by that helper function. if isinstance(type_spec, computation_types.TensorType): if tf.executing_eagerly() and isinstance(value, (tf.Tensor, tf.Variable)): value = value.numpy() py_typecheck.check_type(value, tensorflow_utils.TENSOR_REPRESENTATION_TYPES) inferred_type_spec = type_utils.infer_type(value) if not type_utils.is_assignable_from(type_spec, inferred_type_spec): raise TypeError( 'The tensor type {} of the value representation does not match ' 'the type spec {}.'.format(inferred_type_spec, type_spec)) return value elif isinstance(type_spec, computation_types.NamedTupleType): type_spec_elements = anonymous_tuple.to_elements(type_spec) # Special-casing unodered dictionaries to allow their elements to be fed in # the order in which they're defined in the named tuple type. if (isinstance(value, dict) and (set(value.keys()) == set(k for k, _ in type_spec_elements))): value = collections.OrderedDict([ (k, value[k]) for k, _ in type_spec_elements ]) value = anonymous_tuple.from_container(value) value_elements = anonymous_tuple.to_elements(value) if len(value_elements) != len(type_spec_elements): raise TypeError( 'The number of elements {} in the value tuple {} does not match the ' 'number of elements {} in the type spec {}.'.format( len(value_elements), value, len(type_spec_elements), type_spec)) result_elements = [] for index, (type_elem_name, type_elem) in enumerate(type_spec_elements): value_elem_name, value_elem = value_elements[index] if value_elem_name not in [type_elem_name, None]: raise TypeError( 'Found element named `{}` where `{}` was expected at position {} ' 'in the value tuple. Value: {}. Type: {}'.format( value_elem_name, type_elem_name, index, value, type_spec)) converted_value_elem = to_representation_for_type(value_elem, type_elem, callable_handler) result_elements.append((type_elem_name, converted_value_elem)) return anonymous_tuple.AnonymousTuple(result_elements) elif isinstance(type_spec, computation_types.SequenceType): if isinstance(value, tf.data.Dataset): inferred_type_spec = computation_types.SequenceType( computation_types.to_type(tf.data.experimental.get_structure(value))) if not type_utils.is_assignable_from(type_spec, inferred_type_spec): raise TypeError( 'Value of type {!s} not assignable to expected type {!s}'.format( inferred_type_spec, type_spec)) if tf.executing_eagerly(): return [ to_representation_for_type(v, type_spec.element, callable_handler) for v in value ] else: raise ValueError( 'Processing `tf.data.Datasets` outside of eager mode is not ' 'currently supported.') return [ to_representation_for_type(v, type_spec.element, callable_handler) for v in value ] elif isinstance(type_spec, computation_types.FunctionType): if callable_handler is not None: return callable_handler(value, type_spec) else: raise TypeError( 'Values that are callables have been explicitly disallowed ' 'in this context. If you would like to supply here a function ' 'as a parameter, please construct a computation that contains ' 'this call.') elif isinstance(type_spec, computation_types.AbstractType): raise TypeError( 'Abstract types are not supported by the reference executor.') elif isinstance(type_spec, computation_types.PlacementType): py_typecheck.check_type(value, placement_literals.PlacementLiteral) return value elif isinstance(type_spec, computation_types.FederatedType): if type_spec.all_equal: return to_representation_for_type(value, type_spec.member, callable_handler) elif type_spec.placement is not placements.CLIENTS: raise TypeError( 'Unable to determine a valid value representation for a federated ' 'type with non-equal members placed at {}.'.format( type_spec.placement)) elif not isinstance(value, (list, tuple)): raise ValueError('Please pass a list or tuple to any function that' ' expects a federated type placed at {};' ' you passed {}'.format(type_spec.placement, value)) else: return [ to_representation_for_type(v, type_spec.member, callable_handler) for v in value ] else: raise NotImplementedError( 'Unable to determine valid value representation for {} for what ' 'is currently an unsupported TFF type {}.'.format(value, type_spec))
async def create_tuple(self, elements): return ReferenceResolvingExecutorValue( anonymous_tuple.from_container(elements))
async def create_tuple(self, elements): return LambdaExecutorValue(anonymous_tuple.from_container(elements))
def to_representation_for_type(value, type_spec=None, device=None): """Verifies or converts the `value` to an eager objct matching `type_spec`. WARNING: This function is only partially implemented. It does not support data sets at this point. The output of this function is always an eager tensor, eager dataset, a representation of a TensorFlow computtion, or a nested structure of those that matches `type_spec`, and when `device` has been specified, everything is placed on that device on a best-effort basis. TensorFlow computations are represented here as zero- or one-argument Python callables that accept their entire argument bundle as a single Python object. Args: value: The raw representation of a value to compare against `type_spec` and potentially to be converted. type_spec: An instance of `tff.Type`, can be `None` for values that derive from `typed_object.TypedObject`. device: The optional device to place the value on (for tensor-level values). Returns: Either `value` itself, or a modified version of it. Raises: TypeError: If the `value` is not compatible with `type_spec`. """ if device is not None: py_typecheck.check_type(device, six.string_types) with tf.device(device): return to_representation_for_type(value, type_spec=type_spec, device=None) type_spec = computation_types.to_type(type_spec) if isinstance(value, typed_object.TypedObject): if type_spec is not None: if not type_utils.are_equivalent_types(value.type_signature, type_spec): raise TypeError( 'Expected a value of type {}, found {}.'.format( str(type_spec), str(value.type_signature))) else: type_spec = value.type_signature if type_spec is None: raise ValueError( 'Cannot derive an eager representation for a value of an unknown type.' ) if isinstance(value, EagerValue): return value.internal_representation if isinstance(value, executor_value_base.ExecutorValue): raise TypeError( 'Cannot accept a value embedded within a non-eager executor.') if isinstance(value, computation_base.Computation): return to_representation_for_type( computation_impl.ComputationImpl.get_proto(value), type_spec, device) if isinstance(value, pb.Computation): return embed_tensorflow_computation(value, type_spec, device) if isinstance(type_spec, computation_types.TensorType): if not isinstance(value, tf.Tensor): value = tf.constant(value, type_spec.dtype, type_spec.shape) value_type = (computation_types.TensorType(value.dtype.base_dtype, value.shape)) if not type_utils.is_assignable_from(type_spec, value_type): raise TypeError( 'The apparent type {} of a tensor {} does not match the expected ' 'type {}.'.format(str(value_type), str(value), str(type_spec))) return value elif isinstance(type_spec, computation_types.NamedTupleType): type_elem = anonymous_tuple.to_elements(type_spec) value_elem = (anonymous_tuple.to_elements( anonymous_tuple.from_container(value))) result_elem = [] if len(type_elem) != len(value_elem): raise TypeError( 'Expected a {}-element tuple, found {} elements.'.format( str(len(type_elem)), str(len(value_elem)))) for (t_name, el_type), (v_name, el_val) in zip(type_elem, value_elem): if t_name != v_name: raise TypeError( 'Mismatching element names in type vs. value: {} vs. {}.'. format(t_name, v_name)) el_repr = to_representation_for_type(el_val, el_type, device) result_elem.append((t_name, el_repr)) return anonymous_tuple.AnonymousTuple(result_elem) else: raise TypeError('Unexpected type {}.'.format(str(type_spec)))
def serialize_value(value, type_spec=None): """Serializes a value into `executor_pb2.Value`. Args: value: A value to be serialized. type_spec: Optional type spec, a `tff.Type` or something convertible to it. Returns: A tuple `(value_proto, ret_type_spec)` where `value_proto` is an instance of `executor_pb2.Value` with the serialized content of `value`, and the returned `ret_type_spec` is an instance of `tff.Type` that represents the TFF type of the serialized value. Raises: TypeError: If the arguments are of the wrong types. ValueError: If the value is malformed. """ type_spec = computation_types.to_type(type_spec) if isinstance(value, computation_pb2.Computation): type_spec = type_utils.reconcile_value_type_with_type_spec( type_serialization.deserialize_type(value.type), type_spec) return executor_pb2.Value(computation=value), type_spec elif isinstance(value, computation_impl.ComputationImpl): return serialize_value( computation_impl.ComputationImpl.get_proto(value), type_utils.reconcile_value_with_type_spec(value, type_spec)) elif isinstance(type_spec, computation_types.TensorType): return serialize_tensor_value(value, type_spec) elif isinstance(type_spec, computation_types.NamedTupleType): type_elements = anonymous_tuple.to_elements(type_spec) val_elements = anonymous_tuple.to_elements( anonymous_tuple.from_container(value)) tup_elems = [] for (e_name, e_type), (_, e_val) in zip(type_elements, val_elements): e_proto, _ = serialize_value(e_val, e_type) tup_elems.append( executor_pb2.Value.Tuple.Element( name=e_name if e_name else None, value=e_proto)) result_proto = (executor_pb2.Value(tuple=executor_pb2.Value.Tuple( element=tup_elems))) return result_proto, type_spec elif isinstance(type_spec, computation_types.SequenceType): if not isinstance(value, type_conversions.TF_DATASET_REPRESENTATION_TYPES): raise TypeError( 'Cannot serialize Python type {!s} as TFF type {!s}.'.format( py_typecheck.type_string(type(value)), type_spec if type_spec is not None else 'unknown')) value_type = computation_types.SequenceType( computation_types.to_type(value.element_spec)) if not type_analysis.is_assignable_from(type_spec, value_type): raise TypeError( 'Cannot serialize dataset with elements of type {!s} as TFF type {!s}.' .format(value_type, type_spec if type_spec is not None else 'unknown')) return serialize_sequence_value(value), type_spec elif isinstance(type_spec, computation_types.FederatedType): if type_spec.all_equal: value = [value] else: py_typecheck.check_type(value, list) items = [] for v in value: it, it_type = serialize_value(v, type_spec.member) type_analysis.check_assignable_from(type_spec.member, it_type) items.append(it) result_proto = executor_pb2.Value( federated=executor_pb2.Value.Federated( type=type_serialization.serialize_type(type_spec).federated, value=items)) return result_proto, type_spec else: raise ValueError( 'Unable to serialize value with Python type {} and {} TFF type.'. format(str(py_typecheck.type_string(type(value))), str(type_spec) if type_spec is not None else 'unknown'))
def to_representation_for_type(value, tf_function_cache, type_spec=None, device=None): """Verifies or converts the `value` to an eager object matching `type_spec`. WARNING: This function is only partially implemented. It does not support data sets at this point. The output of this function is always an eager tensor, eager dataset, a representation of a TensorFlow computation, or a nested structure of those that matches `type_spec`, and when `device` has been specified, everything is placed on that device on a best-effort basis. TensorFlow computations are represented here as zero- or one-argument Python callables that accept their entire argument bundle as a single Python object. Args: value: The raw representation of a value to compare against `type_spec` and potentially to be converted. tf_function_cache: A cache obeying `dict` semantics that can be used to look up previously embedded TensorFlow functions. type_spec: An instance of `tff.Type`, can be `None` for values that derive from `typed_object.TypedObject`. device: The optional device to place the value on (for tensor-level values). Returns: Either `value` itself, or a modified version of it. Raises: TypeError: If the `value` is not compatible with `type_spec`. """ type_spec = type_utils.reconcile_value_with_type_spec(value, type_spec) if isinstance(value, computation_base.Computation): return to_representation_for_type( computation_impl.ComputationImpl.get_proto(value), tf_function_cache, type_spec, device) elif isinstance(value, pb.Computation): key = (value.SerializeToString(), str(type_spec), device) cached_fn = tf_function_cache.get(key) if cached_fn: return cached_fn embedded_fn = embed_tensorflow_computation(value, type_spec, device) tf_function_cache[key] = embedded_fn return embedded_fn elif isinstance(type_spec, computation_types.NamedTupleType): type_elem = anonymous_tuple.to_elements(type_spec) value_elem = ( anonymous_tuple.to_elements(anonymous_tuple.from_container(value))) result_elem = [] if len(type_elem) != len(value_elem): raise TypeError('Expected a {}-element tuple, found {} elements.'.format( len(type_elem), len(value_elem))) for (t_name, el_type), (v_name, el_val) in zip(type_elem, value_elem): if t_name != v_name: raise TypeError( 'Mismatching element names in type vs. value: {} vs. {}.'.format( t_name, v_name)) el_repr = to_representation_for_type(el_val, tf_function_cache, el_type, device) result_elem.append((t_name, el_repr)) return anonymous_tuple.AnonymousTuple(result_elem) elif device is not None: py_typecheck.check_type(device, str) with tf.device(device): return to_representation_for_type( value, tf_function_cache, type_spec=type_spec, device=None) elif isinstance(value, EagerValue): return value.internal_representation elif isinstance(value, executor_value_base.ExecutorValue): raise TypeError( 'Cannot accept a value embedded within a non-eager executor.') elif isinstance(type_spec, computation_types.TensorType): if not tf.is_tensor(value): value = tf.convert_to_tensor(value, dtype=type_spec.dtype) elif hasattr(value, 'read_value'): # a tf.Variable-like result, get a proper tensor. value = value.read_value() value_type = ( computation_types.TensorType(value.dtype.base_dtype, value.shape)) if not type_utils.is_assignable_from(type_spec, value_type): raise TypeError( 'The apparent type {} of a tensor {} does not match the expected ' 'type {}.'.format(value_type, value, type_spec)) return value elif isinstance(type_spec, computation_types.SequenceType): if isinstance(value, list): value = tensorflow_utils.make_data_set_from_elements( None, value, type_spec.element) py_typecheck.check_type(value, type_conversions.TF_DATASET_REPRESENTATION_TYPES) element_type = computation_types.to_type(value.element_spec) value_type = computation_types.SequenceType(element_type) type_utils.check_assignable_from(type_spec, value_type) return value else: raise TypeError('Unexpected type {}.'.format(type_spec))
def _ensure_anonymous_tuple(obj): return anonymous_tuple.from_container(obj, True)