def test_is_concrete_instance_of_raises_with_conflicting_names(self): t1 = computation_types.NamedTupleType( [computation_types.AbstractType('T1')] * 2) t2 = computation_types.NamedTupleType([('a', tf.int32), ('b', tf.int32)]) with self.assertRaises(TypeError): type_utils.is_concrete_instance_of(t2, t1)
def __init__(self, uri, type_spec): """Creates an intrinsic. Args: uri: The URI of the intrinsic. type_spec: Either the types.Type that represents the type of this intrinsic, or something convertible to it by types.to_type(). Raises: TypeError: if the arguments are of the wrong types. """ py_typecheck.check_type(uri, six.string_types) if type_spec is None: raise TypeError( 'Intrinsic {} cannot be created without a TFF type.'.format( uri)) type_spec = computation_types.to_type(type_spec) intrinsic_def = intrinsic_defs.uri_to_intrinsic_def(uri) if intrinsic_def: typecheck = type_utils.is_concrete_instance_of( type_spec, intrinsic_def.type_signature) if not typecheck: raise TypeError( 'Tried to construct an Intrinsic with bad type ' 'signature; Intrinsic {} expects type signature {}, ' 'and you tried to construct one of type {}.'.format( uri, intrinsic_def.type_signature, type_spec)) super(Intrinsic, self).__init__(type_spec) self._uri = uri
def test_is_concrete_instance_of_succeeds_function_different_parameter_and_return_types( self): t1 = computation_types.FunctionType( computation_types.AbstractType('T'), computation_types.AbstractType('U')) t2 = computation_types.FunctionType(tf.int32, tf.float32) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_succeeds_under_tuple(self): t1 = computation_types.NamedTupleType( [computation_types.AbstractType('T1')] * 2) t2 = computation_types.NamedTupleType([ computation_types.TensorType(tf.int32), computation_types.TensorType(tf.int32) ]) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_fails_under_tuple_conflicting_concrete_types( self): t1 = computation_types.NamedTupleType( [computation_types.AbstractType('T1')] * 2) t2 = computation_types.NamedTupleType([ computation_types.TensorType(tf.int32), computation_types.TensorType(tf.float32) ]) self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_abstract_federated_types_succeeds(self): t1 = computation_types.FederatedType( [computation_types.AbstractType('T1')] * 2, placements.CLIENTS, all_equal=True) t2 = computation_types.FederatedType([tf.int32] * 2, placements.CLIENTS, all_equal=True) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_abstract_can_be_concretized_abstract_fails_on_different_federated_all_equal_bits( self): t1 = computation_types.FederatedType( [computation_types.AbstractType('T1')] * 2, placements.CLIENTS, all_equal=True) t2 = computation_types.FederatedType([tf.int32] * 2, placements.SERVER, all_equal=True) self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
async def create_value(self, value, type_spec=None): type_spec = computation_types.to_type(type_spec) py_typecheck.check_type(type_spec, computation_types.Type) if isinstance(value, intrinsic_defs.IntrinsicDef): if not type_utils.is_concrete_instance_of(type_spec, value.type_signature): # pytype: disable=attribute-error raise TypeError('Incompatible type {} used with intrinsic {}.'.format( type_spec, value.uri)) # pytype: disable=attribute-error else: return CompositeValue(value, type_spec) elif isinstance(value, pb.Computation): which_computation = value.WhichOneof('computation') if which_computation in ['tensorflow', 'lambda']: return CompositeValue(value, type_spec) elif which_computation == 'intrinsic': intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri) if intr is None: raise ValueError('Encountered an unrecognized intrinsic "{}".'.format( value.intrinsic.uri)) py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef) return await self.create_value(intr, type_spec) else: raise NotImplementedError( 'Unimplemented computation type {}.'.format(which_computation)) elif isinstance(type_spec, computation_types.NamedTupleType): value_tuple = anonymous_tuple.from_container(value) items = await asyncio.gather( *[self.create_value(v, t) for v, t in zip(value_tuple, type_spec)]) type_elemnents_iter = anonymous_tuple.iter_elements(type_spec) return self.create_tuple( anonymous_tuple.AnonymousTuple( (k, i) for (k, _), i in zip(type_elemnents_iter, items))) elif isinstance(type_spec, computation_types.FederatedType): if type_spec.placement == placement_literals.SERVER: if not type_spec.all_equal: raise ValueError( 'Expected an all equal value at the `SERVER` placement, ' 'found {}.'.format(type_spec)) results = await self._parent_executor.create_value( value, type_spec.member) return CompositeValue(results, type_spec) elif type_spec.placement == placement_literals.CLIENTS: if type_spec.all_equal: results = await asyncio.gather(*[ c.create_value(value, type_spec) for c in self._child_executors ]) return CompositeValue(results, type_spec) else: py_typecheck.check_type(value, list) cardinalities = await self._get_cardinalities() total_clients = sum(cardinalities) py_typecheck.check_len(value, total_clients) results = [] offset = 0 for child, num_clients in zip(self._child_executors, cardinalities): new_offset = offset + num_clients result = child.create_value(value[offset:new_offset], type_spec) results.append(result) offset = new_offset return CompositeValue(await asyncio.gather(*results), type_spec) else: raise ValueError('Unexpected placement {}.'.format(type_spec.placement)) else: return CompositeValue( await self._parent_executor.create_value(value, type_spec), type_spec)
def test_is_concrete_instance_of_fails_conflicting_binding_in_parameter_and_result( self): t1 = computation_types.FunctionType( *[computation_types.AbstractType('T')] * 2) t2 = computation_types.FunctionType(tf.int32, tf.float32) self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_succeeds_single_function_type(self): t1 = computation_types.FunctionType( *[computation_types.AbstractType('T')] * 2) t2 = computation_types.FunctionType(tf.int32, tf.int32) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_raises_with_int_second_argument(self): with self.assertRaises(TypeError): type_utils.is_concrete_instance_of( computation_types.to_type(tf.int32), 1)
def test_is_concrete_instance_of_succeeds_abstract_type_under_sequence_type( self): t1 = computation_types.SequenceType( computation_types.AbstractType('T')) t2 = computation_types.SequenceType(tf.int32) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_fails_conflicting_concrete_types_under_sequence( self): t1 = computation_types.SequenceType( [computation_types.AbstractType('T')] * 2) t2 = computation_types.SequenceType([tf.int32, tf.float32]) self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
async def create_value(self, value, type_spec=None): type_spec = computation_types.to_type(type_spec) if isinstance(value, intrinsic_defs.IntrinsicDef): if not type_utils.is_concrete_instance_of(type_spec, value.type_signature): raise TypeError( 'Incompatible type {} used with intrinsic {}.'.format( type_spec, value.uri)) else: return FederatedExecutorValue(value, type_spec) if isinstance(value, placement_literals.PlacementLiteral): if type_spec is not None: py_typecheck.check_type(type_spec, computation_types.PlacementType) return FederatedExecutorValue(value, computation_types.PlacementType()) elif isinstance(value, computation_impl.ComputationImpl): return await self.create_value( computation_impl.ComputationImpl.get_proto(value), type_utils.reconcile_value_with_type_spec(value, type_spec)) elif isinstance(value, pb.Computation): if type_spec is None: type_spec = type_serialization.deserialize_type(value.type) which_computation = value.WhichOneof('computation') if which_computation in ['tensorflow', 'lambda']: return FederatedExecutorValue(value, type_spec) elif which_computation == 'reference': raise ValueError( 'Encountered an unexpected unbound references "{}".'. format(value.reference.name)) elif which_computation == 'intrinsic': intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri) if intr is None: raise ValueError( 'Encountered an unrecognized intrinsic "{}".'.format( value.intrinsic.uri)) py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef) return await self.create_value(intr, type_spec) elif which_computation == 'placement': return await self.create_value( placement_literals.uri_to_placement_literal( value.placement.uri), type_spec) elif which_computation == 'call': parts = [value.call.function] if value.call.argument.WhichOneof('computation'): parts.append(value.call.argument) parts = await asyncio.gather( *[self.create_value(x) for x in parts]) return await self.create_call( parts[0], parts[1] if len(parts) > 1 else None) elif which_computation == 'tuple': element_values = await asyncio.gather( *[self.create_value(x.value) for x in value.tuple.element]) return await self.create_tuple( anonymous_tuple.AnonymousTuple([ (e.name if e.name else None, v) for e, v in zip(value.tuple.element, element_values) ])) elif which_computation == 'selection': which_selection = value.selection.WhichOneof('selection') if which_selection == 'name': name = value.selection.name index = None elif which_selection != 'index': raise ValueError( 'Unrecognized selection type: "{}".'.format( which_selection)) else: index = value.selection.index name = None return await self.create_selection(await self.create_value( value.selection.source), index=index, name=name) else: raise ValueError( 'Unsupported computation building block of type "{}".'. format(which_computation)) else: py_typecheck.check_type(type_spec, computation_types.Type) if isinstance(type_spec, computation_types.FunctionType): raise ValueError( 'Encountered a value of a functional TFF type {} and Python type ' '{} that is not of one of the recognized representations.'. format(type_spec, py_typecheck.type_string(type(value)))) elif isinstance(type_spec, computation_types.FederatedType): children = self._target_executors.get(type_spec.placement) if not children: raise ValueError( 'Placement "{}" is not configured in this executor.'. format(type_spec.placement)) py_typecheck.check_type(children, list) if not type_spec.all_equal: py_typecheck.check_type(value, (list, tuple, set, frozenset)) if not isinstance(value, list): value = list(value) elif isinstance(value, list): raise ValueError( 'An all_equal value should be passed directly, not as a list.' ) else: value = [value for _ in children] if len(value) != len(children): raise ValueError( 'Federated value contains {} items, but the placement {} in this ' 'executor is configured with {} participants.'.format( len(value), type_spec.placement, len(children))) child_vals = await asyncio.gather(*[ c.create_value(v, type_spec.member) for v, c in zip(value, children) ]) return FederatedExecutorValue(child_vals, type_spec) else: child = self._target_executors.get(None) if not child or len(child) > 1: raise RuntimeError( 'Executor is not configured for unplaced values.') else: return FederatedExecutorValue( await child[0].create_value(value, type_spec), type_spec)
def test_is_concrete_instance_of_raises_different_structures(self): with self.assertRaises(TypeError): type_utils.is_concrete_instance_of( computation_types.to_type(tf.int32), computation_types.to_type([tf.int32]))
def test_is_concrete_instance_of_raises_with_different_lengths(self): t1 = computation_types.NamedTupleType( [computation_types.AbstractType('T1')] * 2) t2 = computation_types.NamedTupleType([tf.int32]) with self.assertRaises(TypeError): type_utils.is_concrete_instance_of(t2, t1)
def test_is_concrete_instance_of_raises_with_abstract_type_as_first_arg( self): t1 = computation_types.AbstractType('T1') t2 = computation_types.TensorType(tf.int32) with self.assertRaises(TypeError): type_utils.is_concrete_instance_of(t1, t2)
def test_is_concrete_instance_of_with_single_abstract_type_and_tuple_type( self): t1 = computation_types.AbstractType('T1') t2 = computation_types.NamedTupleType([tf.int32]) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_raises_with_abstract_type_in_second_argument( self): t1 = computation_types.AbstractType('T1') t2 = computation_types.AbstractType('T2') with self.assertRaises(TypeError): type_utils.is_concrete_instance_of(t2, t1)
def test_is_concrete_instance_of_with_single_abstract_type_and_tensor_type( self): t1 = computation_types.AbstractType('T1') t2 = computation_types.TensorType(tf.int32) self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
async def create_value(self, value, type_spec=None): type_spec = computation_types.to_type(type_spec) py_typecheck.check_type(type_spec, computation_types.Type) if isinstance(value, intrinsic_defs.IntrinsicDef): if not type_utils.is_concrete_instance_of(type_spec, value.type_signature): # pytype: disable=attribute-error raise TypeError('Incompatible type {} used with intrinsic {}.'.format( type_spec, value.uri)) # pytype: disable=attribute-error else: return CompositeValue(value, type_spec) elif isinstance(value, pb.Computation): which_computation = value.WhichOneof('computation') if which_computation in ['tensorflow', 'lambda']: return CompositeValue(value, type_spec) elif which_computation == 'intrinsic': intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri) if intr is None: raise ValueError('Encountered an unrecognized intrinsic "{}".'.format( value.intrinsic.uri)) py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef) return await self.create_value(intr, type_spec) else: raise NotImplementedError( 'Unimplemented computation type {}.'.format(which_computation)) elif isinstance(type_spec, computation_types.NamedTupleType): v_el = anonymous_tuple.to_elements(anonymous_tuple.from_container(value)) t_el = anonymous_tuple.to_elements(type_spec) items = await asyncio.gather( *[self.create_value(v, t) for (_, v), (_, t) in zip(v_el, t_el)]) return self.create_tuple( anonymous_tuple.AnonymousTuple([ (k, i) for (k, _), i in zip(t_el, items) ])) elif isinstance(type_spec, computation_types.FederatedType): if type_spec.placement == placement_literals.SERVER: if type_spec.all_equal: return CompositeValue( await self._parent_executor.create_value(value, type_spec.member), type_spec) else: raise ValueError('A non-all_equal value on the server is unexpected.') elif type_spec.placement == placement_literals.CLIENTS: if type_spec.all_equal: return CompositeValue( await asyncio.gather(*[ c.create_value(value, type_spec) for c in self._child_executors ]), type_spec) else: py_typecheck.check_type(value, list) if self._cardinalities is None: self._cardinalities = asyncio.ensure_future( self._get_cardinalities()) cardinalities = await self._cardinalities py_typecheck.check_len(cardinalities, len(self._child_executors)) count = sum(cardinalities) py_typecheck.check_len(value, count) result = [] offset = 0 for c, n in zip(self._child_executors, cardinalities): new_offset = offset + n # The slice opporator is not supported on all the types `value` # supports. # pytype: disable=unsupported-operands result.append(c.create_value(value[offset:new_offset], type_spec)) # pytype: enable=unsupported-operands offset = new_offset return CompositeValue(await asyncio.gather(*result), type_spec) else: raise ValueError('Unexpected placement {}.'.format(type_spec.placement)) else: return CompositeValue( await self._parent_executor.create_value(value, type_spec), type_spec)
async def create_value(self, value, type_spec=None): """A coroutine that creates embedded value from `value` of type `type_spec`. See the `FederatingExecutorValue` for detailed information about the `value`s and `type_spec`s that can be embedded using `create_value`. Args: value: An object that represents the value to embed within the executor. type_spec: An optional `tff.Type` of the value represented by this object, or something convertible to it. Returns: An instance of `FederatingExecutorValue` that represents the embedded value. Raises: TypeError: If the `value` and `type_spec` do not match. ValueError: If `value` is not a kind recognized by the `FederatingExecutor`. """ type_spec = computation_types.to_type(type_spec) if isinstance(type_spec, computation_types.FederatedType): self._check_executor_compatible_with_placement(type_spec.placement) elif (isinstance(type_spec, computation_types.FunctionType) and isinstance(type_spec.result, computation_types.FederatedType)): self._check_executor_compatible_with_placement(type_spec.result.placement) if isinstance(value, intrinsic_defs.IntrinsicDef): if not type_utils.is_concrete_instance_of(type_spec, value.type_signature): raise TypeError('Incompatible type {} used with intrinsic {}.'.format( type_spec, value.uri)) return FederatingExecutorValue(value, type_spec) elif isinstance(value, placement_literals.PlacementLiteral): if type_spec is None: type_spec = computation_types.PlacementType() else: py_typecheck.check_type(type_spec, computation_types.PlacementType) return FederatingExecutorValue(value, type_spec) elif isinstance(value, computation_impl.ComputationImpl): return await self.create_value( computation_impl.ComputationImpl.get_proto(value), type_utils.reconcile_value_with_type_spec(value, type_spec)) elif isinstance(value, pb.Computation): if type_spec is None: type_spec = type_serialization.deserialize_type(value.type) which_computation = value.WhichOneof('computation') if which_computation in ['lambda', 'tensorflow']: return FederatingExecutorValue(value, type_spec) elif which_computation == 'reference': raise ValueError( 'Encountered an unexpected unbound references "{}".'.format( value.reference.name)) elif which_computation == 'intrinsic': intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri) if intr is None: raise ValueError('Encountered an unrecognized intrinsic "{}".'.format( value.intrinsic.uri)) py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef) return await self.create_value(intr, type_spec) elif which_computation == 'placement': return await self.create_value( placement_literals.uri_to_placement_literal(value.placement.uri), type_spec) elif which_computation == 'call': parts = [value.call.function] if value.call.argument.WhichOneof('computation'): parts.append(value.call.argument) parts = await asyncio.gather(*[self.create_value(x) for x in parts]) return await self.create_call(parts[0], parts[1] if len(parts) > 1 else None) elif which_computation == 'tuple': element_values = await asyncio.gather( *[self.create_value(x.value) for x in value.tuple.element]) return await self.create_tuple( anonymous_tuple.AnonymousTuple( (e.name if e.name else None, v) for e, v in zip(value.tuple.element, element_values))) elif which_computation == 'selection': which_selection = value.selection.WhichOneof('selection') if which_selection == 'name': name = value.selection.name index = None elif which_selection != 'index': raise ValueError( 'Unrecognized selection type: "{}".'.format(which_selection)) else: index = value.selection.index name = None return await self.create_selection( await self.create_value(value.selection.source), index=index, name=name) else: raise ValueError( 'Unsupported computation building block of type "{}".'.format( which_computation)) else: py_typecheck.check_type(type_spec, computation_types.Type) if isinstance(type_spec, computation_types.FunctionType): raise ValueError( 'Encountered a value of a functional TFF type {} and Python type ' '{} that is not of one of the recognized representations.'.format( type_spec, py_typecheck.type_string(type(value)))) elif isinstance(type_spec, computation_types.FederatedType): children = self._target_executors.get(type_spec.placement) self._check_value_compatible_with_placement(value, type_spec.placement, type_spec.all_equal) if type_spec.all_equal: value = [value for _ in children] child_vals = await asyncio.gather(*[ c.create_value(v, type_spec.member) for v, c in zip(value, children) ]) return FederatingExecutorValue(child_vals, type_spec) else: child = self._target_executors.get(None) if not child or len(child) > 1: raise ValueError('Executor is not configured for unplaced values.') else: return FederatingExecutorValue( await child[0].create_value(value, type_spec), type_spec)