def test_is_concrete_instance_of_raises_with_conflicting_names(self):
t1 = computation_types.NamedTupleType(
[computation_types.AbstractType('T1')] * 2)
t2 = computation_types.NamedTupleType([('a', tf.int32),
('b', tf.int32)])
with self.assertRaises(TypeError):
type_utils.is_concrete_instance_of(t2, t1)
def __init__(self, uri, type_spec):
"""Creates an intrinsic.
Args:
uri: The URI of the intrinsic.
type_spec: Either the types.Type that represents the type of this
intrinsic, or something convertible to it by types.to_type().
Raises:
TypeError: if the arguments are of the wrong types.
"""
py_typecheck.check_type(uri, six.string_types)
if type_spec is None:
raise TypeError(
'Intrinsic {} cannot be created without a TFF type.'.format(
uri))
type_spec = computation_types.to_type(type_spec)
intrinsic_def = intrinsic_defs.uri_to_intrinsic_def(uri)
if intrinsic_def:
typecheck = type_utils.is_concrete_instance_of(
type_spec, intrinsic_def.type_signature)
if not typecheck:
raise TypeError(
'Tried to construct an Intrinsic with bad type '
'signature; Intrinsic {} expects type signature {}, '
'and you tried to construct one of type {}.'.format(
uri, intrinsic_def.type_signature, type_spec))
super(Intrinsic, self).__init__(type_spec)
self._uri = uri
def test_is_concrete_instance_of_succeeds_function_different_parameter_and_return_types(
self):
t1 = computation_types.FunctionType(
computation_types.AbstractType('T'),
computation_types.AbstractType('U'))
t2 = computation_types.FunctionType(tf.int32, tf.float32)
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_succeeds_under_tuple(self):
t1 = computation_types.NamedTupleType(
[computation_types.AbstractType('T1')] * 2)
t2 = computation_types.NamedTupleType([
computation_types.TensorType(tf.int32),
computation_types.TensorType(tf.int32)
])
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_fails_under_tuple_conflicting_concrete_types(
self):
t1 = computation_types.NamedTupleType(
[computation_types.AbstractType('T1')] * 2)
t2 = computation_types.NamedTupleType([
computation_types.TensorType(tf.int32),
computation_types.TensorType(tf.float32)
])
self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_abstract_federated_types_succeeds(self):
t1 = computation_types.FederatedType(
[computation_types.AbstractType('T1')] * 2,
placements.CLIENTS,
all_equal=True)
t2 = computation_types.FederatedType([tf.int32] * 2,
placements.CLIENTS,
all_equal=True)
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_abstract_can_be_concretized_abstract_fails_on_different_federated_all_equal_bits(
self):
t1 = computation_types.FederatedType(
[computation_types.AbstractType('T1')] * 2,
placements.CLIENTS,
all_equal=True)
t2 = computation_types.FederatedType([tf.int32] * 2,
placements.SERVER,
all_equal=True)
self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
async def create_value(self, value, type_spec=None):
type_spec = computation_types.to_type(type_spec)
py_typecheck.check_type(type_spec, computation_types.Type)
if isinstance(value, intrinsic_defs.IntrinsicDef):
if not type_utils.is_concrete_instance_of(type_spec,
value.type_signature): # pytype: disable=attribute-error
raise TypeError('Incompatible type {} used with intrinsic {}.'.format(
type_spec, value.uri)) # pytype: disable=attribute-error
else:
return CompositeValue(value, type_spec)
elif isinstance(value, pb.Computation):
which_computation = value.WhichOneof('computation')
if which_computation in ['tensorflow', 'lambda']:
return CompositeValue(value, type_spec)
elif which_computation == 'intrinsic':
intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri)
if intr is None:
raise ValueError('Encountered an unrecognized intrinsic "{}".'.format(
value.intrinsic.uri))
py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef)
return await self.create_value(intr, type_spec)
else:
raise NotImplementedError(
'Unimplemented computation type {}.'.format(which_computation))
elif isinstance(type_spec, computation_types.NamedTupleType):
value_tuple = anonymous_tuple.from_container(value)
items = await asyncio.gather(
*[self.create_value(v, t) for v, t in zip(value_tuple, type_spec)])
type_elemnents_iter = anonymous_tuple.iter_elements(type_spec)
return self.create_tuple(
anonymous_tuple.AnonymousTuple(
(k, i) for (k, _), i in zip(type_elemnents_iter, items)))
elif isinstance(type_spec, computation_types.FederatedType):
if type_spec.placement == placement_literals.SERVER:
if not type_spec.all_equal:
raise ValueError(
'Expected an all equal value at the `SERVER` placement, '
'found {}.'.format(type_spec))
results = await self._parent_executor.create_value(
value, type_spec.member)
return CompositeValue(results, type_spec)
elif type_spec.placement == placement_literals.CLIENTS:
if type_spec.all_equal:
results = await asyncio.gather(*[
c.create_value(value, type_spec) for c in self._child_executors
])
return CompositeValue(results, type_spec)
else:
py_typecheck.check_type(value, list)
cardinalities = await self._get_cardinalities()
total_clients = sum(cardinalities)
py_typecheck.check_len(value, total_clients)
results = []
offset = 0
for child, num_clients in zip(self._child_executors, cardinalities):
new_offset = offset + num_clients
result = child.create_value(value[offset:new_offset], type_spec)
results.append(result)
offset = new_offset
return CompositeValue(await asyncio.gather(*results), type_spec)
else:
raise ValueError('Unexpected placement {}.'.format(type_spec.placement))
else:
return CompositeValue(
await self._parent_executor.create_value(value, type_spec), type_spec)
def test_is_concrete_instance_of_fails_conflicting_binding_in_parameter_and_result(
self):
t1 = computation_types.FunctionType(
*[computation_types.AbstractType('T')] * 2)
t2 = computation_types.FunctionType(tf.int32, tf.float32)
self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_succeeds_single_function_type(self):
t1 = computation_types.FunctionType(
*[computation_types.AbstractType('T')] * 2)
t2 = computation_types.FunctionType(tf.int32, tf.int32)
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_raises_with_int_second_argument(self):
with self.assertRaises(TypeError):
type_utils.is_concrete_instance_of(
computation_types.to_type(tf.int32), 1)
def test_is_concrete_instance_of_succeeds_abstract_type_under_sequence_type(
self):
t1 = computation_types.SequenceType(
computation_types.AbstractType('T'))
t2 = computation_types.SequenceType(tf.int32)
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_fails_conflicting_concrete_types_under_sequence(
self):
t1 = computation_types.SequenceType(
[computation_types.AbstractType('T')] * 2)
t2 = computation_types.SequenceType([tf.int32, tf.float32])
self.assertFalse(type_utils.is_concrete_instance_of(t2, t1))
async def create_value(self, value, type_spec=None):
type_spec = computation_types.to_type(type_spec)
if isinstance(value, intrinsic_defs.IntrinsicDef):
if not type_utils.is_concrete_instance_of(type_spec,
value.type_signature):
raise TypeError(
'Incompatible type {} used with intrinsic {}.'.format(
type_spec, value.uri))
else:
return FederatedExecutorValue(value, type_spec)
if isinstance(value, placement_literals.PlacementLiteral):
if type_spec is not None:
py_typecheck.check_type(type_spec,
computation_types.PlacementType)
return FederatedExecutorValue(value,
computation_types.PlacementType())
elif isinstance(value, computation_impl.ComputationImpl):
return await self.create_value(
computation_impl.ComputationImpl.get_proto(value),
type_utils.reconcile_value_with_type_spec(value, type_spec))
elif isinstance(value, pb.Computation):
if type_spec is None:
type_spec = type_serialization.deserialize_type(value.type)
which_computation = value.WhichOneof('computation')
if which_computation in ['tensorflow', 'lambda']:
return FederatedExecutorValue(value, type_spec)
elif which_computation == 'reference':
raise ValueError(
'Encountered an unexpected unbound references "{}".'.
format(value.reference.name))
elif which_computation == 'intrinsic':
intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri)
if intr is None:
raise ValueError(
'Encountered an unrecognized intrinsic "{}".'.format(
value.intrinsic.uri))
py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef)
return await self.create_value(intr, type_spec)
elif which_computation == 'placement':
return await self.create_value(
placement_literals.uri_to_placement_literal(
value.placement.uri), type_spec)
elif which_computation == 'call':
parts = [value.call.function]
if value.call.argument.WhichOneof('computation'):
parts.append(value.call.argument)
parts = await asyncio.gather(
*[self.create_value(x) for x in parts])
return await self.create_call(
parts[0], parts[1] if len(parts) > 1 else None)
elif which_computation == 'tuple':
element_values = await asyncio.gather(
*[self.create_value(x.value) for x in value.tuple.element])
return await self.create_tuple(
anonymous_tuple.AnonymousTuple([
(e.name if e.name else None, v)
for e, v in zip(value.tuple.element, element_values)
]))
elif which_computation == 'selection':
which_selection = value.selection.WhichOneof('selection')
if which_selection == 'name':
name = value.selection.name
index = None
elif which_selection != 'index':
raise ValueError(
'Unrecognized selection type: "{}".'.format(
which_selection))
else:
index = value.selection.index
name = None
return await self.create_selection(await self.create_value(
value.selection.source),
index=index,
name=name)
else:
raise ValueError(
'Unsupported computation building block of type "{}".'.
format(which_computation))
else:
py_typecheck.check_type(type_spec, computation_types.Type)
if isinstance(type_spec, computation_types.FunctionType):
raise ValueError(
'Encountered a value of a functional TFF type {} and Python type '
'{} that is not of one of the recognized representations.'.
format(type_spec, py_typecheck.type_string(type(value))))
elif isinstance(type_spec, computation_types.FederatedType):
children = self._target_executors.get(type_spec.placement)
if not children:
raise ValueError(
'Placement "{}" is not configured in this executor.'.
format(type_spec.placement))
py_typecheck.check_type(children, list)
if not type_spec.all_equal:
py_typecheck.check_type(value,
(list, tuple, set, frozenset))
if not isinstance(value, list):
value = list(value)
elif isinstance(value, list):
raise ValueError(
'An all_equal value should be passed directly, not as a list.'
)
else:
value = [value for _ in children]
if len(value) != len(children):
raise ValueError(
'Federated value contains {} items, but the placement {} in this '
'executor is configured with {} participants.'.format(
len(value), type_spec.placement, len(children)))
child_vals = await asyncio.gather(*[
c.create_value(v, type_spec.member)
for v, c in zip(value, children)
])
return FederatedExecutorValue(child_vals, type_spec)
else:
child = self._target_executors.get(None)
if not child or len(child) > 1:
raise RuntimeError(
'Executor is not configured for unplaced values.')
else:
return FederatedExecutorValue(
await child[0].create_value(value, type_spec),
type_spec)
def test_is_concrete_instance_of_raises_different_structures(self):
with self.assertRaises(TypeError):
type_utils.is_concrete_instance_of(
computation_types.to_type(tf.int32),
computation_types.to_type([tf.int32]))
def test_is_concrete_instance_of_raises_with_different_lengths(self):
t1 = computation_types.NamedTupleType(
[computation_types.AbstractType('T1')] * 2)
t2 = computation_types.NamedTupleType([tf.int32])
with self.assertRaises(TypeError):
type_utils.is_concrete_instance_of(t2, t1)
def test_is_concrete_instance_of_raises_with_abstract_type_as_first_arg(
self):
t1 = computation_types.AbstractType('T1')
t2 = computation_types.TensorType(tf.int32)
with self.assertRaises(TypeError):
type_utils.is_concrete_instance_of(t1, t2)
def test_is_concrete_instance_of_with_single_abstract_type_and_tuple_type(
self):
t1 = computation_types.AbstractType('T1')
t2 = computation_types.NamedTupleType([tf.int32])
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
def test_is_concrete_instance_of_raises_with_abstract_type_in_second_argument(
self):
t1 = computation_types.AbstractType('T1')
t2 = computation_types.AbstractType('T2')
with self.assertRaises(TypeError):
type_utils.is_concrete_instance_of(t2, t1)
def test_is_concrete_instance_of_with_single_abstract_type_and_tensor_type(
self):
t1 = computation_types.AbstractType('T1')
t2 = computation_types.TensorType(tf.int32)
self.assertTrue(type_utils.is_concrete_instance_of(t2, t1))
async def create_value(self, value, type_spec=None):
type_spec = computation_types.to_type(type_spec)
py_typecheck.check_type(type_spec, computation_types.Type)
if isinstance(value, intrinsic_defs.IntrinsicDef):
if not type_utils.is_concrete_instance_of(type_spec,
value.type_signature): # pytype: disable=attribute-error
raise TypeError('Incompatible type {} used with intrinsic {}.'.format(
type_spec, value.uri)) # pytype: disable=attribute-error
else:
return CompositeValue(value, type_spec)
elif isinstance(value, pb.Computation):
which_computation = value.WhichOneof('computation')
if which_computation in ['tensorflow', 'lambda']:
return CompositeValue(value, type_spec)
elif which_computation == 'intrinsic':
intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri)
if intr is None:
raise ValueError('Encountered an unrecognized intrinsic "{}".'.format(
value.intrinsic.uri))
py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef)
return await self.create_value(intr, type_spec)
else:
raise NotImplementedError(
'Unimplemented computation type {}.'.format(which_computation))
elif isinstance(type_spec, computation_types.NamedTupleType):
v_el = anonymous_tuple.to_elements(anonymous_tuple.from_container(value))
t_el = anonymous_tuple.to_elements(type_spec)
items = await asyncio.gather(
*[self.create_value(v, t) for (_, v), (_, t) in zip(v_el, t_el)])
return self.create_tuple(
anonymous_tuple.AnonymousTuple([
(k, i) for (k, _), i in zip(t_el, items)
]))
elif isinstance(type_spec, computation_types.FederatedType):
if type_spec.placement == placement_literals.SERVER:
if type_spec.all_equal:
return CompositeValue(
await self._parent_executor.create_value(value, type_spec.member),
type_spec)
else:
raise ValueError('A non-all_equal value on the server is unexpected.')
elif type_spec.placement == placement_literals.CLIENTS:
if type_spec.all_equal:
return CompositeValue(
await asyncio.gather(*[
c.create_value(value, type_spec)
for c in self._child_executors
]), type_spec)
else:
py_typecheck.check_type(value, list)
if self._cardinalities is None:
self._cardinalities = asyncio.ensure_future(
self._get_cardinalities())
cardinalities = await self._cardinalities
py_typecheck.check_len(cardinalities, len(self._child_executors))
count = sum(cardinalities)
py_typecheck.check_len(value, count)
result = []
offset = 0
for c, n in zip(self._child_executors, cardinalities):
new_offset = offset + n
# The slice opporator is not supported on all the types `value`
# supports.
# pytype: disable=unsupported-operands
result.append(c.create_value(value[offset:new_offset], type_spec))
# pytype: enable=unsupported-operands
offset = new_offset
return CompositeValue(await asyncio.gather(*result), type_spec)
else:
raise ValueError('Unexpected placement {}.'.format(type_spec.placement))
else:
return CompositeValue(
await self._parent_executor.create_value(value, type_spec), type_spec)
async def create_value(self, value, type_spec=None):
"""A coroutine that creates embedded value from `value` of type `type_spec`.
See the `FederatingExecutorValue` for detailed information about the
`value`s and `type_spec`s that can be embedded using `create_value`.
Args:
value: An object that represents the value to embed within the executor.
type_spec: An optional `tff.Type` of the value represented by this object,
or something convertible to it.
Returns:
An instance of `FederatingExecutorValue` that represents the embedded
value.
Raises:
TypeError: If the `value` and `type_spec` do not match.
ValueError: If `value` is not a kind recognized by the
`FederatingExecutor`.
"""
type_spec = computation_types.to_type(type_spec)
if isinstance(type_spec, computation_types.FederatedType):
self._check_executor_compatible_with_placement(type_spec.placement)
elif (isinstance(type_spec, computation_types.FunctionType) and
isinstance(type_spec.result, computation_types.FederatedType)):
self._check_executor_compatible_with_placement(type_spec.result.placement)
if isinstance(value, intrinsic_defs.IntrinsicDef):
if not type_utils.is_concrete_instance_of(type_spec,
value.type_signature):
raise TypeError('Incompatible type {} used with intrinsic {}.'.format(
type_spec, value.uri))
return FederatingExecutorValue(value, type_spec)
elif isinstance(value, placement_literals.PlacementLiteral):
if type_spec is None:
type_spec = computation_types.PlacementType()
else:
py_typecheck.check_type(type_spec, computation_types.PlacementType)
return FederatingExecutorValue(value, type_spec)
elif isinstance(value, computation_impl.ComputationImpl):
return await self.create_value(
computation_impl.ComputationImpl.get_proto(value),
type_utils.reconcile_value_with_type_spec(value, type_spec))
elif isinstance(value, pb.Computation):
if type_spec is None:
type_spec = type_serialization.deserialize_type(value.type)
which_computation = value.WhichOneof('computation')
if which_computation in ['lambda', 'tensorflow']:
return FederatingExecutorValue(value, type_spec)
elif which_computation == 'reference':
raise ValueError(
'Encountered an unexpected unbound references "{}".'.format(
value.reference.name))
elif which_computation == 'intrinsic':
intr = intrinsic_defs.uri_to_intrinsic_def(value.intrinsic.uri)
if intr is None:
raise ValueError('Encountered an unrecognized intrinsic "{}".'.format(
value.intrinsic.uri))
py_typecheck.check_type(intr, intrinsic_defs.IntrinsicDef)
return await self.create_value(intr, type_spec)
elif which_computation == 'placement':
return await self.create_value(
placement_literals.uri_to_placement_literal(value.placement.uri),
type_spec)
elif which_computation == 'call':
parts = [value.call.function]
if value.call.argument.WhichOneof('computation'):
parts.append(value.call.argument)
parts = await asyncio.gather(*[self.create_value(x) for x in parts])
return await self.create_call(parts[0],
parts[1] if len(parts) > 1 else None)
elif which_computation == 'tuple':
element_values = await asyncio.gather(
*[self.create_value(x.value) for x in value.tuple.element])
return await self.create_tuple(
anonymous_tuple.AnonymousTuple(
(e.name if e.name else None, v)
for e, v in zip(value.tuple.element, element_values)))
elif which_computation == 'selection':
which_selection = value.selection.WhichOneof('selection')
if which_selection == 'name':
name = value.selection.name
index = None
elif which_selection != 'index':
raise ValueError(
'Unrecognized selection type: "{}".'.format(which_selection))
else:
index = value.selection.index
name = None
return await self.create_selection(
await self.create_value(value.selection.source),
index=index,
name=name)
else:
raise ValueError(
'Unsupported computation building block of type "{}".'.format(
which_computation))
else:
py_typecheck.check_type(type_spec, computation_types.Type)
if isinstance(type_spec, computation_types.FunctionType):
raise ValueError(
'Encountered a value of a functional TFF type {} and Python type '
'{} that is not of one of the recognized representations.'.format(
type_spec, py_typecheck.type_string(type(value))))
elif isinstance(type_spec, computation_types.FederatedType):
children = self._target_executors.get(type_spec.placement)
self._check_value_compatible_with_placement(value, type_spec.placement,
type_spec.all_equal)
if type_spec.all_equal:
value = [value for _ in children]
child_vals = await asyncio.gather(*[
c.create_value(v, type_spec.member)
for v, c in zip(value, children)
])
return FederatingExecutorValue(child_vals, type_spec)
else:
child = self._target_executors.get(None)
if not child or len(child) > 1:
raise ValueError('Executor is not configured for unplaced values.')
else:
return FederatingExecutorValue(
await child[0].create_value(value, type_spec), type_spec)
