def test_deserialize_federated_value_with_incompatible_member_types_raises( self): x = 10 x_type = computation_types.to_type(tf.int32) int_member_proto, _ = value_serialization.serialize_value(x, x_type) y = 10. y_type = computation_types.to_type(tf.float32) float_member_proto, _ = value_serialization.serialize_value(y, y_type) fully_specified_type_at_clients = type_serialization.serialize_type( computation_types.at_clients(tf.int32)) unspecified_member_federated_type = computation_pb2.FederatedType( placement=fully_specified_type_at_clients.federated.placement, all_equal=False) federated_proto = serialization_bindings.Federated( type=unspecified_member_federated_type, value=[int_member_proto, float_member_proto]) federated_value_proto = serialization_bindings.Value( federated=federated_proto) self.assertIsInstance(int_member_proto, serialization_bindings.Value) self.assertIsInstance(float_member_proto, serialization_bindings.Value) self.assertIsInstance(federated_value_proto, serialization_bindings.Value) with self.assertRaises(TypeError): value_serialization.deserialize_value(federated_value_proto)
def test_create_selection(self): executor = executor_bindings.create_reference_resolving_executor( executor_bindings.create_tensorflow_executor()) expected_type_spec = TensorType(shape=[3], dtype=tf.int64) value_pb, _ = value_serialization.serialize_value( tf.constant([1, 2, 3]), expected_type_spec) value = executor.create_value(value_pb) self.assertEqual(value.ref, 0) # 1. Create a struct from duplicated values. struct_value = executor.create_struct([value.ref, value.ref]) self.assertEqual(struct_value.ref, 1) materialized_value = executor.materialize(struct_value.ref) deserialized_value, type_spec = value_serialization.deserialize_value( materialized_value) struct_type_spec = computation_types.to_type( [expected_type_spec, expected_type_spec]) type_test_utils.assert_types_equivalent(type_spec, struct_type_spec) deserialized_value = type_conversions.type_to_py_container( deserialized_value, struct_type_spec) self.assertAllClose([(1, 2, 3), (1, 2, 3)], deserialized_value) # 2. Select the first value out of the struct. new_value = executor.create_selection(struct_value.ref, 0) materialized_value = executor.materialize(new_value.ref) deserialized_value, type_spec = value_serialization.deserialize_value( materialized_value) type_test_utils.assert_types_equivalent(type_spec, expected_type_spec) deserialized_value = type_conversions.type_to_py_container( deserialized_value, struct_type_spec) self.assertAllClose((1, 2, 3), deserialized_value)
def test_create_value(self): executor = executor_bindings.create_reference_resolving_executor( executor_bindings.create_tensorflow_executor()) # 1. Test a simple tensor. expected_type_spec = TensorType(shape=[3], dtype=tf.int64) value_pb, _ = value_serialization.serialize_value([1, 2, 3], expected_type_spec) value = executor.create_value(value_pb) self.assertIsInstance(value, executor_bindings.OwnedValueId) self.assertEqual(value.ref, 0) self.assertEqual(str(value), '0') self.assertEqual(repr(value), r'<OwnedValueId: 0>') materialized_value = executor.materialize(value.ref) deserialized_value, type_spec = value_serialization.deserialize_value( materialized_value) type_test_utils.assert_types_identical(type_spec, expected_type_spec) self.assertAllEqual(deserialized_value, [1, 2, 3]) # 2. Test a struct of tensors, ensure that we get a different ID. expected_type_spec = StructType([ ('a', TensorType(shape=[3], dtype=tf.int64)), ('b', TensorType(shape=[], dtype=tf.float32)) ]) value_pb, _ = value_serialization.serialize_value( collections.OrderedDict(a=tf.constant([1, 2, 3]), b=tf.constant(42.0)), expected_type_spec) value = executor.create_value(value_pb) self.assertIsInstance(value, executor_bindings.OwnedValueId) # Assert the value ID was incremented. self.assertEqual(value.ref, 1) self.assertEqual(str(value), '1') self.assertEqual(repr(value), r'<OwnedValueId: 1>') materialized_value = executor.materialize(value.ref) deserialized_value, type_spec = value_serialization.deserialize_value( materialized_value) # Note: here we've lost the names `a` and `b` in the output. The output # is a more _strict_ type. self.assertTrue(expected_type_spec.is_assignable_from(type_spec)) deserialized_value = type_conversions.type_to_py_container( deserialized_value, expected_type_spec) self.assertAllClose(deserialized_value, collections.OrderedDict(a=(1, 2, 3), b=42.0)) # 3. Test creating a value from a computation. @tensorflow_computation.tf_computation(tf.int32, tf.int32) def foo(a, b): return tf.add(a, b) value_pb, _ = value_serialization.serialize_value(foo) value = executor.create_value(value_pb) self.assertIsInstance(value, executor_bindings.OwnedValueId) # Assert the value ID was incremented again. self.assertEqual(value.ref, 2) self.assertEqual(str(value), '2') self.assertEqual(repr(value), '<OwnedValueId: 2>')
def test_create_tuple_of_value_sequence(self): datasets = (tf.data.Dataset.range(5), tf.data.Dataset.range(5)) executor = executor_bindings.create_tensorflow_executor() struct_of_sequence_type = StructType([ (None, SequenceType(datasets[0].element_spec)), (None, SequenceType(datasets[0].element_spec)) ]) arg_value_pb, _ = value_serialization.serialize_value( datasets, struct_of_sequence_type) arg = executor.create_value(arg_value_pb) @tensorflow_computation.tf_computation(struct_of_sequence_type) def preprocess(datasets): def double_value(x): return 2 * x @tf.function def add_preprocessing(ds1, ds2): return ds1.map(double_value), ds2.map(double_value) return add_preprocessing(*datasets) comp_pb = executor_pb2.Value( computation=preprocess.get_proto(preprocess)) comp = executor.create_value(comp_pb) result = executor.create_call(comp.ref, arg.ref) output_pb = executor.materialize(result.ref) result, result_type_spec = value_serialization.deserialize_value( output_pb, type_hint=struct_of_sequence_type) type_test_utils.assert_types_identical(result_type_spec, struct_of_sequence_type)
def test_deserialize_federated_value_promotes_types(self): x = [10] smaller_type = computation_types.StructType([ (None, computation_types.to_type(tf.int32)) ]) smaller_type_member_proto, _ = value_serialization.serialize_value( x, smaller_type) larger_type = computation_types.StructType([ ('a', computation_types.to_type(tf.int32)) ]) larger_type_member_proto, _ = value_serialization.serialize_value( x, larger_type) type_at_clients = type_serialization.serialize_type( computation_types.at_clients(tf.int32)) unspecified_member_federated_type = computation_pb2.FederatedType( placement=type_at_clients.federated.placement, all_equal=False) federated_proto = executor_pb2.Value.Federated( type=unspecified_member_federated_type, value=[larger_type_member_proto, smaller_type_member_proto]) federated_value_proto = executor_pb2.Value(federated=federated_proto) _, deserialized_type_spec = value_serialization.deserialize_value( federated_value_proto) type_test_utils.assert_types_identical( deserialized_type_spec, computation_types.at_clients(larger_type))
def test_serialize_deserialize_variable_as_tensor_value(self): x = tf.Variable(10.0) type_spec = TensorType(tf.as_dtype(x.dtype), x.shape) value_proto, value_type = value_serialization.serialize_value(x, type_spec) type_test_utils.assert_types_identical(value_type, TensorType(tf.float32)) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical(type_spec, TensorType(tf.float32)) self.assertAllEqual(x, y)
def test_serialize_deserialize_tensor_value_with_different_dtype(self): x = tf.constant(10) value_proto, value_type = value_serialization.serialize_value( x, TensorType(tf.float32)) type_test_utils.assert_types_identical(value_type, TensorType(tf.float32)) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical(type_spec, TensorType(tf.float32)) self.assertEqual(y, 10.0)
def test_serialize_deserialize_nested_tuple_value_without_names(self): x = (10, 20) x_type = computation_types.to_type((tf.int32, tf.int32)) value_proto, value_type = value_serialization.serialize_value(x, x_type) type_test_utils.assert_types_identical(value_type, x_type) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_equivalent(type_spec, x_type) self.assertEqual(y, structure.from_container((10, 20)))
def test_serialize_deserialize_value_proto_for_generic(self, np_generic): type_spec = computation_types.TensorType( dtype=np_generic.dtype, shape=np_generic.shape) value_proto = value_serialization._value_proto_for_np_array( np_generic, type_spec) deserialized_array, deserialized_type = value_serialization.deserialize_value( value_proto) type_test_utils.assert_type_assignable_from(type_spec, deserialized_type) self.assertAllEqual(np_generic, deserialized_array)
def get_value(self, value_id: str): """Retrieves a value using the `Compute` endpoint.""" response = self._stub.Compute( executor_pb2.ComputeRequest( executor=self._executor_pb, value_ref=executor_pb2.ValueRef(id=value_id))) py_typecheck.check_type(response, executor_pb2.ComputeResponse) value, _ = value_serialization.deserialize_value(response.value) return value
def test_serialize_deserialize_tensor_value_without_hint( self, x, serialize_type_spec): value_proto, value_type = value_serialization.serialize_value( x, serialize_type_spec) type_test_utils.assert_types_identical(value_type, serialize_type_spec) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical(type_spec, serialize_type_spec) self.assertEqual(y.dtype, serialize_type_spec.dtype.as_numpy_dtype) self.assertAllEqual(x, y)
async def _compute(self, value_ref, type_spec): self._check_has_executor_id() py_typecheck.check_type(value_ref, executor_pb2.ValueRef) request = executor_pb2.ComputeRequest( executor=self._executor_id, value_ref=value_ref) response = self._stub.compute(request) py_typecheck.check_type(response, executor_pb2.ComputeResponse) value, _ = value_serialization.deserialize_value(response.value, type_spec) return value
def test_serialize_deserialize_federated_at_server(self): x = 10 x_type = computation_types.at_server(tf.int32) value_proto, value_type = value_serialization.serialize_value(x, x_type) type_test_utils.assert_types_identical( value_type, computation_types.at_server(tf.int32)) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical(type_spec, x_type) self.assertEqual(y, 10)
def test_serialize_deserialize_value_proto_for_array_undefined_shape(self): ndarray = np.array([10]) type_spec = computation_types.TensorType(dtype=ndarray.dtype, shape=[None]) value_proto = value_serialization._value_proto_for_np_array( ndarray, type_spec) deserialized_array, deserialized_type = value_serialization.deserialize_value( value_proto) type_test_utils.assert_type_assignable_from(type_spec, deserialized_type) self.assertAllEqual(ndarray, deserialized_array)
def test_serialize_deserialize_tensor_value_with_nontrivial_shape(self): x = tf.constant([10, 20, 30]) value_proto, value_type = value_serialization.serialize_value( x, TensorType(tf.int32, [3])) type_test_utils.assert_types_identical(value_type, TensorType(tf.int32, [3])) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical(type_spec, TensorType(tf.int32, [3])) self.assertAllEqual(x, y)
def test_serialize_deserialize_tensor_value_unk_shape_without_hint(self): x = np.asarray([1., 2.]) serialize_type_spec = TensorType(tf.float32, [None]) value_proto, value_type = value_serialization.serialize_value( x, serialize_type_spec) type_test_utils.assert_type_assignable_from(value_type, serialize_type_spec) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_type_assignable_from(serialize_type_spec, type_spec) self.assertEqual(y.dtype, serialize_type_spec.dtype.as_numpy_dtype) self.assertAllEqual(x, y)
def test_serialize_deserialize_sequence_of_scalars(self, dataset_fn): ds = tf.data.Dataset.range(5).map(lambda x: x * 2) ds_repr = dataset_fn(ds) value_proto, value_type = value_serialization.serialize_value( ds_repr, computation_types.SequenceType(tf.int64)) type_test_utils.assert_types_identical( value_type, computation_types.SequenceType(tf.int64)) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical( type_spec, computation_types.SequenceType(tf.int64)) self.assertAllEqual(list(y), [x * 2 for x in range(5)])
def test_serialize_deserialize_tensor_value(self): x = tf.constant(10.0) type_spec = computation_types.TensorType(tf.as_dtype(x.dtype), x.shape) value_proto, value_type = value_serialization.serialize_value(x, type_spec) self.assertIsInstance(value_proto, serialization_bindings.Value) self.assert_types_identical(value_type, computation_types.TensorType(tf.float32)) y, type_spec = value_serialization.deserialize_value(value_proto) self.assert_types_identical(type_spec, computation_types.TensorType(tf.float32)) self.assertAllEqual(x, y)
def test_serialize_deserialize_sequence_of_scalars_graph_mode(self): with tf.Graph().as_default(): ds = tf.data.Dataset.range(5).map(lambda x: x * 2) value_proto, value_type = value_serialization.serialize_value( ds, computation_types.SequenceType(tf.int64)) type_test_utils.assert_types_identical( value_type, computation_types.SequenceType(tf.int64)) y, type_spec = value_serialization.deserialize_value(value_proto) type_test_utils.assert_types_identical( type_spec, computation_types.SequenceType(tf.int64)) self.assertAllEqual(list(y), [x * 2 for x in range(5)])
def test_serialize_deserialize_tensor_value_with_different_dtype(self): x = tf.constant(10.0) value_proto, value_type = value_serialization.serialize_value( x, computation_types.TensorType(tf.int32)) self.assertIsInstance(value_proto, serialization_bindings.Value) self.assert_types_identical(value_type, computation_types.TensorType(tf.int32)) y, type_spec = value_serialization.deserialize_value(value_proto) self.assert_types_identical(type_spec, computation_types.TensorType(tf.int32)) self.assertEqual(y, 10)
def test_serialize_deserialize_tensor_value_with_nontrivial_shape(self): x = tf.constant([10, 20, 30]) value_proto, value_type = value_serialization.serialize_value( x, computation_types.TensorType(tf.int32, [3])) self.assertIsInstance(value_proto, serialization_bindings.Value) self.assert_types_identical(value_type, computation_types.TensorType(tf.int32, [3])) y, type_spec = value_serialization.deserialize_value(value_proto) self.assert_types_identical(type_spec, computation_types.TensorType(tf.int32, [3])) self.assertAllEqual(x, y)
def test_serialize_deserialize_sequence_of_scalars(self): ds = tf.data.Dataset.range(5).map(lambda x: x * 2) value_proto, value_type = value_serialization.serialize_value( ds, computation_types.SequenceType(tf.int64)) self.assertIsInstance(value_proto, serialization_bindings.Value) self.assert_types_identical(value_type, computation_types.SequenceType(tf.int64)) y, type_spec = value_serialization.deserialize_value(value_proto) self.assert_types_identical(type_spec, computation_types.SequenceType(tf.int64)) self.assertAllEqual(list(y), [x * 2 for x in range(5)])
def test_serialize_deserialize_federated_at_clients(self): x = [10, 20] x_type = computation_types.at_clients(tf.int32) value_proto, value_type = value_serialization.serialize_value(x, x_type) self.assertIsInstance(value_proto, serialization_bindings.Value) self.assert_types_identical(value_type, computation_types.at_clients(tf.int32)) y, type_spec = value_serialization.deserialize_value(value_proto) self.assert_types_identical(type_spec, computation_types.at_clients(tf.int32)) self.assertEqual(y, [10, 20])
def test_serialize_deserialize_sequence_of_ragged_tensors(self, dataset_fn): self.skipTest('b/235492749') ds = tf.data.Dataset.from_tensor_slices(tf.strings.split(['a b c', 'd e'])) ds_repr = dataset_fn(ds) value_proto, value_type = value_serialization.serialize_value( ds_repr, computation_types.SequenceType(element=ds.element_spec)) expected_type = computation_types.SequenceType(ds.element_spec) type_test_utils.assert_types_identical(value_type, expected_type) _, type_spec = value_serialization.deserialize_value(value_proto) # Only checking for equivalence, we don't have the Python container # after deserialization. type_test_utils.assert_types_equivalent(type_spec, expected_type)
def test_serialize_deserialize_string_value(self, x): tf_type = tf.as_dtype(x.dtype) type_spec = TensorType(tf_type, x.shape) value_proto, value_type = value_serialization.serialize_value(x, type_spec) type_test_utils.assert_types_identical(value_type, TensorType(tf_type, x.shape)) y, type_spec = value_serialization.deserialize_value( value_proto, type_hint=type_spec) type_test_utils.assert_types_identical(type_spec, TensorType(tf_type, x.shape)) self.assertIsInstance(y, bytes) self.assertAllEqual(x, y)
def test_call_no_arg(self): executor = executor_bindings.create_tensorflow_executor() @computations.tf_computation def foo(): return tf.constant(123.0) comp_pb = serialization_bindings.Value(computation=foo.get_proto(foo)) comp = executor.create_value(comp_pb) result = executor.create_call(comp.ref, None) result_value_pb = executor.materialize(result.ref) result_tensor, _ = value_serialization.deserialize_value(result_value_pb) self.assertEqual(result_tensor, 123.0)
def test_serialize_deserialize_tensor_value_without_hint_graph_mode( self, x, serialize_type_spec): # Test serializing non-eager tensors. with tf.Graph().as_default(): tensor_x = tf.convert_to_tensor(x) value_proto, value_type = value_serialization.serialize_value( tensor_x, serialize_type_spec) type_test_utils.assert_types_identical(value_type, serialize_type_spec) y, deserialize_type_spec = value_serialization.deserialize_value( value_proto) type_test_utils.assert_types_identical(deserialize_type_spec, serialize_type_spec) self.assertEqual(y.dtype, serialize_type_spec.dtype.as_numpy_dtype) self.assertAllEqual(x, y)
def test_serialize_deserialize_nested_tuple_value_with_names(self): x = collections.OrderedDict( a=10, b=[20, 30], c=collections.OrderedDict(d=40)) x_type = computation_types.to_type( collections.OrderedDict( a=tf.int32, b=[tf.int32, tf.int32], c=collections.OrderedDict(d=tf.int32))) value_proto, value_type = value_serialization.serialize_value(x, x_type) type_test_utils.assert_types_identical(value_type, x_type) y, type_spec = value_serialization.deserialize_value(value_proto) # Don't assert on the Python container since it is lost in serialization. type_test_utils.assert_types_equivalent(type_spec, x_type) self.assertEqual(y, structure.from_container(x, recursive=True))
def test_roundtrip_sequence_of_sparse_tensors(self): self.skipTest('b/225927855') tensors = (tf.constant([(1, 1)], dtype=tf.int64), (1.,), tf.constant((3, 3), dtype=tf.int64)) ds = tf.data.Dataset.from_tensors(tensors) ds = ds.map(lambda i, v, s: tf.SparseTensor(i, v, s)) # pylint: disable=unnecessary-lambda serialized_value, _ = value_serialization.serialize_value( ds, computation_types.SequenceType(ds.element_spec)) deserialized_ds, _ = value_serialization.deserialize_value(serialized_value) self.assertEqual(ds.element_spec, deserialized_ds.element_spec) self.assertAllEqual( list(deserialized_ds), [tf.sparse.SparseTensor([(1, 1)], (1.), (3, 3))])
def test_serialize_deserialize_computation_value(self): @tensorflow_computation.tf_computation def comp(): return tf.constant(10) value_proto, value_type = value_serialization.serialize_value(comp) self.assertEqual(value_proto.WhichOneof('value'), 'computation') type_test_utils.assert_types_identical( value_type, computation_types.FunctionType(parameter=None, result=tf.int32)) comp, type_spec = value_serialization.deserialize_value(value_proto) # self.assertIsInstance(comp, computation_pb2.Computation) type_test_utils.assert_types_identical( type_spec, computation_types.FunctionType(parameter=None, result=tf.int32))