def test_construction(self, weighted):
aggregation_factory = (mean.MeanFactory()
if weighted else sum_factory.SumFactory())
iterative_process = optimizer_utils.build_model_delta_optimizer_process(
model_fn=model_examples.LinearRegression,
model_to_client_delta_fn=DummyClientDeltaFn,
server_optimizer_fn=tf.keras.optimizers.SGD,
model_update_aggregation_factory=aggregation_factory)
if weighted:
aggregate_state = collections.OrderedDict(value_sum_process=(),
weight_sum_process=())
aggregate_metrics = collections.OrderedDict(mean_value=(),
mean_weight=())
else:
aggregate_state = ()
aggregate_metrics = ()
server_state_type = computation_types.FederatedType(
optimizer_utils.ServerState(model=model_utils.ModelWeights(
trainable=[
computation_types.TensorType(tf.float32, [2, 1]),
computation_types.TensorType(tf.float32)
],
non_trainable=[computation_types.TensorType(tf.float32)]),
optimizer_state=[tf.int64],
delta_aggregate_state=aggregate_state,
model_broadcast_state=()),
placements.SERVER)
self.assert_types_equivalent(
computation_types.FunctionType(parameter=None,
result=server_state_type),
iterative_process.initialize.type_signature)
dataset_type = computation_types.FederatedType(
computation_types.SequenceType(
collections.OrderedDict(
x=computation_types.TensorType(tf.float32, [None, 2]),
y=computation_types.TensorType(tf.float32, [None, 1]))),
placements.CLIENTS)
metrics_type = computation_types.FederatedType(
collections.OrderedDict(
broadcast=(),
aggregation=aggregate_metrics,
train=collections.OrderedDict(
loss=computation_types.TensorType(tf.float32),
num_examples=computation_types.TensorType(tf.int32)),
stat=collections.OrderedDict(
num_examples=computation_types.TensorType(tf.float32))),
placements.SERVER)
self.assert_types_equivalent(
computation_types.FunctionType(parameter=collections.OrderedDict(
server_state=server_state_type,
federated_dataset=dataset_type,
),
result=(server_state_type,
metrics_type)),
iterative_process.next.type_signature)
def server_init_tff():
"""Returns a state placed at `tff.SERVER`."""
tf_init_tuple = intrinsics.federated_eval(server_init_tf,
placements.SERVER)
return intrinsics.federated_zip(
optimizer_utils.ServerState(
model=tf_init_tuple[0],
optimizer_state=tf_init_tuple[1],
delta_aggregate_state=aggregation_init_computation(),
model_broadcast_state=broadcast_init_computation()))
def test_orchestration_type_signature(self):
iterative_process = optimizer_utils.build_model_delta_optimizer_process(
model_fn=model_examples.TrainableLinearRegression,
model_to_client_delta_fn=DummyClientDeltaFn,
server_optimizer_fn=lambda: gradient_descent.SGD(learning_rate=1.0
))
expected_model_weights_type = model_utils.ModelWeights(
collections.OrderedDict([('a', tff.TensorType(tf.float32, [2, 1])),
('b', tf.float32)]),
collections.OrderedDict([('c', tf.float32)]))
# ServerState consists of a model and optimizer_state. The optimizer_state
# is provided by TensorFlow, TFF doesn't care what the actual value is.
expected_federated_server_state_type = tff.FederatedType(
optimizer_utils.ServerState(expected_model_weights_type,
test.AnyType(), test.AnyType(),
test.AnyType()),
placement=tff.SERVER,
all_equal=True)
expected_federated_dataset_type = tff.FederatedType(tff.SequenceType(
model_examples.TrainableLinearRegression().input_spec),
tff.CLIENTS,
all_equal=False)
expected_model_output_types = tff.FederatedType(
collections.OrderedDict([
('loss', tff.TensorType(tf.float32, [])),
('num_examples', tff.TensorType(tf.int32, [])),
]),
tff.SERVER,
all_equal=True)
# `initialize` is expected to be a funcion of no arguments to a ServerState.
self.assertEqual(
tff.FunctionType(parameter=None,
result=expected_federated_server_state_type),
iterative_process.initialize.type_signature)
# `next` is expected be a function of (ServerState, Datasets) to
# ServerState.
self.assertEqual(
tff.FunctionType(parameter=[
expected_federated_server_state_type,
expected_federated_dataset_type
],
result=(expected_federated_server_state_type,
expected_model_output_types)),
iterative_process.next.type_signature)
def test_orchestration_typecheck(self):
iterative_process = federated_sgd.build_federated_sgd_process(
model_fn=model_examples.LinearRegression)
expected_model_weights_type = model_utils.ModelWeights(
collections.OrderedDict([('a', tff.TensorType(tf.float32, [2, 1])),
('b', tf.float32)]),
collections.OrderedDict([('c', tf.float32)]))
# ServerState consists of a model and optimizer_state. The optimizer_state
# is provided by TensorFlow, TFF doesn't care what the actual value is.
expected_federated_server_state_type = tff.FederatedType(
optimizer_utils.ServerState(expected_model_weights_type,
test.AnyType()),
placement=tff.SERVER,
all_equal=True)
expected_federated_dataset_type = tff.FederatedType(
tff.SequenceType(
model_examples.LinearRegression.make_batch(
tff.TensorType(tf.float32, [None, 2]),
tff.TensorType(tf.float32, [None, 1]))),
tff.CLIENTS,
all_equal=False)
expected_model_output_types = tff.FederatedType(
collections.OrderedDict([
('loss', tff.TensorType(tf.float32, [])),
('num_examples', tff.TensorType(tf.int32, [])),
]),
tff.SERVER,
all_equal=True)
# `initialize` is expected to be a funcion of no arguments to a ServerState.
self.assertEqual(
tff.FunctionType(
parameter=None, result=expected_federated_server_state_type),
iterative_process.initialize.type_signature)
# `next` is expected be a function of (ServerState, Datasets) to
# ServerState.
self.assertEqual(
tff.FunctionType(
parameter=[
expected_federated_server_state_type,
expected_federated_dataset_type
],
result=(expected_federated_server_state_type,
expected_model_output_types)),
iterative_process.next.type_signature)
def test_construction(self):
iterative_process = optimizer_utils.build_model_delta_optimizer_process(
model_fn=model_examples.LinearRegression,
model_to_client_delta_fn=DummyClientDeltaFn,
server_optimizer_fn=tf.keras.optimizers.SGD)
server_state_type = computation_types.FederatedType(
optimizer_utils.ServerState(
model=model_utils.ModelWeights(
trainable=[
computation_types.TensorType(tf.float32, [2, 1]),
computation_types.TensorType(tf.float32)
],
non_trainable=[computation_types.TensorType(tf.float32)]),
optimizer_state=[tf.int64],
delta_aggregate_state=(),
model_broadcast_state=()), placements.SERVER)
self.assertEqual(
str(iterative_process.initialize.type_signature),
str(
computation_types.FunctionType(
parameter=None, result=server_state_type)))
dataset_type = computation_types.FederatedType(
computation_types.SequenceType(
collections.OrderedDict(
x=computation_types.TensorType(tf.float32, [None, 2]),
y=computation_types.TensorType(tf.float32, [None, 1]))),
placements.CLIENTS)
metrics_type = computation_types.FederatedType(
collections.OrderedDict(
broadcast=(),
aggregation=(),
train=collections.OrderedDict(
loss=computation_types.TensorType(tf.float32),
num_examples=computation_types.TensorType(tf.int32))),
placements.SERVER)
self.assertEqual(
str(iterative_process.next.type_signature),
str(
computation_types.FunctionType(
parameter=collections.OrderedDict(
server_state=server_state_type,
federated_dataset=dataset_type,
),
result=(server_state_type, metrics_type))))
def test_state_with_new_model_weights(self):
trainable = [np.array([1.0, 2.0]), np.array([[1.0]])]
non_trainable = [np.array(1)]
state = anonymous_tuple.from_container(
optimizer_utils.ServerState(
model=model_utils.ModelWeights(
trainable=trainable, non_trainable=non_trainable),
optimizer_state=[],
delta_aggregate_state=tf.constant(0),
model_broadcast_state=tf.constant(0)),
recursive=True)
new_state = optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0]),
np.array([[3.0]])],
non_trainable_weights=[np.array(3)])
self.assertAllClose(
new_state.model.trainable,
[np.array([3.0, 3.0]), np.array([[3.0]])])
self.assertAllClose(new_state.model.non_trainable, [3])
with self.assertRaisesRegex(TypeError, 'tensor type'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0]),
np.array([[3]])],
non_trainable_weights=[np.array(3.0)])
with self.assertRaisesRegex(TypeError, 'tensor type'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0]),
np.array([3.0])],
non_trainable_weights=[np.array(3)])
with self.assertRaisesRegex(TypeError, 'different lengths'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0])],
non_trainable_weights=[np.array(3)])
with self.assertRaisesRegex(TypeError, 'cannot be handled'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights={'a': np.array([3.0, 3.0])},
non_trainable_weights=[np.array(3)])
def test_state_with_new_model_weights(self):
trainable = [('b', np.array([1.0, 2.0])), ('a', np.array([[1.0]]))]
non_trainable = [('c', np.array(1))]
state = anonymous_tuple.from_container(
optimizer_utils.ServerState(
model=model_utils.ModelWeights(
trainable=collections.OrderedDict(trainable),
non_trainable=collections.OrderedDict(non_trainable)),
optimizer_state=[],
delta_aggregate_state=tf.constant(0),
model_broadcast_state=tf.constant(0)),
recursive=True)
new_state = optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0]),
np.array([[3.0]])],
non_trainable_weights=[np.array(3)])
self.assertEqual(list(new_state.model.trainable.keys()), ['b', 'a'])
self.assertEqual(list(new_state.model.non_trainable.keys()), ['c'])
self.assertAllClose(new_state.model.trainable['b'], [3.0, 3.0])
self.assertAllClose(new_state.model.trainable['a'], [[3.0]])
self.assertAllClose(new_state.model.non_trainable['c'], 3)
with self.assertRaisesRegexp(ValueError, 'dtype'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0]),
np.array([[3]])],
non_trainable_weights=[np.array(3.0)])
with self.assertRaisesRegexp(ValueError, 'shape'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0]),
np.array([3.0])],
non_trainable_weights=[np.array(3)])
with self.assertRaisesRegexp(ValueError, 'Lengths differ'):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=[np.array([3.0, 3.0])],
non_trainable_weights=[np.array(3)])
def test_state_with_new_model_weights_failure(self, new_trainable,
new_non_trainable,
expected_err_msg):
trainable = [np.array([1.0, 2.0]), np.array([[1.0]]), np.int64(3)]
non_trainable = [np.array(1), b'bytes type', 5, 2.0]
state = optimizer_utils.ServerState(
model=model_utils.ModelWeights(trainable=trainable,
non_trainable=non_trainable),
optimizer_state=[],
delta_aggregate_state=tf.constant(0),
model_broadcast_state=tf.constant(0))
new_trainable = trainable if new_trainable is None else new_trainable
non_trainable = non_trainable if new_non_trainable is None else non_trainable
with self.assertRaisesRegex(TypeError, expected_err_msg):
optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=new_trainable,
non_trainable_weights=new_non_trainable)
def test_state_with_model_weights_success(self):
trainable = [np.array([1.0, 2.0]), np.array([[1.0]]), np.int64(3)]
non_trainable = [np.array(1), b'bytes type', 5, 2.0]
new_trainable = [np.array([3.0, 3.0]), np.array([[3.0]]), np.int64(4)]
new_non_trainable = [np.array(3), b'bytes check', 6, 3.0]
state = optimizer_utils.ServerState(
model=model_utils.ModelWeights(trainable=trainable,
non_trainable=non_trainable),
optimizer_state=[],
delta_aggregate_state=tf.constant(0),
model_broadcast_state=tf.constant(0))
new_state = optimizer_utils.state_with_new_model_weights(
state,
trainable_weights=new_trainable,
non_trainable_weights=new_non_trainable)
self.assertAllClose(new_state.model.trainable, new_trainable)
self.assertEqual(new_state.model.non_trainable, new_non_trainable)