def test_process_type_signature(self, value_template):
query = tensorflow_privacy.GaussianSumQuery(4.0, 0.0)
value_type = type_conversions.type_from_tensors(value_template)
dp_aggregate_process = differential_privacy.build_dp_aggregate_process(
value_type, query)
global_state = query.initial_global_state()
server_state_type = computation_types.FederatedType(
type_conversions.type_from_tensors(global_state), placements.SERVER)
self.assertEqual(
dp_aggregate_process.initialize.type_signature,
computation_types.FunctionType(
parameter=None, result=server_state_type))
metrics_type = type_conversions.type_from_tensors(
query.derive_metrics(global_state))
client_value_type = computation_types.FederatedType(value_type,
placements.CLIENTS)
client_value_weight_type = computation_types.FederatedType(
tf.float32, placements.CLIENTS)
server_result_type = computation_types.FederatedType(
value_type, placements.SERVER)
server_metrics_type = computation_types.FederatedType(
metrics_type, placements.SERVER)
self.assertEqual(
dp_aggregate_process.next.type_signature,
computation_types.FunctionType(
parameter=(server_state_type, client_value_type,
client_value_weight_type),
result=collections.OrderedDict(
state=server_state_type,
result=server_result_type,
measurements=server_metrics_type)))
def test_with_non_convert_tensors(self):
v1 = tf.Variable(0, name='foo', dtype=tf.int32, shape=[])
v2 = {'bar'}
d = collections.OrderedDict([('v1', v1), ('v2', v2)])
# TODO(b/122081673): Change Exception back to ValueError once TFF moves to
# be TF 2.0 only
with self.assertRaisesRegex(Exception, 'supported type'):
type_conversions.type_from_tensors(d)
def test_central_aggregation_with_secure_sum(self, value_shape, arity,
l1_bound):
value_type = computation_types.to_type((tf.float32, (value_shape,)))
factory_ = hihi_factory.create_central_hierarchical_histogram_factory(
arity=arity, secure_sum=True)
self.assertIsInstance(factory_,
differential_privacy.DifferentiallyPrivateFactory)
process = factory_.create(value_type)
self.assertIsInstance(process, aggregation_process.AggregationProcess)
query_state = _test_central_dp_query.initial_global_state()
query_state_type = type_conversions.type_from_tensors(query_state)
query_metrics_type = type_conversions.type_from_tensors(
_test_central_dp_query.derive_metrics(query_state))
server_state_type = computation_types.at_server((query_state_type, ()))
expected_initialize_type = computation_types.FunctionType(
parameter=None, result=server_state_type)
self.assertTrue(
process.initialize.type_signature.is_equivalent_to(
expected_initialize_type))
secure_dp_type = collections.OrderedDict(
secure_upper_clipped_count=tf.int32,
secure_lower_clipped_count=tf.int32,
secure_upper_threshold=tf.float32,
secure_lower_threshold=tf.float32)
expected_measurements_type = computation_types.at_server(
collections.OrderedDict(
dp_query_metrics=query_metrics_type, dp=secure_dp_type))
result_value_type = computation_types.to_type(
collections.OrderedDict([
('flat_values',
computation_types.TensorType(tf.float32, tf.TensorShape(None))),
('nested_row_splits', [(tf.int64, (None,))])
]))
expected_next_type = computation_types.FunctionType(
parameter=collections.OrderedDict(
state=server_state_type,
value=computation_types.at_clients(value_type)),
result=measured_process.MeasuredProcessOutput(
state=server_state_type,
result=computation_types.at_server(result_value_type),
measurements=expected_measurements_type))
self.assertTrue(
process.next.type_signature.is_equivalent_to(expected_next_type))
def test_keras_metric_finalizer_succeeds_with_different_metric_variables(
self, metric, unfinalized_metric_values, expected_result):
finalizer_computation = wrap_tf_function_in_tff_tf_computation(
metric,
type_conversions.type_from_tensors(unfinalized_metric_values))
finalized_metric = finalizer_computation(unfinalized_metric_values)
self.assertEqual(finalized_metric, expected_result)
def test_dp_sum_structure_nested_odict(self):
query = tensorflow_privacy.GaussianSumQuery(5.0, 0.0)
def datapoint(a, b, c):
return collections.OrderedDict([('a', (a,)),
('bc',
collections.OrderedDict([('b', [b]),
('c', (c,))]))])
data = [
datapoint(1.0, 2.0, 1.0),
datapoint(2.0, 3.0, 1.0),
datapoint(6.0, 8.0, 0.0), # Clipped to 3.0, 4.0, 0.0
]
value_type = type_conversions.type_from_tensors(data[0])
dp_aggregate_process = differential_privacy.build_dp_aggregate_process(
value_type, query)
global_state = dp_aggregate_process.initialize()
output = dp_aggregate_process.next(global_state, data, [1.0, 1.0, 1.0])
self.assertEqual(output['state']['l2_norm_clip'], 5.0)
self.assertEqual(output['state']['stddev'], 0.0)
self.assertEqual(output['result']['a'][0], 6.0)
self.assertEqual(output['result']['bc']['b'][0], 9.0)
self.assertEqual(output['result']['bc']['c'][0], 2.0)
def test_process_type_signature(self, value_template):
query = tensorflow_privacy.GaussianSumQuery(4.0, 0.0)
value_type = type_conversions.type_from_tensors(value_template)
dp_aggregate_process = differential_privacy.build_dp_aggregate_process(
value_type, query)
server_state_type = computation_types.FederatedType(
computation_types.NamedTupleType([('l2_norm_clip', tf.float32),
('stddev', tf.float32)]),
placements.SERVER)
self.assertEqual(
dp_aggregate_process.initialize.type_signature,
computation_types.FunctionType(
parameter=None, result=server_state_type))
client_value_type = computation_types.FederatedType(value_type,
placements.CLIENTS)
client_value_weight_type = computation_types.FederatedType(
tf.float32, placements.CLIENTS)
server_result_type = computation_types.FederatedType(
value_type, placements.SERVER)
server_metrics_type = computation_types.FederatedType((), placements.SERVER)
self.assertEqual(
dp_aggregate_process.next.type_signature,
computation_types.FunctionType(
parameter=computation_types.NamedTupleType([
(None, server_state_type), (None, client_value_type),
(None, client_value_weight_type)
]),
result=computation_types.NamedTupleType([
('state', server_state_type), ('result', server_result_type),
('measurements', server_metrics_type)
])))
def test_tff_model_from_functional_federated_aggregate_metrics_succeeds(self):
dataset = create_test_dataset()
input_spec = dataset.element_spec
functional_model = functional.FunctionalModel(initial_weights(),
forward_pass,
predict_on_batch, input_spec)
metric_constructors = [
lambda: tf.keras.metrics.MeanSquaredError(name='mse'),
lambda: tf.keras.metrics.MeanAbsoluteError(name='mae')
]
tff_model = functional.model_from_functional(functional_model,
metric_constructors)
client_1_local_outputs = collections.OrderedDict(
loss=[1.0, 2.0], mse=[1.0, 2.0], mae=[1.0, 2.0])
client_2_local_outputs = collections.OrderedDict(
loss=[2.0, 4.0], mse=[2.0, 2.0], mae=[1.0, 6.0])
metrics_aggregator = aggregator.sum_then_finalize
unfinalized_metrics_type = type_conversions.type_from_tensors(
tff_model.report_local_unfinalized_metrics())
metrics_aggregation_computation = metrics_aggregator(
tff_model.metric_finalizers(), unfinalized_metrics_type)
aggregated_metrics = metrics_aggregation_computation(
[client_1_local_outputs, client_2_local_outputs])
self.assertAllClose(
aggregated_metrics,
# loss = (1.0+2.0)/(2.0+4.0) = 0.5
# mse = (1.0+2.0)/(2.0+2.0) = 0.75
# mae = (1.0+1.0)/(2.0+6.0) = 0.25
collections.OrderedDict(loss=0.5, mse=0.75, mae=0.25))
def test_sum_then_finalize_metrics_with_initial_values(self):
aggregate_factory = aggregation_factory.SumThenFinalizeFactory()
metric_finalizers = collections.OrderedDict(
num_examples=tf.function(func=lambda x: x),
loss=tf.function(func=lambda x: tf.math.divide_no_nan(x[0], x[1])))
local_unfinalized_metrics = collections.OrderedDict(num_examples=1.0,
loss=[2.0, 1.0])
local_unfinalized_metrics_type = type_conversions.type_from_tensors(
local_unfinalized_metrics)
initial_unfinalized_metrics = collections.OrderedDict(num_examples=2.0,
loss=[3.0, 2.0])
process = aggregate_factory.create(metric_finalizers,
local_unfinalized_metrics_type,
initial_unfinalized_metrics)
state = process.initialize()
_, unfinalized_metrics_accumulators = state
self.assertEqual(unfinalized_metrics_accumulators,
initial_unfinalized_metrics)
client_data = [local_unfinalized_metrics, local_unfinalized_metrics]
output = process.next(state, client_data)
_, unfinalized_metrics_accumulators = output.state
current_round_metrics, total_rounds_metrics = output.result
self.assertEqual(
unfinalized_metrics_accumulators,
collections.OrderedDict(num_examples=4.0, loss=[7.0, 4.0]))
self.assertEqual(current_round_metrics,
collections.OrderedDict(num_examples=2.0, loss=2.0))
self.assertEqual(total_rounds_metrics,
collections.OrderedDict(num_examples=4.0, loss=1.75))
def test_type_properties(self, metric_finalizers, unfinalized_metrics):
aggregate_factory = aggregation_factory.SumThenFinalizeFactory()
self.assertIsInstance(aggregate_factory,
factory.UnweightedAggregationFactory)
local_unfinalized_metrics_type = type_conversions.type_from_tensors(
unfinalized_metrics)
process = aggregate_factory.create(metric_finalizers,
local_unfinalized_metrics_type)
self.assertIsInstance(process, aggregation_process.AggregationProcess)
expected_state_type = computation_types.FederatedType(
((), local_unfinalized_metrics_type), placements.SERVER)
expected_initialize_type = computation_types.FunctionType(
parameter=None, result=expected_state_type)
self.assertTrue(
process.initialize.type_signature.is_equivalent_to(
expected_initialize_type))
finalized_metrics_type = _get_finalized_metrics_type(
metric_finalizers, unfinalized_metrics)
result_value_type = computation_types.FederatedType(
(finalized_metrics_type, finalized_metrics_type),
placements.SERVER)
measurements_type = computation_types.FederatedType((),
placements.SERVER)
expected_next_type = computation_types.FunctionType(
parameter=collections.OrderedDict(
state=expected_state_type,
unfinalized_metrics=computation_types.FederatedType(
local_unfinalized_metrics_type, placements.CLIENTS)),
result=measured_process.MeasuredProcessOutput(
expected_state_type, result_value_type, measurements_type))
self.assertTrue(
process.next.type_signature.is_equivalent_to(expected_next_type))
def test_dp_stateful_mean(self):
class ShrinkingSumQuery(tensorflow_privacy.GaussianSumQuery):
def get_noised_result(self, sample_state, global_state):
global_state = self._GlobalState(
tf.maximum(global_state.l2_norm_clip - 1, 0.0),
global_state.stddev)
return sample_state, global_state
query = ShrinkingSumQuery(4.0, 0.0)
value_type = type_conversions.type_from_tensors(0.0)
dp_aggregate_process = differential_privacy.build_dp_aggregate_process(
value_type, query)
global_state = dp_aggregate_process.initialize()
records = [1.0, 3.0, 5.0]
def run_and_check(global_state, expected_l2_norm_clip,
expected_result):
output = dp_aggregate_process.next(global_state, records,
[1.0, 1.0, 1.0])
self.assertEqual(output.state.l2_norm_clip, expected_l2_norm_clip)
self.assertEqual(output.result, expected_result)
return output.state
self.assertEqual(global_state.l2_norm_clip, 4.0)
global_state = run_and_check(global_state, 3.0, 8.0)
global_state = run_and_check(global_state, 2.0, 7.0)
global_state = run_and_check(global_state, 1.0, 5.0)
global_state = run_and_check(global_state, 0.0, 3.0)
global_state = run_and_check(global_state, 0.0, 0.0)
def test_dp_sum_structure_list(self):
query = tensorflow_privacy.GaussianSumQuery(5.0, 0.0)
def datapoint(a, b):
return [tf.Variable(a, name='a'), tf.Variable(b, name='b')]
data = [
datapoint(1.0, 2.0),
datapoint(2.0, 3.0),
datapoint(6.0, 8.0), # Clipped to 3.0, 4.0
]
value_type = type_conversions.type_from_tensors(data[0])
dp_aggregate_process = differential_privacy.build_dp_aggregate_process(
value_type, query)
global_state = dp_aggregate_process.initialize()
output = dp_aggregate_process.next(global_state, data, [1.0, 1.0, 1.0])
self.assertEqual(output.state.l2_norm_clip, 5.0)
self.assertEqual(output.state.stddev, 0.0)
result = list(output.result)
self.assertEqual(result[0], 6.0)
self.assertEqual(result[1], 9.0)
def test_adaptation_up(self, shapes):
value_type = type_conversions.type_from_tensors(_make_value(0, shapes))
mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process(
value_type=value_type,
initial_quantile_estimate=1.0,
target_quantile=1.0,
multiplier=1.0,
increment=1.0,
learning_rate=np.log(2.0),
norm_order=np.inf)
global_state = mean_process.initialize()
values = [_make_value(x, shapes) for x in [90, 91, 92]]
output = mean_process.next(global_state, values, [1, 1, 1])
self._check_result(output, 0, shapes)
global_state = output.state
metrics = output.measurements
self.assertAllClose(metrics.zeroing_threshold, 3.0)
self.assertEqual(metrics.num_zeroed, 3)
output = mean_process.next(global_state, values, [1, 1, 1])
self._check_result(output, 0, shapes)
global_state = output.state
metrics = output.measurements
self.assertAllClose(metrics.zeroing_threshold, 5.0)
self.assertEqual(metrics.num_zeroed, 3)
def build_encoded_sum_process_from_model(
model_fn: _ModelConstructor,
encoder_fn: _EncoderConstructor) -> measured_process.MeasuredProcess:
"""Builds `MeasuredProcess` for weights of model returned by `model_fn`.
This method creates a `GatherEncoder` for every trainable weight of model
created by `model_fn`, as returned by `encoder_fn`.
Args:
model_fn: A Python callable with no arguments function that returns a
`tff.learning.Model`.
encoder_fn: A Python callable with a single argument, which is expected to
be a `tf.Tensor` of shape and dtype to be encoded. The function must
return a `tensor_encoding.core.SimpleEncoder`, which expects a `tf.Tensor`
with compatible type as the input to its `encode` method.
Returns:
A `MeasuredProcess` for encoding and summing the weights of model created by
`model_fn`.
Raises:
TypeError: If `model_fn` or `encoder_fn` are not callable objects.
"""
py_typecheck.check_callable(model_fn)
py_typecheck.check_callable(encoder_fn)
trainable_weights = _weights_from_model_fn(model_fn).trainable
encoders = tf.nest.map_structure(encoder_fn, trainable_weights)
weight_type = type_conversions.type_from_tensors(trainable_weights)
return encoding_utils.build_encoded_sum_process(weight_type, encoders)
def test_adaptation_achieved_with_multiplier(self, shapes):
value_type = type_conversions.type_from_tensors(_make_value(0, shapes))
mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process(
value_type=value_type,
initial_quantile_estimate=100.0,
target_quantile=0.5,
multiplier=2.0,
increment=10.0,
learning_rate=np.log(4.0),
norm_order=np.inf)
global_state = mean_process.initialize()
values = [_make_value(x, shapes) for x in [30, 60]]
# With target 0.5, learning rate λ=ln(4), estimate should be cut in
# half in first round: exp(ln(4)(-0.5)) = 1/sqrt(4) = 0.5.
output = mean_process.next(global_state, values, [1, 1])
self._check_result(output, 45, shapes)
global_state = output.state
metrics = output.measurements
self.assertAllClose(metrics.zeroing_threshold, 110.0)
self.assertEqual(metrics.num_zeroed, 0)
# In second round, target is achieved, no adaptation occurs, and no updates
# are zeroed becaues of multiplier.
output = mean_process.next(global_state, values, [1, 1])
self._check_result(output, 45, shapes)
global_state = output.state
metrics = output.measurements
self.assertAllClose(metrics.zeroing_threshold, 110.0)
self.assertEqual(metrics.num_zeroed, 0)
def test_dp_sum_structure_complex(self):
query = tensorflow_privacy.GaussianSumQuery(5.0, 0.0)
def datapoint(a, b, c):
return collections.OrderedDict(a=(a, ), bc=([b], (c, )))
data = [
datapoint(1.0, 2.0, 1.0),
datapoint(2.0, 3.0, 1.0),
datapoint(6.0, 8.0, 0.0), # Clipped to 3.0, 4.0, 0.0
]
value_type = type_conversions.type_from_tensors(data[0])
dp_aggregate_process = differential_privacy.build_dp_aggregate_process(
value_type, query)
global_state = dp_aggregate_process.initialize()
output = dp_aggregate_process.next(global_state, data, [1.0, 1.0, 1.0])
self.assertEqual(output.state.l2_norm_clip, 5.0)
self.assertEqual(output.state.stddev, 0.0)
self.assertEqual(output.result['a'][0], 6.0)
self.assertEqual(output.result['bc'][0][0], 9.0)
self.assertEqual(output.result['bc'][1][0], 2.0)
def test_adaptation_achieved(self, shapes):
value_type = type_conversions.type_from_tensors(_make_value(0, shapes))
mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process(
value_type, 100.0, 0.5, 1.0, np.log(4.0), np.inf)
global_state = mean_process.initialize()
self.assertEqual(global_state.current_estimate, 100.0)
values = [_make_value(x, shapes) for x in [30, 60]]
# With target 0.5, learning rate λ=ln(4), estimate should be cut in
# half in first round: exp(ln(4)(-0.5)) = 1/sqrt(4) = 0.5.
output = mean_process.next(global_state, values, [1, 1])
self._check_result(output, 45, shapes)
global_state = output['state']
self.assertAllClose(global_state.current_estimate, 50.0)
metrics = output['measurements']
self.assertAllClose(metrics.current_threshold, 50.0)
self.assertEqual(metrics.num_zeroed, 0)
# In second round, target is achieved, no adaptation occurs, but one update
# is zeroed.
output = mean_process.next(global_state, values, [1, 1])
self._check_result(output, 30, shapes)
global_state = output['state']
self.assertAllClose(global_state.current_estimate, 50.0)
metrics = output['measurements']
self.assertAllClose(metrics.current_threshold, 50.0)
self.assertEqual(metrics.num_zeroed, 1)
def build_encoded_mean(values, encoders):
"""Builds `StatefulAggregateFn` for `values`, to be encoded by `encoders`.
Args:
values: Values to be encoded by the `StatefulAggregateFn`. Must be
convertible to `tff.Value`.
encoders: A collection of `GatherEncoder` objects to be used for encoding
`values`. Must have the same structure as `values`.
Returns:
A `StatefulAggregateFn` of which `next_fn` encodes the input at
`tff.CLIENTS`, and computes their mean at `tff.SERVER`, automatically
splitting the decoding part based on its commutativity with sum.
Raises:
ValueError: If `values` and `encoders` do not have the same structure.
TypeError: If `encoders` are not instances of `GatherEncoder`, or if
`values` are not compatible with the expected input of the `encoders`.
"""
warnings.warn(
'Deprecation warning: tff.utils.build_encoded_mean() is deprecated, use '
'tff.utils.build_encoded_mean_process() instead.', DeprecationWarning)
tf.nest.assert_same_structure(values, encoders)
tf.nest.map_structure(
lambda e, v: _validate_encoder(e, v, tensor_encoding.core.GatherEncoder
), encoders, values)
value_type = type_conversions.type_from_tensors(values)
initial_state_fn, state_type = _build_initial_state_tf_computation(
encoders)
nest_encoder = _build_tf_computations_for_gather(state_type, value_type,
encoders)
encoded_sum_fn = _build_encoded_sum_fn(nest_encoder)
@computations.tf_computation(value_type, tf.float32)
def multiply_fn(value, weight):
return tf.nest.map_structure(lambda v: v * tf.cast(weight, v.dtype),
value)
@computations.tf_computation(value_type, tf.float32)
def divide_fn(value, denominator):
return tf.nest.map_structure(
lambda v: v / tf.cast(denominator, v.dtype), value)
def encoded_mean_fn(state, values, weight):
weighted_values = intrinsics.federated_map(multiply_fn,
[values, weight])
updated_state, summed_decoded_values = encoded_sum_fn(
state, weighted_values)
summed_weights = intrinsics.federated_sum(weight)
decoded_values = intrinsics.federated_map(
divide_fn, [summed_decoded_values, summed_weights])
return updated_state, decoded_values
return computation_utils.StatefulAggregateFn(
initialize_fn=initial_state_fn, next_fn=encoded_mean_fn)
def test_incorrect_finalizers_type_raises(self, bad_finalizers):
local_unfinalized_metrics = collections.OrderedDict(num_examples=1.0)
local_unfinalized_metrics_type = type_conversions.type_from_tensors(
local_unfinalized_metrics)
aggregate_factory = aggregation_factory.SumThenFinalizeFactory()
with self.assertRaises(TypeError):
aggregate_factory.create(bad_finalizers,
local_unfinalized_metrics_type)
def test_process_type_signature(self, value_template):
value_type = type_conversions.type_from_tensors(value_template)
mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process(
value_type=value_type,
initial_quantile_estimate=100.0,
target_quantile=0.99,
multiplier=2.0,
increment=1.0,
learning_rate=1.0,
norm_order=np.inf)
dummy_quantile_query = tensorflow_privacy.NoPrivacyQuantileEstimatorQuery(
50.0, 0.99, 1.0, True)
quantile_query_state = dummy_quantile_query.initial_global_state()
server_state_type = computation_types.FederatedType(
type_conversions.type_from_tensors(quantile_query_state),
placements.SERVER)
self.assertEqual(
mean_process.initialize.type_signature,
computation_types.FunctionType(parameter=None,
result=server_state_type))
client_value_type = computation_types.FederatedType(
value_type, placements.CLIENTS)
client_value_weight_type = computation_types.FederatedType(
tf.float32, placements.CLIENTS)
server_result_type = computation_types.FederatedType(
value_type, placements.SERVER)
server_metrics_type = computation_types.FederatedType(
adaptive_zeroing.AdaptiveZeroingMetrics(
zeroing_threshold=tf.float32, num_zeroed=tf.int32),
placements.SERVER)
self.assertTrue(
mean_process.next.type_signature.is_equivalent_to(
computation_types.FunctionType(
parameter=collections.OrderedDict(
global_state=server_state_type,
value=client_value_type,
weight=client_value_weight_type),
result=collections.OrderedDict(
state=server_state_type,
result=server_result_type,
measurements=server_metrics_type,
))))
def test_evaluation_computation_custom_stateless_broadcaster(
self, model_fn):
def loss_fn():
return tf.keras.losses.MeanSquaredError()
def metrics_fn():
return [counters.NumExamplesCounter(), NumOverCounter(5.0)]
model_weights_type = type_conversions.type_from_tensors(
reconstruction_utils.get_global_variables(model_fn()))
def build_custom_stateless_broadcaster(
model_weights_type) -> measured_process_lib.MeasuredProcess:
"""Builds a `MeasuredProcess` that wraps `tff.federated_broadcast`."""
@federated_computation.federated_computation()
def test_server_initialization():
return intrinsics.federated_value((), placements.SERVER)
@federated_computation.federated_computation(
computation_types.FederatedType((), placements.SERVER),
computation_types.FederatedType(model_weights_type,
placements.SERVER),
)
def stateless_broadcast(state, value):
test_metrics = intrinsics.federated_value(
3.0, placements.SERVER)
return measured_process_lib.MeasuredProcessOutput(
state=state,
result=intrinsics.federated_broadcast(value),
measurements=test_metrics)
return measured_process_lib.MeasuredProcess(
initialize_fn=test_server_initialization,
next_fn=stateless_broadcast)
evaluate = evaluation_computation.build_federated_evaluation(
model_fn,
loss_fn=loss_fn,
metrics_fn=metrics_fn,
reconstruction_optimizer_fn=lambda: tf.keras.optimizers.SGD(0.1),
broadcast_process=build_custom_stateless_broadcaster(
model_weights_type=model_weights_type))
self.assertEqual(
str(evaluate.type_signature),
'(<server_model_weights=<trainable=<float32[1,1]>,'
'non_trainable=<>>@SERVER,federated_dataset={<x=float32[?,1],'
'y=float32[?,1]>*}@CLIENTS> -> <broadcast=float32,eval='
'<loss=float32,num_examples=int64,num_over=float32>>@SERVER)')
result = evaluate(
collections.OrderedDict([
('trainable', [[[1.0]]]),
('non_trainable', []),
]), create_client_data())
self.assertEqual(result['broadcast'], 3.0)
def test_keras_metric_finalizer_returns_correct_result(self, metric):
# The unfinalized accuracy contains two tensors `total` and `count`.
unfinalized_accuracy = [tf.constant(2.0), tf.constant(2.0)]
finalizer_computation = wrap_tf_function_in_tff_tf_computation(
metric, type_conversions.type_from_tensors(unfinalized_accuracy))
finalized_accuracy = finalizer_computation(unfinalized_accuracy)
self.assertEqual(
# The expected value is computed by dividing `total` by `count`.
finalized_accuracy,
unfinalized_accuracy[0] / unfinalized_accuracy[1])
def test_returns_correct_results(self, metric_finalizers,
local_unfinalized_metrics_at_clients,
expected_aggregated_metrics):
aggregator_computation = aggregator.sum_then_finalize(
metric_finalizers=metric_finalizers,
local_unfinalized_metrics_type=type_conversions.type_from_tensors(
local_unfinalized_metrics_at_clients[0]))
aggregated_metrics = aggregator_computation(
local_unfinalized_metrics_at_clients)
self.assertAllEqual(aggregated_metrics, expected_aggregated_metrics)
def test_raises_with_empty_value(self):
value_type = type_conversions.type_from_tensors(())
with self.assertRaises(ValueError):
adaptive_zeroing.build_adaptive_zeroing_mean_process(
value_type=value_type,
initial_quantile_estimate=100.0,
target_quantile=0.99,
multiplier=2.0,
increment=1.0,
learning_rate=1.0,
norm_order=np.inf)
def test_keras_metric_finalizer_fails_with_unmatched_unfinalized_metric_values(
self, invalid_unfinalized_metric_values, error_type,
error_message):
# The expected unfinalized metric values for `SparseCategoricalAccuracy` is
# a list of two `tf.Tensor`s and each has shape () and dtype tf.float32.
metric = tf.keras.metrics.SparseCategoricalAccuracy()
with self.assertRaisesRegex(error_type, error_message):
wrap_tf_function_in_tff_tf_computation(
metric,
type_conversions.type_from_tensors(
invalid_unfinalized_metric_values))
def test_raises_on_invalid_distributor(self):
model_weights_type = type_conversions.type_from_tensors(
model_utils.ModelWeights.from_model(
model_examples.LinearRegression()))
distributor = distributors.build_broadcast_process(model_weights_type)
invalid_distributor = iterative_process.IterativeProcess(
distributor.initialize, distributor.next)
with self.assertRaises(TypeError):
fed_avg.build_weighted_fed_avg(
model_fn=model_examples.LinearRegression,
client_optimizer_fn=sgdm.build_sgdm(1.0),
model_distributor=invalid_distributor)
def test_type_properties(self, value_type, inner_agg_factory):
agg_factory = dp_factory.DifferentiallyPrivateFactory(
_test_dp_query, inner_agg_factory)
self.assertIsInstance(agg_factory,
factory.UnweightedAggregationFactory)
value_type = computation_types.to_type(value_type)
process = agg_factory.create_unweighted(value_type)
self.assertIsInstance(process, aggregation_process.AggregationProcess)
query_state = _test_dp_query.initial_global_state()
query_state_type = type_conversions.type_from_tensors(query_state)
query_metrics_type = type_conversions.type_from_tensors(
_test_dp_query.derive_metrics(query_state))
inner_state_type = tf.int32 if inner_agg_factory else ()
server_state_type = computation_types.at_server(
(query_state_type, inner_state_type))
expected_initialize_type = computation_types.FunctionType(
parameter=None, result=server_state_type)
self.assertTrue(
process.initialize.type_signature.is_equivalent_to(
expected_initialize_type))
inner_measurements_type = tf.int32 if inner_agg_factory else ()
expected_measurements_type = computation_types.at_server(
collections.OrderedDict(
query_metrics=query_metrics_type,
record_agg_process=inner_measurements_type))
expected_next_type = computation_types.FunctionType(
parameter=collections.OrderedDict(
state=server_state_type,
value=computation_types.at_clients(value_type)),
result=measured_process.MeasuredProcessOutput(
state=server_state_type,
result=computation_types.at_server(value_type),
measurements=expected_measurements_type))
self.assertTrue(
process.next.type_signature.is_equivalent_to(expected_next_type))
def test_unfinalized_metrics_type_and_initial_values_mismatch_raises(self):
aggregate_factory = aggregation_factory.SumThenFinalizeFactory()
metric_finalizers = collections.OrderedDict(num_examples=tf.function(
func=lambda x: x))
local_unfinalized_metrics_type = type_conversions.type_from_tensors(
collections.OrderedDict(num_examples=1.0))
initial_unfinalized_metrics = collections.OrderedDict(
num_examples=[1.0])
with self.assertRaisesRegex(TypeError,
'initial unfinalized metrics type'):
aggregate_factory.create(metric_finalizers,
local_unfinalized_metrics_type,
initial_unfinalized_metrics)
def test_raises_on_invalid_distributor(self):
model_weights_type = type_conversions.type_from_tensors(
model_utils.ModelWeights.from_model(
model_examples.LinearRegression()))
distributor = distributors.build_broadcast_process(model_weights_type)
invalid_distributor = iterative_process.IterativeProcess(
distributor.initialize, distributor.next)
with self.assertRaises(TypeError):
mime.build_weighted_mime_lite(
model_fn=model_examples.LinearRegression,
base_optimizer=sgdm.build_sgdm(learning_rate=0.01,
momentum=0.9),
model_distributor=invalid_distributor)
def test_sum_then_finalize_metrics(self):
aggregate_factory = aggregation_factory.SumThenFinalizeFactory()
metric_finalizers = collections.OrderedDict(
num_examples=tf.function(func=lambda x: x),
loss=tf.function(func=lambda x: tf.math.divide_no_nan(x[0], x[1])),
custom_sum=tf.function(
func=lambda x: tf.add_n(map(tf.math.reduce_sum, x))))
local_unfinalized_metrics = collections.OrderedDict(
num_examples=1.0,
loss=[2.0, 1.0],
custom_sum=[tf.constant(1.0),
tf.constant([1.0, 1.0])])
local_unfinalized_metrics_type = type_conversions.type_from_tensors(
local_unfinalized_metrics)
process = aggregate_factory.create(metric_finalizers,
local_unfinalized_metrics_type)
state = process.initialize()
_, unfinalized_metrics_accumulators = state
expected_unfinalized_metrics_accumulators = collections.OrderedDict(
num_examples=0.0,
loss=[0.0, 0.0],
custom_sum=[tf.constant(0.0),
tf.constant([0.0, 0.0])])
tf.nest.map_structure(self.assertAllEqual,
unfinalized_metrics_accumulators,
expected_unfinalized_metrics_accumulators)
client_data = [local_unfinalized_metrics, local_unfinalized_metrics]
output = process.next(state, client_data)
_, unfinalized_metrics_accumulators = output.state
current_round_metrics, total_rounds_metrics = output.result
expected_unfinalized_metrics_accumulators = collections.OrderedDict(
num_examples=2.0,
loss=[4.0, 2.0],
custom_sum=[tf.constant(2.0),
tf.constant([2.0, 2.0])])
tf.nest.map_structure(self.assertAllEqual,
unfinalized_metrics_accumulators,
expected_unfinalized_metrics_accumulators)
self.assertEqual(
current_round_metrics,
collections.OrderedDict(num_examples=2.0,
loss=2.0,
custom_sum=tf.constant(6.0)))
self.assertEqual(
total_rounds_metrics,
collections.OrderedDict(num_examples=2.0,
loss=2.0,
custom_sum=tf.constant(6.0)))
def test_default_value_ranges_returns_correct_results(
self, metric_finalizers, local_unfinalized_metrics_at_clients,
expected_aggregated_metrics):
aggregator_computation = aggregator.secure_sum_then_finalize(
metric_finalizers=metric_finalizers,
local_unfinalized_metrics_type=type_conversions.type_from_tensors(
local_unfinalized_metrics_at_clients[0]))
try:
static_assert.assert_not_contains_unsecure_aggregation(
aggregator_computation)
except: # pylint: disable=bare-except
self.fail(
'Metric aggregation contains non-secure summation aggregation')
aggregated_metrics = aggregator_computation(
local_unfinalized_metrics_at_clients)
no_clipped_values = collections.OrderedDict(
secure_upper_clipped_count=0,
secure_lower_clipped_count=0,
secure_upper_threshold=aggregator.DEFAULT_SECURE_UPPER_BOUND,
secure_lower_threshold=aggregator.DEFAULT_SECURE_LOWER_BOUND)
factory_keys = collections.OrderedDict()
for value in tf.nest.flatten(local_unfinalized_metrics_at_clients[0]):
tensor = tf.constant(value)
if tensor.dtype.is_floating:
lower = float(aggregator.DEFAULT_SECURE_LOWER_BOUND)
upper = float(aggregator.DEFAULT_SECURE_UPPER_BOUND)
elif tensor.dtype.is_integer:
lower = int(aggregator.DEFAULT_SECURE_LOWER_BOUND)
upper = int(aggregator.DEFAULT_SECURE_UPPER_BOUND)
else:
raise TypeError(
f'Expected float or int, found tensors of dtype {tensor.dtype}.'
)
factory_key = aggregator._create_factory_key(
lower, upper, tensor.dtype)
factory_keys[factory_key] = 1
expected_measurements = collections.OrderedDict(
(factory_key, no_clipped_values) for factory_key in factory_keys)
secure_sum_measurements = aggregated_metrics.pop(
'secure_sum_measurements')
self.assertAllClose(secure_sum_measurements, expected_measurements)
self.assertAllClose(aggregated_metrics,
expected_aggregated_metrics,
rtol=1e-5,
atol=1e-5)
