def test_process_type_signature(self, value_template): query = tensorflow_privacy.GaussianSumQuery(4.0, 0.0) value_type = type_conversions.type_from_tensors(value_template) dp_aggregate_process = differential_privacy.build_dp_aggregate_process( value_type, query) global_state = query.initial_global_state() server_state_type = computation_types.FederatedType( type_conversions.type_from_tensors(global_state), placements.SERVER) self.assertEqual( dp_aggregate_process.initialize.type_signature, computation_types.FunctionType( parameter=None, result=server_state_type)) metrics_type = type_conversions.type_from_tensors( query.derive_metrics(global_state)) client_value_type = computation_types.FederatedType(value_type, placements.CLIENTS) client_value_weight_type = computation_types.FederatedType( tf.float32, placements.CLIENTS) server_result_type = computation_types.FederatedType( value_type, placements.SERVER) server_metrics_type = computation_types.FederatedType( metrics_type, placements.SERVER) self.assertEqual( dp_aggregate_process.next.type_signature, computation_types.FunctionType( parameter=(server_state_type, client_value_type, client_value_weight_type), result=collections.OrderedDict( state=server_state_type, result=server_result_type, measurements=server_metrics_type)))
def test_with_non_convert_tensors(self): v1 = tf.Variable(0, name='foo', dtype=tf.int32, shape=[]) v2 = {'bar'} d = collections.OrderedDict([('v1', v1), ('v2', v2)]) # TODO(b/122081673): Change Exception back to ValueError once TFF moves to # be TF 2.0 only with self.assertRaisesRegex(Exception, 'supported type'): type_conversions.type_from_tensors(d)
def test_central_aggregation_with_secure_sum(self, value_shape, arity, l1_bound): value_type = computation_types.to_type((tf.float32, (value_shape,))) factory_ = hihi_factory.create_central_hierarchical_histogram_factory( arity=arity, secure_sum=True) self.assertIsInstance(factory_, differential_privacy.DifferentiallyPrivateFactory) process = factory_.create(value_type) self.assertIsInstance(process, aggregation_process.AggregationProcess) query_state = _test_central_dp_query.initial_global_state() query_state_type = type_conversions.type_from_tensors(query_state) query_metrics_type = type_conversions.type_from_tensors( _test_central_dp_query.derive_metrics(query_state)) server_state_type = computation_types.at_server((query_state_type, ())) expected_initialize_type = computation_types.FunctionType( parameter=None, result=server_state_type) self.assertTrue( process.initialize.type_signature.is_equivalent_to( expected_initialize_type)) secure_dp_type = collections.OrderedDict( secure_upper_clipped_count=tf.int32, secure_lower_clipped_count=tf.int32, secure_upper_threshold=tf.float32, secure_lower_threshold=tf.float32) expected_measurements_type = computation_types.at_server( collections.OrderedDict( dp_query_metrics=query_metrics_type, dp=secure_dp_type)) result_value_type = computation_types.to_type( collections.OrderedDict([ ('flat_values', computation_types.TensorType(tf.float32, tf.TensorShape(None))), ('nested_row_splits', [(tf.int64, (None,))]) ])) expected_next_type = computation_types.FunctionType( parameter=collections.OrderedDict( state=server_state_type, value=computation_types.at_clients(value_type)), result=measured_process.MeasuredProcessOutput( state=server_state_type, result=computation_types.at_server(result_value_type), measurements=expected_measurements_type)) self.assertTrue( process.next.type_signature.is_equivalent_to(expected_next_type))
def test_keras_metric_finalizer_succeeds_with_different_metric_variables( self, metric, unfinalized_metric_values, expected_result): finalizer_computation = wrap_tf_function_in_tff_tf_computation( metric, type_conversions.type_from_tensors(unfinalized_metric_values)) finalized_metric = finalizer_computation(unfinalized_metric_values) self.assertEqual(finalized_metric, expected_result)
def test_dp_sum_structure_nested_odict(self): query = tensorflow_privacy.GaussianSumQuery(5.0, 0.0) def datapoint(a, b, c): return collections.OrderedDict([('a', (a,)), ('bc', collections.OrderedDict([('b', [b]), ('c', (c,))]))]) data = [ datapoint(1.0, 2.0, 1.0), datapoint(2.0, 3.0, 1.0), datapoint(6.0, 8.0, 0.0), # Clipped to 3.0, 4.0, 0.0 ] value_type = type_conversions.type_from_tensors(data[0]) dp_aggregate_process = differential_privacy.build_dp_aggregate_process( value_type, query) global_state = dp_aggregate_process.initialize() output = dp_aggregate_process.next(global_state, data, [1.0, 1.0, 1.0]) self.assertEqual(output['state']['l2_norm_clip'], 5.0) self.assertEqual(output['state']['stddev'], 0.0) self.assertEqual(output['result']['a'][0], 6.0) self.assertEqual(output['result']['bc']['b'][0], 9.0) self.assertEqual(output['result']['bc']['c'][0], 2.0)
def test_process_type_signature(self, value_template): query = tensorflow_privacy.GaussianSumQuery(4.0, 0.0) value_type = type_conversions.type_from_tensors(value_template) dp_aggregate_process = differential_privacy.build_dp_aggregate_process( value_type, query) server_state_type = computation_types.FederatedType( computation_types.NamedTupleType([('l2_norm_clip', tf.float32), ('stddev', tf.float32)]), placements.SERVER) self.assertEqual( dp_aggregate_process.initialize.type_signature, computation_types.FunctionType( parameter=None, result=server_state_type)) client_value_type = computation_types.FederatedType(value_type, placements.CLIENTS) client_value_weight_type = computation_types.FederatedType( tf.float32, placements.CLIENTS) server_result_type = computation_types.FederatedType( value_type, placements.SERVER) server_metrics_type = computation_types.FederatedType((), placements.SERVER) self.assertEqual( dp_aggregate_process.next.type_signature, computation_types.FunctionType( parameter=computation_types.NamedTupleType([ (None, server_state_type), (None, client_value_type), (None, client_value_weight_type) ]), result=computation_types.NamedTupleType([ ('state', server_state_type), ('result', server_result_type), ('measurements', server_metrics_type) ])))
def test_tff_model_from_functional_federated_aggregate_metrics_succeeds(self): dataset = create_test_dataset() input_spec = dataset.element_spec functional_model = functional.FunctionalModel(initial_weights(), forward_pass, predict_on_batch, input_spec) metric_constructors = [ lambda: tf.keras.metrics.MeanSquaredError(name='mse'), lambda: tf.keras.metrics.MeanAbsoluteError(name='mae') ] tff_model = functional.model_from_functional(functional_model, metric_constructors) client_1_local_outputs = collections.OrderedDict( loss=[1.0, 2.0], mse=[1.0, 2.0], mae=[1.0, 2.0]) client_2_local_outputs = collections.OrderedDict( loss=[2.0, 4.0], mse=[2.0, 2.0], mae=[1.0, 6.0]) metrics_aggregator = aggregator.sum_then_finalize unfinalized_metrics_type = type_conversions.type_from_tensors( tff_model.report_local_unfinalized_metrics()) metrics_aggregation_computation = metrics_aggregator( tff_model.metric_finalizers(), unfinalized_metrics_type) aggregated_metrics = metrics_aggregation_computation( [client_1_local_outputs, client_2_local_outputs]) self.assertAllClose( aggregated_metrics, # loss = (1.0+2.0)/(2.0+4.0) = 0.5 # mse = (1.0+2.0)/(2.0+2.0) = 0.75 # mae = (1.0+1.0)/(2.0+6.0) = 0.25 collections.OrderedDict(loss=0.5, mse=0.75, mae=0.25))
def test_sum_then_finalize_metrics_with_initial_values(self): aggregate_factory = aggregation_factory.SumThenFinalizeFactory() metric_finalizers = collections.OrderedDict( num_examples=tf.function(func=lambda x: x), loss=tf.function(func=lambda x: tf.math.divide_no_nan(x[0], x[1]))) local_unfinalized_metrics = collections.OrderedDict(num_examples=1.0, loss=[2.0, 1.0]) local_unfinalized_metrics_type = type_conversions.type_from_tensors( local_unfinalized_metrics) initial_unfinalized_metrics = collections.OrderedDict(num_examples=2.0, loss=[3.0, 2.0]) process = aggregate_factory.create(metric_finalizers, local_unfinalized_metrics_type, initial_unfinalized_metrics) state = process.initialize() _, unfinalized_metrics_accumulators = state self.assertEqual(unfinalized_metrics_accumulators, initial_unfinalized_metrics) client_data = [local_unfinalized_metrics, local_unfinalized_metrics] output = process.next(state, client_data) _, unfinalized_metrics_accumulators = output.state current_round_metrics, total_rounds_metrics = output.result self.assertEqual( unfinalized_metrics_accumulators, collections.OrderedDict(num_examples=4.0, loss=[7.0, 4.0])) self.assertEqual(current_round_metrics, collections.OrderedDict(num_examples=2.0, loss=2.0)) self.assertEqual(total_rounds_metrics, collections.OrderedDict(num_examples=4.0, loss=1.75))
def test_type_properties(self, metric_finalizers, unfinalized_metrics): aggregate_factory = aggregation_factory.SumThenFinalizeFactory() self.assertIsInstance(aggregate_factory, factory.UnweightedAggregationFactory) local_unfinalized_metrics_type = type_conversions.type_from_tensors( unfinalized_metrics) process = aggregate_factory.create(metric_finalizers, local_unfinalized_metrics_type) self.assertIsInstance(process, aggregation_process.AggregationProcess) expected_state_type = computation_types.FederatedType( ((), local_unfinalized_metrics_type), placements.SERVER) expected_initialize_type = computation_types.FunctionType( parameter=None, result=expected_state_type) self.assertTrue( process.initialize.type_signature.is_equivalent_to( expected_initialize_type)) finalized_metrics_type = _get_finalized_metrics_type( metric_finalizers, unfinalized_metrics) result_value_type = computation_types.FederatedType( (finalized_metrics_type, finalized_metrics_type), placements.SERVER) measurements_type = computation_types.FederatedType((), placements.SERVER) expected_next_type = computation_types.FunctionType( parameter=collections.OrderedDict( state=expected_state_type, unfinalized_metrics=computation_types.FederatedType( local_unfinalized_metrics_type, placements.CLIENTS)), result=measured_process.MeasuredProcessOutput( expected_state_type, result_value_type, measurements_type)) self.assertTrue( process.next.type_signature.is_equivalent_to(expected_next_type))
def test_dp_stateful_mean(self): class ShrinkingSumQuery(tensorflow_privacy.GaussianSumQuery): def get_noised_result(self, sample_state, global_state): global_state = self._GlobalState( tf.maximum(global_state.l2_norm_clip - 1, 0.0), global_state.stddev) return sample_state, global_state query = ShrinkingSumQuery(4.0, 0.0) value_type = type_conversions.type_from_tensors(0.0) dp_aggregate_process = differential_privacy.build_dp_aggregate_process( value_type, query) global_state = dp_aggregate_process.initialize() records = [1.0, 3.0, 5.0] def run_and_check(global_state, expected_l2_norm_clip, expected_result): output = dp_aggregate_process.next(global_state, records, [1.0, 1.0, 1.0]) self.assertEqual(output.state.l2_norm_clip, expected_l2_norm_clip) self.assertEqual(output.result, expected_result) return output.state self.assertEqual(global_state.l2_norm_clip, 4.0) global_state = run_and_check(global_state, 3.0, 8.0) global_state = run_and_check(global_state, 2.0, 7.0) global_state = run_and_check(global_state, 1.0, 5.0) global_state = run_and_check(global_state, 0.0, 3.0) global_state = run_and_check(global_state, 0.0, 0.0)
def test_dp_sum_structure_list(self): query = tensorflow_privacy.GaussianSumQuery(5.0, 0.0) def datapoint(a, b): return [tf.Variable(a, name='a'), tf.Variable(b, name='b')] data = [ datapoint(1.0, 2.0), datapoint(2.0, 3.0), datapoint(6.0, 8.0), # Clipped to 3.0, 4.0 ] value_type = type_conversions.type_from_tensors(data[0]) dp_aggregate_process = differential_privacy.build_dp_aggregate_process( value_type, query) global_state = dp_aggregate_process.initialize() output = dp_aggregate_process.next(global_state, data, [1.0, 1.0, 1.0]) self.assertEqual(output.state.l2_norm_clip, 5.0) self.assertEqual(output.state.stddev, 0.0) result = list(output.result) self.assertEqual(result[0], 6.0) self.assertEqual(result[1], 9.0)
def test_adaptation_up(self, shapes): value_type = type_conversions.type_from_tensors(_make_value(0, shapes)) mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process( value_type=value_type, initial_quantile_estimate=1.0, target_quantile=1.0, multiplier=1.0, increment=1.0, learning_rate=np.log(2.0), norm_order=np.inf) global_state = mean_process.initialize() values = [_make_value(x, shapes) for x in [90, 91, 92]] output = mean_process.next(global_state, values, [1, 1, 1]) self._check_result(output, 0, shapes) global_state = output.state metrics = output.measurements self.assertAllClose(metrics.zeroing_threshold, 3.0) self.assertEqual(metrics.num_zeroed, 3) output = mean_process.next(global_state, values, [1, 1, 1]) self._check_result(output, 0, shapes) global_state = output.state metrics = output.measurements self.assertAllClose(metrics.zeroing_threshold, 5.0) self.assertEqual(metrics.num_zeroed, 3)
def build_encoded_sum_process_from_model( model_fn: _ModelConstructor, encoder_fn: _EncoderConstructor) -> measured_process.MeasuredProcess: """Builds `MeasuredProcess` for weights of model returned by `model_fn`. This method creates a `GatherEncoder` for every trainable weight of model created by `model_fn`, as returned by `encoder_fn`. Args: model_fn: A Python callable with no arguments function that returns a `tff.learning.Model`. encoder_fn: A Python callable with a single argument, which is expected to be a `tf.Tensor` of shape and dtype to be encoded. The function must return a `tensor_encoding.core.SimpleEncoder`, which expects a `tf.Tensor` with compatible type as the input to its `encode` method. Returns: A `MeasuredProcess` for encoding and summing the weights of model created by `model_fn`. Raises: TypeError: If `model_fn` or `encoder_fn` are not callable objects. """ py_typecheck.check_callable(model_fn) py_typecheck.check_callable(encoder_fn) trainable_weights = _weights_from_model_fn(model_fn).trainable encoders = tf.nest.map_structure(encoder_fn, trainable_weights) weight_type = type_conversions.type_from_tensors(trainable_weights) return encoding_utils.build_encoded_sum_process(weight_type, encoders)
def test_adaptation_achieved_with_multiplier(self, shapes): value_type = type_conversions.type_from_tensors(_make_value(0, shapes)) mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process( value_type=value_type, initial_quantile_estimate=100.0, target_quantile=0.5, multiplier=2.0, increment=10.0, learning_rate=np.log(4.0), norm_order=np.inf) global_state = mean_process.initialize() values = [_make_value(x, shapes) for x in [30, 60]] # With target 0.5, learning rate λ=ln(4), estimate should be cut in # half in first round: exp(ln(4)(-0.5)) = 1/sqrt(4) = 0.5. output = mean_process.next(global_state, values, [1, 1]) self._check_result(output, 45, shapes) global_state = output.state metrics = output.measurements self.assertAllClose(metrics.zeroing_threshold, 110.0) self.assertEqual(metrics.num_zeroed, 0) # In second round, target is achieved, no adaptation occurs, and no updates # are zeroed becaues of multiplier. output = mean_process.next(global_state, values, [1, 1]) self._check_result(output, 45, shapes) global_state = output.state metrics = output.measurements self.assertAllClose(metrics.zeroing_threshold, 110.0) self.assertEqual(metrics.num_zeroed, 0)
def test_dp_sum_structure_complex(self): query = tensorflow_privacy.GaussianSumQuery(5.0, 0.0) def datapoint(a, b, c): return collections.OrderedDict(a=(a, ), bc=([b], (c, ))) data = [ datapoint(1.0, 2.0, 1.0), datapoint(2.0, 3.0, 1.0), datapoint(6.0, 8.0, 0.0), # Clipped to 3.0, 4.0, 0.0 ] value_type = type_conversions.type_from_tensors(data[0]) dp_aggregate_process = differential_privacy.build_dp_aggregate_process( value_type, query) global_state = dp_aggregate_process.initialize() output = dp_aggregate_process.next(global_state, data, [1.0, 1.0, 1.0]) self.assertEqual(output.state.l2_norm_clip, 5.0) self.assertEqual(output.state.stddev, 0.0) self.assertEqual(output.result['a'][0], 6.0) self.assertEqual(output.result['bc'][0][0], 9.0) self.assertEqual(output.result['bc'][1][0], 2.0)
def test_adaptation_achieved(self, shapes): value_type = type_conversions.type_from_tensors(_make_value(0, shapes)) mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process( value_type, 100.0, 0.5, 1.0, np.log(4.0), np.inf) global_state = mean_process.initialize() self.assertEqual(global_state.current_estimate, 100.0) values = [_make_value(x, shapes) for x in [30, 60]] # With target 0.5, learning rate λ=ln(4), estimate should be cut in # half in first round: exp(ln(4)(-0.5)) = 1/sqrt(4) = 0.5. output = mean_process.next(global_state, values, [1, 1]) self._check_result(output, 45, shapes) global_state = output['state'] self.assertAllClose(global_state.current_estimate, 50.0) metrics = output['measurements'] self.assertAllClose(metrics.current_threshold, 50.0) self.assertEqual(metrics.num_zeroed, 0) # In second round, target is achieved, no adaptation occurs, but one update # is zeroed. output = mean_process.next(global_state, values, [1, 1]) self._check_result(output, 30, shapes) global_state = output['state'] self.assertAllClose(global_state.current_estimate, 50.0) metrics = output['measurements'] self.assertAllClose(metrics.current_threshold, 50.0) self.assertEqual(metrics.num_zeroed, 1)
def build_encoded_mean(values, encoders): """Builds `StatefulAggregateFn` for `values`, to be encoded by `encoders`. Args: values: Values to be encoded by the `StatefulAggregateFn`. Must be convertible to `tff.Value`. encoders: A collection of `GatherEncoder` objects to be used for encoding `values`. Must have the same structure as `values`. Returns: A `StatefulAggregateFn` of which `next_fn` encodes the input at `tff.CLIENTS`, and computes their mean at `tff.SERVER`, automatically splitting the decoding part based on its commutativity with sum. Raises: ValueError: If `values` and `encoders` do not have the same structure. TypeError: If `encoders` are not instances of `GatherEncoder`, or if `values` are not compatible with the expected input of the `encoders`. """ warnings.warn( 'Deprecation warning: tff.utils.build_encoded_mean() is deprecated, use ' 'tff.utils.build_encoded_mean_process() instead.', DeprecationWarning) tf.nest.assert_same_structure(values, encoders) tf.nest.map_structure( lambda e, v: _validate_encoder(e, v, tensor_encoding.core.GatherEncoder ), encoders, values) value_type = type_conversions.type_from_tensors(values) initial_state_fn, state_type = _build_initial_state_tf_computation( encoders) nest_encoder = _build_tf_computations_for_gather(state_type, value_type, encoders) encoded_sum_fn = _build_encoded_sum_fn(nest_encoder) @computations.tf_computation(value_type, tf.float32) def multiply_fn(value, weight): return tf.nest.map_structure(lambda v: v * tf.cast(weight, v.dtype), value) @computations.tf_computation(value_type, tf.float32) def divide_fn(value, denominator): return tf.nest.map_structure( lambda v: v / tf.cast(denominator, v.dtype), value) def encoded_mean_fn(state, values, weight): weighted_values = intrinsics.federated_map(multiply_fn, [values, weight]) updated_state, summed_decoded_values = encoded_sum_fn( state, weighted_values) summed_weights = intrinsics.federated_sum(weight) decoded_values = intrinsics.federated_map( divide_fn, [summed_decoded_values, summed_weights]) return updated_state, decoded_values return computation_utils.StatefulAggregateFn( initialize_fn=initial_state_fn, next_fn=encoded_mean_fn)
def test_incorrect_finalizers_type_raises(self, bad_finalizers): local_unfinalized_metrics = collections.OrderedDict(num_examples=1.0) local_unfinalized_metrics_type = type_conversions.type_from_tensors( local_unfinalized_metrics) aggregate_factory = aggregation_factory.SumThenFinalizeFactory() with self.assertRaises(TypeError): aggregate_factory.create(bad_finalizers, local_unfinalized_metrics_type)
def test_process_type_signature(self, value_template): value_type = type_conversions.type_from_tensors(value_template) mean_process = adaptive_zeroing.build_adaptive_zeroing_mean_process( value_type=value_type, initial_quantile_estimate=100.0, target_quantile=0.99, multiplier=2.0, increment=1.0, learning_rate=1.0, norm_order=np.inf) dummy_quantile_query = tensorflow_privacy.NoPrivacyQuantileEstimatorQuery( 50.0, 0.99, 1.0, True) quantile_query_state = dummy_quantile_query.initial_global_state() server_state_type = computation_types.FederatedType( type_conversions.type_from_tensors(quantile_query_state), placements.SERVER) self.assertEqual( mean_process.initialize.type_signature, computation_types.FunctionType(parameter=None, result=server_state_type)) client_value_type = computation_types.FederatedType( value_type, placements.CLIENTS) client_value_weight_type = computation_types.FederatedType( tf.float32, placements.CLIENTS) server_result_type = computation_types.FederatedType( value_type, placements.SERVER) server_metrics_type = computation_types.FederatedType( adaptive_zeroing.AdaptiveZeroingMetrics( zeroing_threshold=tf.float32, num_zeroed=tf.int32), placements.SERVER) self.assertTrue( mean_process.next.type_signature.is_equivalent_to( computation_types.FunctionType( parameter=collections.OrderedDict( global_state=server_state_type, value=client_value_type, weight=client_value_weight_type), result=collections.OrderedDict( state=server_state_type, result=server_result_type, measurements=server_metrics_type, ))))
def test_evaluation_computation_custom_stateless_broadcaster( self, model_fn): def loss_fn(): return tf.keras.losses.MeanSquaredError() def metrics_fn(): return [counters.NumExamplesCounter(), NumOverCounter(5.0)] model_weights_type = type_conversions.type_from_tensors( reconstruction_utils.get_global_variables(model_fn())) def build_custom_stateless_broadcaster( model_weights_type) -> measured_process_lib.MeasuredProcess: """Builds a `MeasuredProcess` that wraps `tff.federated_broadcast`.""" @federated_computation.federated_computation() def test_server_initialization(): return intrinsics.federated_value((), placements.SERVER) @federated_computation.federated_computation( computation_types.FederatedType((), placements.SERVER), computation_types.FederatedType(model_weights_type, placements.SERVER), ) def stateless_broadcast(state, value): test_metrics = intrinsics.federated_value( 3.0, placements.SERVER) return measured_process_lib.MeasuredProcessOutput( state=state, result=intrinsics.federated_broadcast(value), measurements=test_metrics) return measured_process_lib.MeasuredProcess( initialize_fn=test_server_initialization, next_fn=stateless_broadcast) evaluate = evaluation_computation.build_federated_evaluation( model_fn, loss_fn=loss_fn, metrics_fn=metrics_fn, reconstruction_optimizer_fn=lambda: tf.keras.optimizers.SGD(0.1), broadcast_process=build_custom_stateless_broadcaster( model_weights_type=model_weights_type)) self.assertEqual( str(evaluate.type_signature), '(<server_model_weights=<trainable=<float32[1,1]>,' 'non_trainable=<>>@SERVER,federated_dataset={<x=float32[?,1],' 'y=float32[?,1]>*}@CLIENTS> -> <broadcast=float32,eval=' '<loss=float32,num_examples=int64,num_over=float32>>@SERVER)') result = evaluate( collections.OrderedDict([ ('trainable', [[[1.0]]]), ('non_trainable', []), ]), create_client_data()) self.assertEqual(result['broadcast'], 3.0)
def test_keras_metric_finalizer_returns_correct_result(self, metric): # The unfinalized accuracy contains two tensors `total` and `count`. unfinalized_accuracy = [tf.constant(2.0), tf.constant(2.0)] finalizer_computation = wrap_tf_function_in_tff_tf_computation( metric, type_conversions.type_from_tensors(unfinalized_accuracy)) finalized_accuracy = finalizer_computation(unfinalized_accuracy) self.assertEqual( # The expected value is computed by dividing `total` by `count`. finalized_accuracy, unfinalized_accuracy[0] / unfinalized_accuracy[1])
def test_returns_correct_results(self, metric_finalizers, local_unfinalized_metrics_at_clients, expected_aggregated_metrics): aggregator_computation = aggregator.sum_then_finalize( metric_finalizers=metric_finalizers, local_unfinalized_metrics_type=type_conversions.type_from_tensors( local_unfinalized_metrics_at_clients[0])) aggregated_metrics = aggregator_computation( local_unfinalized_metrics_at_clients) self.assertAllEqual(aggregated_metrics, expected_aggregated_metrics)
def test_raises_with_empty_value(self): value_type = type_conversions.type_from_tensors(()) with self.assertRaises(ValueError): adaptive_zeroing.build_adaptive_zeroing_mean_process( value_type=value_type, initial_quantile_estimate=100.0, target_quantile=0.99, multiplier=2.0, increment=1.0, learning_rate=1.0, norm_order=np.inf)
def test_keras_metric_finalizer_fails_with_unmatched_unfinalized_metric_values( self, invalid_unfinalized_metric_values, error_type, error_message): # The expected unfinalized metric values for `SparseCategoricalAccuracy` is # a list of two `tf.Tensor`s and each has shape () and dtype tf.float32. metric = tf.keras.metrics.SparseCategoricalAccuracy() with self.assertRaisesRegex(error_type, error_message): wrap_tf_function_in_tff_tf_computation( metric, type_conversions.type_from_tensors( invalid_unfinalized_metric_values))
def test_raises_on_invalid_distributor(self): model_weights_type = type_conversions.type_from_tensors( model_utils.ModelWeights.from_model( model_examples.LinearRegression())) distributor = distributors.build_broadcast_process(model_weights_type) invalid_distributor = iterative_process.IterativeProcess( distributor.initialize, distributor.next) with self.assertRaises(TypeError): fed_avg.build_weighted_fed_avg( model_fn=model_examples.LinearRegression, client_optimizer_fn=sgdm.build_sgdm(1.0), model_distributor=invalid_distributor)
def test_type_properties(self, value_type, inner_agg_factory): agg_factory = dp_factory.DifferentiallyPrivateFactory( _test_dp_query, inner_agg_factory) self.assertIsInstance(agg_factory, factory.UnweightedAggregationFactory) value_type = computation_types.to_type(value_type) process = agg_factory.create_unweighted(value_type) self.assertIsInstance(process, aggregation_process.AggregationProcess) query_state = _test_dp_query.initial_global_state() query_state_type = type_conversions.type_from_tensors(query_state) query_metrics_type = type_conversions.type_from_tensors( _test_dp_query.derive_metrics(query_state)) inner_state_type = tf.int32 if inner_agg_factory else () server_state_type = computation_types.at_server( (query_state_type, inner_state_type)) expected_initialize_type = computation_types.FunctionType( parameter=None, result=server_state_type) self.assertTrue( process.initialize.type_signature.is_equivalent_to( expected_initialize_type)) inner_measurements_type = tf.int32 if inner_agg_factory else () expected_measurements_type = computation_types.at_server( collections.OrderedDict( query_metrics=query_metrics_type, record_agg_process=inner_measurements_type)) expected_next_type = computation_types.FunctionType( parameter=collections.OrderedDict( state=server_state_type, value=computation_types.at_clients(value_type)), result=measured_process.MeasuredProcessOutput( state=server_state_type, result=computation_types.at_server(value_type), measurements=expected_measurements_type)) self.assertTrue( process.next.type_signature.is_equivalent_to(expected_next_type))
def test_unfinalized_metrics_type_and_initial_values_mismatch_raises(self): aggregate_factory = aggregation_factory.SumThenFinalizeFactory() metric_finalizers = collections.OrderedDict(num_examples=tf.function( func=lambda x: x)) local_unfinalized_metrics_type = type_conversions.type_from_tensors( collections.OrderedDict(num_examples=1.0)) initial_unfinalized_metrics = collections.OrderedDict( num_examples=[1.0]) with self.assertRaisesRegex(TypeError, 'initial unfinalized metrics type'): aggregate_factory.create(metric_finalizers, local_unfinalized_metrics_type, initial_unfinalized_metrics)
def test_raises_on_invalid_distributor(self): model_weights_type = type_conversions.type_from_tensors( model_utils.ModelWeights.from_model( model_examples.LinearRegression())) distributor = distributors.build_broadcast_process(model_weights_type) invalid_distributor = iterative_process.IterativeProcess( distributor.initialize, distributor.next) with self.assertRaises(TypeError): mime.build_weighted_mime_lite( model_fn=model_examples.LinearRegression, base_optimizer=sgdm.build_sgdm(learning_rate=0.01, momentum=0.9), model_distributor=invalid_distributor)
def test_sum_then_finalize_metrics(self): aggregate_factory = aggregation_factory.SumThenFinalizeFactory() metric_finalizers = collections.OrderedDict( num_examples=tf.function(func=lambda x: x), loss=tf.function(func=lambda x: tf.math.divide_no_nan(x[0], x[1])), custom_sum=tf.function( func=lambda x: tf.add_n(map(tf.math.reduce_sum, x)))) local_unfinalized_metrics = collections.OrderedDict( num_examples=1.0, loss=[2.0, 1.0], custom_sum=[tf.constant(1.0), tf.constant([1.0, 1.0])]) local_unfinalized_metrics_type = type_conversions.type_from_tensors( local_unfinalized_metrics) process = aggregate_factory.create(metric_finalizers, local_unfinalized_metrics_type) state = process.initialize() _, unfinalized_metrics_accumulators = state expected_unfinalized_metrics_accumulators = collections.OrderedDict( num_examples=0.0, loss=[0.0, 0.0], custom_sum=[tf.constant(0.0), tf.constant([0.0, 0.0])]) tf.nest.map_structure(self.assertAllEqual, unfinalized_metrics_accumulators, expected_unfinalized_metrics_accumulators) client_data = [local_unfinalized_metrics, local_unfinalized_metrics] output = process.next(state, client_data) _, unfinalized_metrics_accumulators = output.state current_round_metrics, total_rounds_metrics = output.result expected_unfinalized_metrics_accumulators = collections.OrderedDict( num_examples=2.0, loss=[4.0, 2.0], custom_sum=[tf.constant(2.0), tf.constant([2.0, 2.0])]) tf.nest.map_structure(self.assertAllEqual, unfinalized_metrics_accumulators, expected_unfinalized_metrics_accumulators) self.assertEqual( current_round_metrics, collections.OrderedDict(num_examples=2.0, loss=2.0, custom_sum=tf.constant(6.0))) self.assertEqual( total_rounds_metrics, collections.OrderedDict(num_examples=2.0, loss=2.0, custom_sum=tf.constant(6.0)))
def test_default_value_ranges_returns_correct_results( self, metric_finalizers, local_unfinalized_metrics_at_clients, expected_aggregated_metrics): aggregator_computation = aggregator.secure_sum_then_finalize( metric_finalizers=metric_finalizers, local_unfinalized_metrics_type=type_conversions.type_from_tensors( local_unfinalized_metrics_at_clients[0])) try: static_assert.assert_not_contains_unsecure_aggregation( aggregator_computation) except: # pylint: disable=bare-except self.fail( 'Metric aggregation contains non-secure summation aggregation') aggregated_metrics = aggregator_computation( local_unfinalized_metrics_at_clients) no_clipped_values = collections.OrderedDict( secure_upper_clipped_count=0, secure_lower_clipped_count=0, secure_upper_threshold=aggregator.DEFAULT_SECURE_UPPER_BOUND, secure_lower_threshold=aggregator.DEFAULT_SECURE_LOWER_BOUND) factory_keys = collections.OrderedDict() for value in tf.nest.flatten(local_unfinalized_metrics_at_clients[0]): tensor = tf.constant(value) if tensor.dtype.is_floating: lower = float(aggregator.DEFAULT_SECURE_LOWER_BOUND) upper = float(aggregator.DEFAULT_SECURE_UPPER_BOUND) elif tensor.dtype.is_integer: lower = int(aggregator.DEFAULT_SECURE_LOWER_BOUND) upper = int(aggregator.DEFAULT_SECURE_UPPER_BOUND) else: raise TypeError( f'Expected float or int, found tensors of dtype {tensor.dtype}.' ) factory_key = aggregator._create_factory_key( lower, upper, tensor.dtype) factory_keys[factory_key] = 1 expected_measurements = collections.OrderedDict( (factory_key, no_clipped_values) for factory_key in factory_keys) secure_sum_measurements = aggregated_metrics.pop( 'secure_sum_measurements') self.assertAllClose(secure_sum_measurements, expected_measurements) self.assertAllClose(aggregated_metrics, expected_aggregated_metrics, rtol=1e-5, atol=1e-5)