def on_configure_evaluate( self, rnd: int, weights: Weights, client_manager: ClientManager ) -> List[Tuple[ClientProxy, EvaluateIns]]: """Configure the next round of evaluation.""" # Do not configure federated evaluation if a centralized evaluation # function is provided if self.eval_fn is not None: return [] # Parameters and config parameters = weights_to_parameters(weights) config = {} if self.on_evaluate_config_fn is not None: # Custom evaluation config function provided config = self.on_evaluate_config_fn(rnd) evaluate_ins = EvaluateIns(parameters, config) # Sample clients if rnd >= 0: sample_size, min_num_clients = self.num_evaluation_clients( client_manager.num_available()) clients = client_manager.sample(num_clients=sample_size, min_num_clients=min_num_clients) else: clients = list(client_manager.all().values()) # Return client/config pairs return [(client, evaluate_ins) for client in clients]
def test_eval_clients() -> None: """Test eval_clients.""" # Prepare clients: List[ClientProxy] = [ FailingCLient("0"), SuccessClient("1"), ] arr = np.array([[1, 2], [3, 4], [5, 6]]) arr_serialized = ndarray_to_bytes(arr) ins: EvaluateIns = EvaluateIns( Parameters(tensors=[arr_serialized], tensor_type=""), {}, ) client_instructions = [(c, ins) for c in clients] # Execute results, failures = evaluate_clients(client_instructions) # Assert assert len(results) == 1 assert len(failures) == 1 assert results[0][1].num_examples == 1 assert results[0][1].loss == 1.0