def on_configure_evaluate(
self, rnd: int, weights: Weights, client_manager: ClientManager
) -> List[Tuple[ClientProxy, EvaluateIns]]:
"""Configure the next round of evaluation."""
# Do not configure federated evaluation if a centralized evaluation
# function is provided
if self.eval_fn is not None:
return []
# Parameters and config
parameters = weights_to_parameters(weights)
config = {}
if self.on_evaluate_config_fn is not None:
# Custom evaluation config function provided
config = self.on_evaluate_config_fn(rnd)
evaluate_ins = EvaluateIns(parameters, config)
# Sample clients
if rnd >= 0:
sample_size, min_num_clients = self.num_evaluation_clients(
client_manager.num_available())
clients = client_manager.sample(num_clients=sample_size,
min_num_clients=min_num_clients)
else:
clients = list(client_manager.all().values())
# Return client/config pairs
return [(client, evaluate_ins) for client in clients]
def test_eval_clients() -> None:
"""Test eval_clients."""
# Prepare
clients: List[ClientProxy] = [
FailingCLient("0"),
SuccessClient("1"),
]
arr = np.array([[1, 2], [3, 4], [5, 6]])
arr_serialized = ndarray_to_bytes(arr)
ins: EvaluateIns = EvaluateIns(
Parameters(tensors=[arr_serialized], tensor_type=""),
{},
)
client_instructions = [(c, ins) for c in clients]
# Execute
results, failures = evaluate_clients(client_instructions)
# Assert
assert len(results) == 1
assert len(failures) == 1
assert results[0][1].num_examples == 1
assert results[0][1].loss == 1.0