def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
"""Evaluate the provided parameters using the locally held dataset."""
parameters: List[np.ndarray] = parameters_to_weights(ins.parameters)
results = self.numpy_client.evaluate(parameters, ins.config)
if len(results) == 3:
if (
isinstance(results[0], float)
and isinstance(results[1], int)
and isinstance(results[2], dict)
):
# Forward-compatible case: loss, num_examples, metrics
results = cast(Tuple[float, int, Metrics], results)
loss, num_examples, metrics = results
evaluate_res = EvaluateRes(
loss=loss,
num_examples=num_examples,
metrics=metrics,
)
elif (
isinstance(results[0], int)
and isinstance(results[1], float)
and isinstance(results[2], float)
):
# Legacy case: num_examples, loss, accuracy
# This will be removed in a future release
print(DEPRECATION_WARNING_EVALUATE_0)
results = cast(Tuple[int, float, float], results)
num_examples, loss, accuracy = results
evaluate_res = EvaluateRes(
loss=loss,
num_examples=num_examples,
accuracy=accuracy, # Deprecated
)
else:
raise Exception(
"Return value expected to be of type (float, int, dict)."
)
elif len(results) == 4:
# Legacy case: num_examples, loss, accuracy, metrics
# This will be removed in a future release
print(DEPRECATION_WARNING_EVALUATE_1)
results = cast(Tuple[int, float, float, Metrics], results)
assert isinstance(results[0], int)
assert isinstance(results[1], float)
assert isinstance(results[2], float)
assert isinstance(results[3], dict)
num_examples, loss, accuracy, metrics = results
evaluate_res = EvaluateRes(
loss=loss,
num_examples=num_examples,
accuracy=accuracy, # Deprecated
metrics=metrics,
)
else:
raise Exception(EXCEPTION_MESSAGE_WRONG_RETURN_TYPE)
return evaluate_res
def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
"""Evaluate the provided parameters using the locally held dataset."""
parameters: List[np.ndarray] = parameters_to_weights(ins.parameters)
num_examples, loss, accuracy = self.numpy_client.evaluate(
parameters, ins.config
)
return EvaluateRes(num_examples=num_examples, loss=loss, accuracy=accuracy)
def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
"""Evaluate the provided weights using the locally held dataset."""
weights: Weights = parameters_to_weights(ins.parameters)
num_examples, loss, accuracy = self.keras_client.evaluate(
weights, ins.config)
return EvaluateRes(num_examples=num_examples,
loss=loss,
accuracy=accuracy)
def federated_personalized_evaluate(
self, ins: EvaluateIns) -> Tuple[EvaluateRes, EvaluateRes]:
"""Evaluate the provided parameters using the locally held dataset."""
parameters: List[np.ndarray] = parameters_to_weights(ins.parameters)
results = self.numpy_client.federated_personalized_evaluate(
parameters, ins.config)
if len(results) == 2: # two tuples of (baseline_res, personalized_res)
if (isinstance(results[0], tuple)
and isinstance(results[1], tuple)):
if (isinstance(results[0][0], float)
and isinstance(results[0][1], int)
and isinstance(results[0][2], dict)
and isinstance(results[1][0], float)
and isinstance(results[1][1], int)
and isinstance(results[1][2], dict)):
# Forward-compatible case: loss, num_examples, metrics
baseline_evaluate_results = cast(
Tuple[float, int, Metrics], results[0])
personalized_evaluate_results = cast(
Tuple[float, int, Metrics], results[1])
baseline_loss, baseline_num_examples, baseline_metrics = baseline_evaluate_results
personalized_loss, personalized_num_examples, personalized_metrics = personalized_evaluate_results
baseline_eval_res = EvaluateRes(
loss=baseline_loss,
num_examples=baseline_num_examples,
metrics=baseline_metrics,
)
personalized_eval_res = EvaluateRes(
loss=personalized_loss,
num_examples=personalized_num_examples,
metrics=personalized_metrics,
)
else:
raise Exception(
"Return value expected to be of type (tuple, tuple).")
return baseline_eval_res, personalized_eval_res
def test_aggregate_evaluate_no_failures() -> None:
"""Test evaluate function."""
# Prepare
strategy = FaultTolerantFedAvg(min_completion_rate_evaluate=0.99)
results: List[Tuple[ClientProxy, EvaluateRes]] = [
(MagicMock(), EvaluateRes(1, 2.3, 0.12))
]
failures: List[BaseException] = []
expected: Optional[float] = 2.3
# Execute
actual = strategy.aggregate_evaluate(1, results, failures)
# Assert
assert actual == expected
def test_aggregate_evaluate_just_enough_results() -> None:
"""Test evaluate function."""
# Prepare
strategy = FaultTolerantFedAvg(min_completion_rate_evaluate=0.5)
results: List[Tuple[ClientProxy, EvaluateRes]] = [
(MagicMock(), EvaluateRes(loss=2.3, num_examples=1))
]
failures: List[BaseException] = [Exception()]
expected: Optional[float] = 2.3
# Execute
actual, _ = strategy.aggregate_evaluate(1, results, failures)
# Assert
assert actual == expected
def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
"""Evaluate the provided weights using the locally held dataset."""
weights: Weights = parameters_to_weights(ins.parameters)
results = self.keras_client.evaluate(weights, ins.config)
# Note that accuracy is deprecated and will be removed in a future release
if len(results) == 3:
results = cast(Tuple[int, float, float], results)
num_examples, loss, accuracy = results
metrics: Optional[Metrics] = None
elif len(results) == 4:
results = cast(Tuple[int, float, float, Metrics], results)
num_examples, loss, accuracy, metrics = results
return EvaluateRes(num_examples=num_examples,
loss=loss,
accuracy=accuracy,
metrics=metrics)
def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
print(f"Client {self.cid}: evaluate")
weights = fl.common.parameters_to_weights(ins.parameters)
# Use provided weights to update the local model
self.model.set_weights(weights)
# Evaluate the updated model on the local dataset
testloader = torch.utils.data.DataLoader(self.testset,
batch_size=32,
shuffle=False)
loss, accuracy = cifar.test(self.model, testloader, device=DEVICE)
# Return the number of evaluation examples and the evaluation result (loss)
return EvaluateRes(num_examples=len(self.testset),
loss=float(loss),
accuracy=float(accuracy))
def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
return EvaluateRes(1, 1.0, 0.1)
def evaluate(self, ins: EvaluateIns) -> EvaluateRes:
return EvaluateRes(loss=1.0, num_examples=1)
