def evaluate(self, ins: EvaluateIns) -> EvaluateRes: """Evaluate the provided parameters using the locally held dataset.""" parameters: List[np.ndarray] = parameters_to_weights(ins.parameters) results = self.numpy_client.evaluate(parameters, ins.config) if len(results) == 3: if ( isinstance(results[0], float) and isinstance(results[1], int) and isinstance(results[2], dict) ): # Forward-compatible case: loss, num_examples, metrics results = cast(Tuple[float, int, Metrics], results) loss, num_examples, metrics = results evaluate_res = EvaluateRes( loss=loss, num_examples=num_examples, metrics=metrics, ) elif ( isinstance(results[0], int) and isinstance(results[1], float) and isinstance(results[2], float) ): # Legacy case: num_examples, loss, accuracy # This will be removed in a future release print(DEPRECATION_WARNING_EVALUATE_0) results = cast(Tuple[int, float, float], results) num_examples, loss, accuracy = results evaluate_res = EvaluateRes( loss=loss, num_examples=num_examples, accuracy=accuracy, # Deprecated ) else: raise Exception( "Return value expected to be of type (float, int, dict)." ) elif len(results) == 4: # Legacy case: num_examples, loss, accuracy, metrics # This will be removed in a future release print(DEPRECATION_WARNING_EVALUATE_1) results = cast(Tuple[int, float, float, Metrics], results) assert isinstance(results[0], int) assert isinstance(results[1], float) assert isinstance(results[2], float) assert isinstance(results[3], dict) num_examples, loss, accuracy, metrics = results evaluate_res = EvaluateRes( loss=loss, num_examples=num_examples, accuracy=accuracy, # Deprecated metrics=metrics, ) else: raise Exception(EXCEPTION_MESSAGE_WRONG_RETURN_TYPE) return evaluate_res
def evaluate(self, ins: EvaluateIns) -> EvaluateRes: """Evaluate the provided parameters using the locally held dataset.""" parameters: List[np.ndarray] = parameters_to_weights(ins.parameters) num_examples, loss, accuracy = self.numpy_client.evaluate( parameters, ins.config ) return EvaluateRes(num_examples=num_examples, loss=loss, accuracy=accuracy)
def evaluate(self, ins: EvaluateIns) -> EvaluateRes: """Evaluate the provided weights using the locally held dataset.""" weights: Weights = parameters_to_weights(ins.parameters) num_examples, loss, accuracy = self.keras_client.evaluate( weights, ins.config) return EvaluateRes(num_examples=num_examples, loss=loss, accuracy=accuracy)
def federated_personalized_evaluate( self, ins: EvaluateIns) -> Tuple[EvaluateRes, EvaluateRes]: """Evaluate the provided parameters using the locally held dataset.""" parameters: List[np.ndarray] = parameters_to_weights(ins.parameters) results = self.numpy_client.federated_personalized_evaluate( parameters, ins.config) if len(results) == 2: # two tuples of (baseline_res, personalized_res) if (isinstance(results[0], tuple) and isinstance(results[1], tuple)): if (isinstance(results[0][0], float) and isinstance(results[0][1], int) and isinstance(results[0][2], dict) and isinstance(results[1][0], float) and isinstance(results[1][1], int) and isinstance(results[1][2], dict)): # Forward-compatible case: loss, num_examples, metrics baseline_evaluate_results = cast( Tuple[float, int, Metrics], results[0]) personalized_evaluate_results = cast( Tuple[float, int, Metrics], results[1]) baseline_loss, baseline_num_examples, baseline_metrics = baseline_evaluate_results personalized_loss, personalized_num_examples, personalized_metrics = personalized_evaluate_results baseline_eval_res = EvaluateRes( loss=baseline_loss, num_examples=baseline_num_examples, metrics=baseline_metrics, ) personalized_eval_res = EvaluateRes( loss=personalized_loss, num_examples=personalized_num_examples, metrics=personalized_metrics, ) else: raise Exception( "Return value expected to be of type (tuple, tuple).") return baseline_eval_res, personalized_eval_res
def test_aggregate_evaluate_no_failures() -> None: """Test evaluate function.""" # Prepare strategy = FaultTolerantFedAvg(min_completion_rate_evaluate=0.99) results: List[Tuple[ClientProxy, EvaluateRes]] = [ (MagicMock(), EvaluateRes(1, 2.3, 0.12)) ] failures: List[BaseException] = [] expected: Optional[float] = 2.3 # Execute actual = strategy.aggregate_evaluate(1, results, failures) # Assert assert actual == expected
def test_aggregate_evaluate_just_enough_results() -> None: """Test evaluate function.""" # Prepare strategy = FaultTolerantFedAvg(min_completion_rate_evaluate=0.5) results: List[Tuple[ClientProxy, EvaluateRes]] = [ (MagicMock(), EvaluateRes(loss=2.3, num_examples=1)) ] failures: List[BaseException] = [Exception()] expected: Optional[float] = 2.3 # Execute actual, _ = strategy.aggregate_evaluate(1, results, failures) # Assert assert actual == expected
def evaluate(self, ins: EvaluateIns) -> EvaluateRes: """Evaluate the provided weights using the locally held dataset.""" weights: Weights = parameters_to_weights(ins.parameters) results = self.keras_client.evaluate(weights, ins.config) # Note that accuracy is deprecated and will be removed in a future release if len(results) == 3: results = cast(Tuple[int, float, float], results) num_examples, loss, accuracy = results metrics: Optional[Metrics] = None elif len(results) == 4: results = cast(Tuple[int, float, float, Metrics], results) num_examples, loss, accuracy, metrics = results return EvaluateRes(num_examples=num_examples, loss=loss, accuracy=accuracy, metrics=metrics)
def evaluate(self, ins: EvaluateIns) -> EvaluateRes: print(f"Client {self.cid}: evaluate") weights = fl.common.parameters_to_weights(ins.parameters) # Use provided weights to update the local model self.model.set_weights(weights) # Evaluate the updated model on the local dataset testloader = torch.utils.data.DataLoader(self.testset, batch_size=32, shuffle=False) loss, accuracy = cifar.test(self.model, testloader, device=DEVICE) # Return the number of evaluation examples and the evaluation result (loss) return EvaluateRes(num_examples=len(self.testset), loss=float(loss), accuracy=float(accuracy))
def evaluate(self, ins: EvaluateIns) -> EvaluateRes: return EvaluateRes(1, 1.0, 0.1)
def evaluate(self, ins: EvaluateIns) -> EvaluateRes: return EvaluateRes(loss=1.0, num_examples=1)