def test_aggregate_fit() -> None:
"""Tests if adagrad fucntion is aggregating correclty."""
# Prepare
previous_weights: Weights = [array([0.1, 0.1, 0.1, 0.1], dtype=float32)]
strategy = FedAdagrad(
eta=0.1, eta_l=0.316, tau=0.5, initial_parameters=previous_weights
)
param_0: Parameters = weights_to_parameters(
[array([0.2, 0.2, 0.2, 0.2], dtype=float32)]
)
param_1: Parameters = weights_to_parameters(
[array([1.0, 1.0, 1.0, 1.0], dtype=float32)]
)
bridge = MagicMock()
client_0 = GrpcClientProxy(cid="0", bridge=bridge)
client_1 = GrpcClientProxy(cid="1", bridge=bridge)
results: List[Tuple[ClientProxy, FitRes]] = [
(
client_0,
FitRes(param_0, num_examples=5, num_examples_ceil=5, fit_duration=0.1),
),
(
client_1,
FitRes(param_1, num_examples=5, num_examples_ceil=5, fit_duration=0.1),
),
]
expected: Weights = [array([0.15, 0.15, 0.15, 0.15], dtype=float32)]
# Execute
actual_list = strategy.aggregate_fit(rnd=1, results=results, failures=[])
if actual_list:
actual = actual_list[0]
assert (actual == expected[0]).all()
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
if not results:
return None, {}
# Do not aggregate if there are failures and failures are not accepted
if not self.accept_failures and failures:
return None, {}
# Convert results
def norm_grad(grad_list: List[Weights]) -> float:
# input: nested gradients
# output: square of the L-2 norm
client_grads = grad_list[0]
for i in range(1, len(grad_list)):
client_grads = np.append(
client_grads, grad_list[i]
) # output a flattened array
return float(np.sum(np.square(client_grads)))
deltas = []
hs_ffl = []
if self.pre_weights is None:
raise Exception("QffedAvg pre_weights are None in aggregate_fit")
weights_before = self.pre_weights
eval_result = self.evaluate(weights_to_parameters(weights_before))
if eval_result is not None:
loss, _ = eval_result
for _, fit_res in results:
new_weights = parameters_to_weights(fit_res.parameters)
# plug in the weight updates into the gradient
grads = [
(u - v) * 1.0 / self.learning_rate
for u, v in zip(weights_before, new_weights)
]
deltas.append(
[np.float_power(loss + 1e-10, self.q_param) * grad for grad in grads]
)
# estimation of the local Lipschitz constant
hs_ffl.append(
self.q_param
* np.float_power(loss + 1e-10, (self.q_param - 1))
* norm_grad(grads)
+ (1.0 / self.learning_rate)
* np.float_power(loss + 1e-10, self.q_param)
)
weights_aggregated: Weights = aggregate_qffl(weights_before, deltas, hs_ffl)
return weights_to_parameters(weights_aggregated), {}
def fit(self, ins: FitIns) -> FitRes:
"""Refine the provided weights using the locally held dataset."""
# Deconstruct FitIns
parameters: List[np.ndarray] = parameters_to_weights(ins.parameters)
# Train
fit_begin = timeit.default_timer()
results = self.numpy_client.fit(parameters, ins.config)
if len(results) == 2:
print(DEPRECATION_WARNING_FIT)
results = cast(Tuple[List[np.ndarray], int], results)
parameters_prime, num_examples = results
metrics: Optional[Metrics] = None
elif len(results) == 3:
results = cast(Tuple[List[np.ndarray], int, Metrics], results)
parameters_prime, num_examples, metrics = results
# Return FitRes
fit_duration = timeit.default_timer() - fit_begin
parameters_prime_proto = weights_to_parameters(parameters_prime)
return FitRes(
parameters=parameters_prime_proto,
num_examples=num_examples,
num_examples_ceil=num_examples, # Deprecated
fit_duration=fit_duration, # Deprecated
metrics=metrics,
)
def fit(self, ins: FitIns) -> FitRes:
"""Refine the provided weights using the locally held dataset."""
# Deconstruct FitIns
weights: Weights = parameters_to_weights(ins.parameters)
# Train
fit_begin = timeit.default_timer()
results = self.keras_client.fit(weights, ins.config)
if len(results) == 3:
results = cast(Tuple[List[np.ndarray], int, int], results)
weights_prime, num_examples, num_examples_ceil = results
metrics: Optional[Metrics] = None
elif len(results) == 4:
results = cast(Tuple[List[np.ndarray], int, int, Metrics], results)
weights_prime, num_examples, num_examples_ceil, metrics = results
# Return FitRes
fit_duration = timeit.default_timer() - fit_begin
weights_prime_proto = weights_to_parameters(weights_prime)
return FitRes(
parameters=weights_prime_proto,
num_examples=num_examples,
num_examples_ceil=num_examples_ceil,
fit_duration=fit_duration,
metrics=metrics,
)
def on_configure_evaluate(
self, rnd: int, weights: Weights, client_manager: ClientManager
) -> List[Tuple[ClientProxy, EvaluateIns]]:
"""Configure the next round of evaluation."""
# Do not configure federated evaluation if a centralized evaluation
# function is provided
if self.eval_fn is not None:
return []
# Parameters and config
parameters = weights_to_parameters(weights)
config = {}
if self.on_evaluate_config_fn is not None:
# Custom evaluation config function provided
config = self.on_evaluate_config_fn(rnd)
evaluate_ins = (parameters, config)
# Sample clients
sample_size, min_num_clients = self.num_evaluation_clients(
client_manager.num_available())
clients = client_manager.sample(num_clients=sample_size,
min_num_clients=min_num_clients)
# Return client/config pairs
return [(client, evaluate_ins) for client in clients]
def initialize_parameters(
self, client_manager: ClientManager) -> Optional[Parameters]:
"""Initialize global model parameters."""
initial_parameters = self.initial_parameters
self.initial_parameters = None # Don't keep initial parameters in memory
if isinstance(initial_parameters, list):
initial_parameters = weights_to_parameters(
weights=initial_parameters)
return initial_parameters
def initialize_parameters(
self, client_manager: ClientManager
) -> Optional[Parameters]:
"""Initialize global model parameters."""
initial_parameters = self.initial_parameters
self.initial_parameters = None # Don't keep initial parameters in memory
if isinstance(initial_parameters, list):
log(WARNING, DEPRECATION_WARNING_INITIAL_PARAMETERS)
initial_parameters = weights_to_parameters(weights=initial_parameters)
return initial_parameters
def fit_round(
self, rnd: int
) -> Optional[Tuple[Optional[Parameters], Dict[str, Scalar],
FitResultsAndFailures]]:
"""Perform a single round of federated averaging."""
# Get clients and their respective instructions from strategy
client_instructions = self.strategy.configure_fit(
rnd=rnd,
parameters=self.parameters,
client_manager=self._client_manager)
if not client_instructions:
log(INFO, "fit_round: no clients selected, cancel")
return None
log(
DEBUG,
"fit_round: strategy sampled %s clients (out of %s)",
len(client_instructions),
self._client_manager.num_available(),
)
# Collect `fit` results from all clients participating in this round
results, failures = fit_clients(client_instructions)
log(
DEBUG,
"fit_round received %s results and %s failures",
len(results),
len(failures),
)
# Aggregate training results
aggregated_result: Union[Tuple[Optional[Parameters], Dict[str,
Scalar]],
Optional[Weights], # Deprecated
] = self.strategy.aggregate_fit(
rnd, results, failures)
metrics_aggregated: Dict[str, Scalar] = {}
if aggregated_result is None:
# Backward-compatibility, this will be removed in a future update
log(WARNING, DEPRECATION_WARNING_FIT_ROUND)
parameters_aggregated = None
elif isinstance(aggregated_result, list):
# Backward-compatibility, this will be removed in a future update
log(WARNING, DEPRECATION_WARNING_FIT_ROUND)
parameters_aggregated = weights_to_parameters(aggregated_result)
else:
parameters_aggregated, metrics_aggregated = aggregated_result
return parameters_aggregated, metrics_aggregated, (results, failures)
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
if not results:
return None, {}
# Check if enough results are available
completion_rate = len(results) / (len(results) + len(failures))
if completion_rate < self.min_completion_rate_fit:
# Not enough results for aggregation
return None, {}
# Convert results
weights_results = [(parameters_to_weights(fit_res.parameters),
fit_res.num_examples)
for client, fit_res in results]
weights_prime = aggregate(weights_results)
if self.importance_sampling:
# Track contributions to the global model
for client, fit_res in results:
cid = client.cid
assert fit_res.num_examples_ceil is not None
contribution: Tuple[int, int, int] = (
rnd,
fit_res.num_examples,
fit_res.num_examples_ceil,
)
if cid not in self.contributions.keys():
self.contributions[cid] = []
self.contributions[cid].append(contribution)
if self.dynamic_timeout:
self.durations = []
for client, fit_res in results:
assert fit_res.fit_duration is not None
assert fit_res.num_examples_ceil is not None
cid_duration = (
client.cid,
fit_res.fit_duration,
fit_res.num_examples,
fit_res.num_examples_ceil,
)
self.durations.append(cid_duration)
return weights_to_parameters(weights_prime), {}
def aggregate_fit(self, rnd: int, results: List[Tuple[ClientProxy,
FitRes]],
failures: List[BaseException]):
# get step
config = self.on_fit_config_fn(rnd)
# discriminate the aggregation to be performed
if config['model'] == 'k-FED':
# initial checks
if not results:
return None, {}
# Do not aggregate if there are failures and failures are not accepted
if not self.accept_failures and failures:
return None, {}
# getting all centroids --> (n_clients, n_centroids, n_dimensions)
all_centroids = np.array([
parameters_to_weights(fit_res.parameters)
for _, fit_res in results
])
print('All centroids\' shape: {}'.format(all_centroids.shape))
# pick, randomly, one client's centroids
idx = self.rng.integers(0, all_centroids.shape[0], 1)
# basis to be completed
base_centroids = all_centroids[idx][0]
# all other centroids
other_centroids = all_centroids[
np.arange(len(all_centroids)) != idx]
other_centroids = np.concatenate(other_centroids, axis=0)
# loop for completing the basis
while base_centroids.shape[0] < config['n_clusters']:
# all distances from the basis of centroids
distances = [
distance_from_centroids(base_centroids, c)
for c in other_centroids
]
# get the index of the maximum distance
idx = np.argmax(distances)
# add the new centroid --> (n_centroids, n_dimensions)
base_centroids = np.concatenate(
(base_centroids, [other_centroids[idx]]), axis=0)
print(base_centroids.shape)
# Save base_centroids
print(f"Saving base centroids...")
np.savez("base_centroids.npz", *base_centroids)
return weights_to_parameters(base_centroids), {}
else:
aggregated_weights = super().aggregate_fit(rnd, results, failures)
# Save aggregated_weights
print("Saving aggregated weights...")
np.savez(agg_weights_filename, *aggregated_weights)
return aggregated_weights
def fit(self, ins: FitIns) -> FitRes:
"""Refine the provided weights using the locally held dataset."""
# Deconstruct FitIns
weights: Weights = parameters_to_weights(ins[0])
config = ins[1]
# Train
fit_begin = timeit.default_timer()
weights_prime, num_examples, num_examples_ceil = self.keras_client.fit(
weights, config)
fit_duration = timeit.default_timer() - fit_begin
# Return FitRes
parameters = weights_to_parameters(weights_prime)
return parameters, num_examples, num_examples_ceil, fit_duration
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
if not results:
return None, {}
# Do not aggregate if there are failures and failures are not accepted
if not self.accept_failures and failures:
return None, {}
# Convert results
weights_results = [(parameters_to_weights(fit_res.parameters),
fit_res.num_examples)
for client, fit_res in results]
return weights_to_parameters(aggregate(weights_results)), {}
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
fedavg_parameters_aggregated, metrics_aggregated = super(
).aggregate_fit(rnd=rnd, results=results, failures=failures)
if fedavg_parameters_aggregated is None:
return None, {}
fedavg_weights_aggregate = parameters_to_weights(
fedavg_parameters_aggregated)
# Yogi
delta_t = [
x - y
for x, y in zip(fedavg_weights_aggregate, self.current_weights)
]
# m_t
if not self.m_t:
self.m_t = [np.zeros_like(x) for x in delta_t]
self.m_t = [
self.beta_1 * x + (1 - self.beta_1) * y
for x, y in zip(self.m_t, delta_t)
]
# v_t
if not self.v_t:
self.v_t = [np.zeros_like(x) for x in delta_t]
self.v_t = [
x - (1.0 - self.beta_2) * np.multiply(y, y) *
np.sign(x - np.multiply(y, y)) for x, y in zip(self.v_t, delta_t)
]
new_weights = [
x + self.eta * y / (np.sqrt(z) + self.tau)
for x, y, z in zip(self.current_weights, self.m_t, self.v_t)
]
self.current_weights = new_weights
return weights_to_parameters(self.current_weights), metrics_aggregated
def fit(self, ins: FitIns) -> FitRes:
"""Refine the provided weights using the locally held dataset."""
# Deconstruct FitIns
parameters: List[np.ndarray] = parameters_to_weights(ins.parameters)
# Train
fit_begin = timeit.default_timer()
parameters_prime, num_examples = self.numpy_client.fit(parameters, ins.config)
fit_duration = timeit.default_timer() - fit_begin
# Return FitRes
parameters_prime_proto = weights_to_parameters(parameters_prime)
return FitRes(
parameters=parameters_prime_proto,
num_examples=num_examples,
num_examples_ceil=num_examples, # num_examples == num_examples_ceil
fit_duration=fit_duration,
)
def on_configure_fit(
self, rnd: int, weights: Weights,
client_manager: ClientManager) -> List[Tuple[ClientProxy, FitIns]]:
"""Configure the next round of training."""
parameters = weights_to_parameters(weights)
config = {}
if self.on_fit_config_fn is not None:
# Custom fit config function provided
config = self.on_fit_config_fn(rnd)
fit_ins = FitIns(parameters, config)
# Sample clients
sample_size, min_num_clients = self.num_fit_clients(
client_manager.num_available())
clients = client_manager.sample(num_clients=sample_size,
min_num_clients=min_num_clients)
# Return client/config pairs
return [(client, fit_ins) for client in clients]
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
if not results:
return None, {}
# Check if enough results are available
completion_rate = len(results) / (len(results) + len(failures))
if completion_rate < self.completion_rate_fit:
# Not enough results for aggregation
return None, {}
# Convert results
weights_results = [(parameters_to_weights(fit_res.parameters),
fit_res.num_examples)
for client, fit_res in results]
return weights_to_parameters(aggregate(weights_results)), {}
def configure_fit(
self, rnd: int, weights: Weights, client_manager: ClientManager
) -> List[Tuple[ClientProxy, FitIns]]:
"""Configure the next round of training."""
# Block until `min_num_clients` are available
sample_size, min_num_clients = self.num_fit_clients(
client_manager.num_available()
)
success = client_manager.wait_for(
num_clients=min_num_clients, timeout=WAIT_TIMEOUT
)
if not success:
# Do not continue if not enough clients are available
log(
INFO,
"FedFS: not enough clients available after timeout %s",
WAIT_TIMEOUT,
)
return []
# Sample clients
clients = self._contribution_based_sampling(
sample_size=sample_size, client_manager=client_manager
)
# Prepare parameters and config
parameters = weights_to_parameters(weights)
config = {}
if self.on_fit_config_fn is not None:
# Use custom fit config function if provided
config = self.on_fit_config_fn(rnd)
# Set timeout for this round
use_fast_timeout = is_fast_round(rnd - 1, self.r_fast, self.r_slow)
config["timeout"] = str(self.t_fast if use_fast_timeout else self.t_slow)
# Fit instructions
fit_ins = FitIns(parameters, config)
# Return client/config pairs
return [(client, fit_ins) for client in clients]
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
fedavg_parameters_aggregated, metrics_aggregated = super(
).aggregate_fit(rnd=rnd, results=results, failures=failures)
if fedavg_parameters_aggregated is None:
return None, {}
fedavg_aggregate = parameters_to_weights(fedavg_parameters_aggregated)
aggregated_updates = [
subset_weights - self.current_weights[idx]
for idx, subset_weights in enumerate(fedavg_aggregate)
]
# Adagrad
delta_t = aggregated_updates
if not self.v_t:
self.v_t = [
np.zeros_like(subset_weights) for subset_weights in delta_t
]
self.v_t = [
self.v_t[idx] + np.multiply(subset_weights, subset_weights)
for idx, subset_weights in enumerate(delta_t)
]
new_weights = [
self.current_weights[idx] + self.eta * delta_t[idx] /
(np.sqrt(self.v_t[idx]) + self.tau) for idx in range(len(delta_t))
]
self.current_weights = new_weights
return weights_to_parameters(self.current_weights), metrics_aggregated
def get_parameters(self) -> ParametersRes:
"""Return the current local model parameters."""
weights = self.keras_client.get_weights()
parameters = weights_to_parameters(weights)
return ParametersRes(parameters=parameters)
def get_parameters(self) -> ParametersRes:
"""Return the current local model parameters."""
parameters = self.numpy_client.get_parameters()
parameters_proto = weights_to_parameters(parameters)
return ParametersRes(parameters=parameters_proto)
def configure_fit(
self, rnd: int, weights: Weights,
client_manager: ClientManager) -> List[Tuple[ClientProxy, FitIns]]:
"""Configure the next round of training."""
# Block until `min_num_clients` are available
sample_size, min_num_clients = self.num_fit_clients(
client_manager.num_available())
success = client_manager.wait_for(num_clients=min_num_clients,
timeout=WAIT_TIMEOUT)
if not success:
# Do not continue if not enough clients are available
log(
INFO,
"FedFS: not enough clients available after timeout %s",
WAIT_TIMEOUT,
)
return []
# Sample clients
msg = "FedFS round %s, sample %s clients (based on all previous contributions)"
if self.alternating_timeout:
log(
DEBUG,
msg,
str(rnd),
str(sample_size),
)
clients = self._contribution_based_sampling(
sample_size=sample_size, client_manager=client_manager)
elif self.importance_sampling:
if rnd == 1:
# Sample with 1/k in the first round
log(
DEBUG,
"FedFS round %s, sample %s clients with 1/k",
str(rnd),
str(sample_size),
)
clients = self._one_over_k_sampling(
sample_size=sample_size, client_manager=client_manager)
else:
fast_round = is_fast_round(rnd - 1,
r_fast=self.r_fast,
r_slow=self.r_slow)
log(
DEBUG,
"FedFS round %s, sample %s clients, fast_round %s",
str(rnd),
str(sample_size),
str(fast_round),
)
clients = self._fs_based_sampling(
sample_size=sample_size,
client_manager=client_manager,
fast_round=fast_round,
)
else:
clients = self._one_over_k_sampling(sample_size=sample_size,
client_manager=client_manager)
# Prepare parameters and config
parameters = weights_to_parameters(weights)
config = {}
if self.on_fit_config_fn is not None:
# Use custom fit config function if provided
config = self.on_fit_config_fn(rnd)
# Set timeout for this round
if self.dynamic_timeout:
if self.durations:
candidates = timeout_candidates(
durations=self.durations,
max_timeout=self.t_slow,
)
timeout = next_timeout(
candidates=candidates,
percentile=self.dynamic_timeout_percentile,
)
config["timeout"] = str(timeout)
else:
# Initial round has not past durations, use max_timeout
config["timeout"] = str(self.t_slow)
elif self.alternating_timeout:
use_fast_timeout = is_fast_round(rnd - 1, self.r_fast, self.r_slow)
config["timeout"] = str(
self.t_fast if use_fast_timeout else self.t_slow)
else:
config["timeout"] = str(self.t_slow)
# Fit instructions
fit_ins = FitIns(parameters, config)
# Return client/config pairs
return [(client, fit_ins) for client in clients]