def Register(self, request, context):
"""
register new clients on the fly.
Each client must get registered before getting the global model.
The server will expect updates from the registered clients
for multiple federated rounds.
This function does not change min_num_clients and max_num_clients.
"""
token = self.login_client(request, context)
client_ip = context.peer().split(':')[1]
# allow model requests/updates from this client
if self.is_from_authorized_client(token):
# previously known client, potentially contributed already
# will join the next round
return fed_msg.FederatedSummary(comment='Already registered',
token=token)
if len(self.auth_client_id) >= self.max_num_clients:
context.abort(grpc.StatusCode.RESOURCE_EXHAUSTED,
'Maximum number of clients reached')
# new client will join the current round immediately
self.auth_client_id.update({token: time.time()})
self.tokens[token] = self.model_meta_info
self.logger.info(
'Client: New client {} joined. Sent token: {}. Total clients: {}'.
format(request.client_id + '@' + client_ip, token,
len(self.auth_client_id)))
return fed_msg.FederatedSummary(comment='New client registered',
token=token)
def SubmitUpdate(self, request, context):
"""
handling client's submission of the federated updates
running aggregation if there are enough updates
"""
self.update_error = False
token = self.validate_client(request.client, context)
if token is None:
response_comment = 'Ignored the submit from invalid client. '
self.logger.info(response_comment)
# if len(self.accumulator) > self.min_num_clients:
# context.abort(grpc.StatusCode.ALREADY_EXISTS,
# 'Contrib: already enough in the current round')
else:
model_meta = self.is_valid_contribution(request.client.meta)
if model_meta is None:
context.abort(grpc.StatusCode.FAILED_PRECONDITION,
'Contrib: invalid for the current round')
response_comment = 'Invalid contribution. '
self.logger.info(response_comment)
else:
client_contrib_id = '{}_{}_{}'.format(model_meta.task.name, token,
model_meta.current_round)
start_time = request.client.meta.created
timenow = Timestamp()
timenow.GetCurrentTime()
time_seconds = timenow.seconds - start_time.seconds
self.logger.info(
'received %s (%s Bytes, %s seconds)', client_contrib_id,
request.ByteSize(), time_seconds or 'less than 1')
if self.save_contribution(client_contrib_id, request):
with self.lock:
self.accumulator.append(request)
# if self.get_enough_updates():
# self.aggregate()
num_of_updates = len(self.accumulator)
# Only the first one meets the minimum clients trigger the aggregation.
if num_of_updates == self.min_num_clients:
if num_of_updates < len(self.auth_client_id):
self.logger.debug("Starting to wait. {}".format(self.wait_after_min_clients))
time.sleep(self.wait_after_min_clients)
self.aggregate()
response_comment = \
'Received round {} from {} ({} Bytes, {} seconds)'.format(
request.client.meta.current_round, request.client.uid,
request.ByteSize(), time_seconds or 'less than 1')
summary_info = fed_msg.FederatedSummary(comment=response_comment)
if self.model_meta_info is not None:
summary_info.meta.CopyFrom(self.model_meta_info)
return summary_info
def Quit(self, request, context):
"""
existing client quits the federated training process.
Server will stop sharing the global model with the client,
further contribution will be rejected.
This function does not change min_num_clients and max_num_clients.
"""
token = self.validate_client(request, context)
self.auth_client_id.pop(token)
self.tokens.pop(token, None)
self.logger.info('Client: {} left. Total clients: {}'.format(
token, len(self.auth_client_id)))
return fed_msg.FederatedSummary(comment='Removed client')
def Heartbeat(self, request, context):
token = request.token
self.auth_client_id.update({token: time.time()})
self.logger.debug('Receive heartbeat from Client:{}'.format(token))
summary_info = fed_msg.FederatedSummary()
return summary_info