def _listen_to_model_validation_request_stream(self):
"""Subscribe to the model validation request stream. """
r = fedn.ClientAvailableMessage()
r.sender.name = self.name
r.sender.role = fedn.WORKER
while True:
try:
for request in self.orchestrator.ModelValidationRequestStream(r):
# Process validation request
model_id = request.model_id
self._send_status("Recieved model validation request.", log_level=fedn.Status.AUDIT,
type=fedn.StatusType.MODEL_VALIDATION_REQUEST, request=request)
self.inbox.put(('validate', request))
except grpc.RpcError as e:
status_code = e.code()
# TODO: make configurable
timeout = 5
#print("CLIENT __listen_to_model_validation_request_stream: GRPC ERROR {} retrying in {}..".format(
# status_code.name, timeout), flush=True)
time.sleep(timeout)
except:
raise
if not self._attached:
return
def _listen_to_model_update_request_stream(self):
"""Subscribe to the model update request stream. """
r = fedn.ClientAvailableMessage()
r.sender.name = self.name
r.sender.role = fedn.WORKER
metadata = [('client', r.sender.name)]
_disconnect = False
while True:
try:
for request in self.orchestrator.ModelUpdateRequestStream(r, metadata=metadata):
if request.sender.role == fedn.COMBINER:
# Process training request
self._send_status("Received model update request.", log_level=fedn.Status.AUDIT,
type=fedn.StatusType.MODEL_UPDATE_REQUEST, request=request)
self.inbox.put(('train', request))
if not self._attached:
return
except grpc.RpcError as e:
status_code = e.code()
#TODO: make configurable
timeout = 5
#print("CLIENT __listen_to_model_update_request_stream: GRPC ERROR {} retrying in {}..".format(
# status_code.name, timeout), flush=True)
time.sleep(timeout)
except:
raise
if not self._attached:
return
def __listen_to_model_update_request_stream(self):
""" Subscribe to the model update request stream. """
r = fedn.ClientAvailableMessage()
r.sender.name = self.name
r.sender.role = fedn.WORKER
metadata = [('client', r.sender.name)]
while True:
try:
for request in self.orchestrator.ModelUpdateRequestStream(
r, metadata=metadata):
if request.sender.role == fedn.COMBINER:
# Process training request
global_model_id = request.model_id
# TODO: Error handling
self.send_status(
"Received model update request.",
log_level=fedn.Status.AUDIT,
type=fedn.StatusType.MODEL_UPDATE_REQUEST,
request=request)
model_id = self.__process_training_request(
global_model_id)
if model_id != None:
# Notify the requesting client that a model update is available
update = fedn.ModelUpdate()
update.sender.name = self.name
update.sender.role = fedn.WORKER
update.receiver.name = request.sender.name
update.receiver.role = request.sender.role
update.model_id = request.model_id
update.model_update_id = str(model_id)
update.timestamp = str(datetime.now())
update.correlation_id = request.correlation_id
response = self.orchestrator.SendModelUpdate(
update)
self.send_status("Model update completed.",
log_level=fedn.Status.AUDIT,
type=fedn.StatusType.MODEL_UPDATE,
request=update)
else:
self.send_status(
"Client {} failed to complete model update.",
log_level=fedn.Status.WARNING,
request=request)
except grpc.RpcError as e:
status_code = e.code()
timeout = 5
print(
"CLIENT __listen_to_model_update_request_stream: GRPC ERROR {} retrying in {}.."
.format(status_code.name, timeout),
flush=True)
import time
time.sleep(timeout)
def __listen_to_status_stream(self):
r = alliance.ClientAvailableMessage()
r.sender.name = self.name
r.sender.role = alliance.OTHER
for status in self.connection.AllianceStatusStream(r):
print("MONITOR: Recived status:{}".format(status), flush=True)
data = MessageToDict(status, including_default_value_fields=True)
print(data, flush=True)
# Log to MongoDB
if self.collection:
self.collection.insert_one(data)
def __listen_to_model_validation_request_stream(self):
"""Subscribe to the model validation request stream. """
r = fedn.ClientAvailableMessage()
r.sender.name = self.name
r.sender.role = fedn.WORKER
while True:
try:
for request in self.orchestrator.ModelValidationRequestStream(
r):
# Process training request
model_id = request.model_id
# TODO: Error handling
self.send_status(
"Recieved model validation request.",
log_level=fedn.Status.AUDIT,
type=fedn.StatusType.MODEL_VALIDATION_REQUEST,
request=request)
metrics = self.__process_validation_request(model_id)
if metrics != None:
# Send validation
validation = fedn.ModelValidation()
validation.sender.name = self.name
validation.sender.role = fedn.WORKER
validation.receiver.name = request.sender.name
validation.receiver.role = request.sender.role
validation.model_id = str(model_id)
validation.data = json.dumps(metrics)
self.str = str(datetime.now())
validation.timestamp = self.str
validation.correlation_id = request.correlation_id
response = self.orchestrator.SendModelValidation(
validation)
self.send_status("Model validation completed.",
log_level=fedn.Status.AUDIT,
type=fedn.StatusType.MODEL_VALIDATION,
request=validation)
else:
self.send_status(
"Client {} failed to complete model validation.".
format(self.name),
log_level=fedn.Status.WARNING,
request=request)
except grpc.RpcError as e:
status_code = e.code()
timeout = 5
print(
"CLIENT __listen_to_model_validation_request_stream: GRPC ERROR {} retrying in {}.."
.format(status_code.name, timeout),
flush=True)
import time
time.sleep(timeout)