def run_job(self, job, state=None):
"""
Identifies a DMLJob type and executes it.
If the runner is already executing a job, it silently does nothing.
"""
assert job.job_type.upper() in self.JOB_CALLBACKS, \
'DMLJob type ({0}) is not valid'.format(job.job_type)
logging.info("Running job (type: {0})...".format(job.job_type))
callback = callback_handler_no_default(
job.job_type,
self.JOB_CALLBACKS,
)
try:
if job.job_type in self.JOBS_NEEDING_STATE:
results = callback(job, state)
else:
results = callback(job)
except Exception as e:
logging.error("RunnerError: " + str(e))
results = DMLResult(
status='failed',
job=job,
error_message=str(e),
results={},
)
self.current_job = None
logging.info("Finished running job! (type: {0})".format(job.job_type))
return results
def ask(self, event_type, payload):
"""
Processes an event_type by calling the corresponding "LEVEL 1" Callback.
"""
callback = callback_handler_no_default(event_type,
self.LEVEL1_CALLBACKS)
return callback(payload)
def _create_session(self, payload):
"""
Creates a new session and optimizer based on the parameters sent by a
model developer. It then asks the optimizer what's the first thing to do
and the service starts working on that session from that point on.
"""
# NOTE: We removed the 'optimizer_type' argument since for the MVP we're
# only considering the 'FederatedAveragingOptimizer' for now.
# TODO: We need to incorporate session id's when we're ready.
if not self.dataset_manager:
raise Exception("Dataset Manager has not been set. Communication Manager needs to be configured first!")
assert payload.get(TxEnum.KEY.name) is MessageEventTypes.NEW_SESSION.name, \
"Expected a new session but got {}".format(payload.get(TxEnum.KEY.name))
initialization_payload = payload.get(TxEnum.CONTENT.name)
job_info = initialization_payload.get(TxEnum.CONTENT.name)
optimizer_type = job_info["optimizer_params"].get("optimizer_type", None)
optimizer_to_init = callback_handler_with_default(
optimizer_type,
self.OPTIMIZER_CALLBACK,
OptimizerTypes.FEDAVG.name
)
logging.info("New optimizer session is being set up...")
self.optimizer = optimizer_to_init(
initialization_payload,
self.dataset_manager
)
logging.info("Optimizer session is set! Now doing kickoff...")
event_type, payload = self.optimizer.kickoff()
callback = callback_handler_no_default(
event_type,
self.EVENT_TYPE_2_CALLBACK,
)
callback(payload)
logging.info("Kickoff complete.")
def _create_session(self, payload):
"""
Creates a new session and optimizer based on the parameters sent by a
model developer. It then asks the optimizer what's the first thing to do
and the service starts working on that session from that point on.
"""
# NOTE: We removed the 'optimizer_type' argument since for the MVP we're
# only considering the 'FederatedAveragingOptimizer' for now.
# TODO: We need to incorporate session id's when we're ready.
if not self.dataset_manager:
raise Exception("Dataset Manager has not been set. Communication \
Manager needs to be configured first!")
logging.info("New optimizer session is being set up...")
initialization_payload = payload.get(TxEnum.CONTENT.name)
self.optimizer = FederatedAveragingOptimizer(
initialization_payload,
self.dataset_manager
)
logging.info("Optimizer session is set! Now doing kickoff...")
event_type, payload = self.optimizer.kickoff()
callback = callback_handler_no_default(
event_type,
self.EVENT_TYPE_2_CALLBACK,
)
callback(payload)
logging.info("Kickoff complete.")
def _handle_job_done(self, dmlresult_obj):
"""
"LEVEL 1" Callback that handles the conversion of a DML Result into a
new DML Job by calling the appropriate "LEVEL 2" Callback.
"""
assert isinstance(dmlresult_obj, DMLResult)
callback = callback_handler_no_default(dmlresult_obj.job.job_type,
self.LEVEL2_JOB_DONE_CALLBACKS)
logging.info("Job completed: {}".format(dmlresult_obj.job.job_type))
return callback(dmlresult_obj)
def _handle_new_info(self, payload):
"""
"LEVEL 1" Callback to handle new information from the blockchain.
Payload structure should be:
{TxEnum.KEY.name: <e.g. NEW_WEIGHTS>,
TxEnum.CONTENT.name: <e.g. weights>}
"""
# TODO: Some assert on the payload, like in `_handle_job_done()`.
callback = callback_handler_no_default(payload[TxEnum.KEY.name],
self.LEVEL_2_NEW_INFO_CALLBACKS)
return callback(payload)
def run_job(self, job):
"""
Identifies a DMLJob type and executes it.
If the runner is already executing a job, it silently does nothing.
"""
assert job.job_type.upper() in self.JOB_CALLBACKS, \
'DMLJob type ({0}) is not valid'.format(job.job_type)
self.logger.info("Running job (type: {0})...".format(job.job_type))
callback = callback_handler_no_default(
job.job_type,
self.JOB_CALLBACKS,
)
results = callback(job)
results.session_id = job.session_id
self.current_job = None
self.logger.info("Finished running job! (type: {0})".format(
job.job_type))
return results
def inform(self, event_type, payload):
"""
Method called by other modules to inform the Communication Manager about
events that are going on in the service.
If there isn't an active optimizer, the Communication Manager will handle it
Else, payloads are relayed to the active optimizer which decides how
and to whom the Communication Manager should communicate the event/message.
For example: A runner could inform the Communication Manager that the
node is done training a particular model, to which the Optimizer could
decide it's time to communicate the new weights to the network.
"""
logging.info("Information has been received: {}".format(event_type))
if self.optimizer:
# We have an active session so we ask the optimizer what to do.
event_type, payload = self.optimizer.ask(event_type, payload)
callback = callback_handler_no_default(
event_type,
self.EVENT_TYPE_2_CALLBACK
)
callback(payload)