def sync_batch_result(self):
storage.update_model(self.model_id,
total_epoch=self.total_epoch,
nth_epoch=self.nth_epoch,
total_batch=self.total_batch,
nth_batch=self.nth_batch,
last_batch_loss=self.last_batch_loss)
def model_load_prediction_result(id):
thread = PredictionThread.jobs.get(id, None)
if thread is None:
saved_model = storage.fetch_model(id)
if saved_model is None:
raise Exception("Model id {} is not found".format(id))
# If the state == STOPPED, client will never throw request.
if saved_model["state"] != State.STOPPED.value:
storage.update_model(id, state=State.STOPPED.value,
running_state=RunningState.STOPPING.value)
saved_model = storage.fetch_model(id)
return {"result": saved_model['last_prediction_result']}
else:
thread.need_pull = False
return {"result": thread.prediction_result}
def model_load_best_result(id):
thread = TrainThread.jobs.get(id, None)
if thread is None:
saved_model = storage.fetch_model(id)
if saved_model is None:
return
# If the state == STOPPED, client will never throw request.
if saved_model["state"] != State.STOPPED.value:
storage.update_model(id, state=State.STOPPED.value,
running_state=RunningState.STOPPING.value)
saved_model = storage.fetch_model(id)
return {"best_result": saved_model['best_epoch_valid_result']}
else:
thread.returned_best_result2client()
return {"best_result": thread.best_epoch_valid_result}
def sync_result(self):
storage.update_model(self.model_id,
last_prediction_result=self.prediction_result)
def sync_state(self):
storage.update_model(self.model_id,
state=self.state.value,
running_state=self.running_state.value)
def sync_best_valid_result(self):
storage.update_model(
self.model_id,
best_epoch_valid_result=self.best_epoch_valid_result)
def sync_valid_loss(self):
storage.update_model(self.model_id,
valid_loss_list=self.valid_loss_list)
def sync_train_loss(self):
storage.update_model(self.model_id,
train_loss_list=self.train_loss_list)
def polling_train(id):
"""
Cations:
This function is possible to return empty dictionary.
"""
threads = TrainThread.jobs
active_train_thread = threads.get(id, None)
if active_train_thread is None:
saved_model = storage.fetch_model(id)
if saved_model is None:
return
# If the state == STOPPED, client will never throw request.
if saved_model["state"] != State.STOPPED.value:
storage.update_model(id, state=State.STOPPED.value,
running_state=RunningState.STOPPING.value)
saved_model = storage.fetch_model(id)
return {
"state": saved_model["state"],
"running_state": saved_model["running_state"],
"total_epoch": saved_model["total_epoch"],
"nth_epoch": saved_model["nth_epoch"],
"total_batch": saved_model["total_batch"],
"nth_batch": saved_model["nth_batch"],
"last_batch_loss": saved_model["last_batch_loss"],
"total_valid_batch": 0,
"nth_valid_batch": 0,
"best_result_changed": False,
"train_loss_list": saved_model["train_loss_list"],
"valid_loss_list": saved_model["valid_loss_list"],
}
elif active_train_thread.state == State.RESERVED or \
active_train_thread.state == State.CREATED:
for _ in range(60):
if active_train_thread.state == State.RESERVED or \
active_train_thread.state == State.CREATED:
time.sleep(1)
if active_train_thread.updated:
active_train_thread.returned2client()
break
else:
time.sleep(1)
break
active_train_thread.consume_error()
return {
"state": active_train_thread.state.value,
"running_state": active_train_thread.running_state.value,
"total_epoch": 0,
"nth_epoch": 0,
"total_batch": 0,
"nth_batch": 0,
"last_batch_loss": 0,
"total_valid_batch": 0,
"nth_valid_batch": 0,
"best_result_changed": False,
"train_loss_list": [],
"valid_loss_list": [],
}
else:
for _ in range(10):
time.sleep(0.5) # Avoid many request.
if active_train_thread.updated:
break
active_train_thread.consume_error()
active_train_thread.returned2client()
return {
"state": active_train_thread.state.value,
"running_state": active_train_thread.running_state.value,
"total_epoch": active_train_thread.total_epoch,
"nth_epoch": active_train_thread.nth_epoch,
"total_batch": active_train_thread.total_batch,
"nth_batch": active_train_thread.nth_batch,
"last_batch_loss": active_train_thread.last_batch_loss,
"total_valid_batch": 0,
"nth_valid_batch": 0,
"best_result_changed": active_train_thread.best_valid_changed,
"train_loss_list": active_train_thread.train_loss_list,
"valid_loss_list": active_train_thread.valid_loss_list,
}