def sync_batch_result(self): storage.update_model(self.model_id, total_epoch=self.total_epoch, nth_epoch=self.nth_epoch, total_batch=self.total_batch, nth_batch=self.nth_batch, last_batch_loss=self.last_batch_loss)
def model_load_prediction_result(id): thread = PredictionThread.jobs.get(id, None) if thread is None: saved_model = storage.fetch_model(id) if saved_model is None: raise Exception("Model id {} is not found".format(id)) # If the state == STOPPED, client will never throw request. if saved_model["state"] != State.STOPPED.value: storage.update_model(id, state=State.STOPPED.value, running_state=RunningState.STOPPING.value) saved_model = storage.fetch_model(id) return {"result": saved_model['last_prediction_result']} else: thread.need_pull = False return {"result": thread.prediction_result}
def model_load_best_result(id): thread = TrainThread.jobs.get(id, None) if thread is None: saved_model = storage.fetch_model(id) if saved_model is None: return # If the state == STOPPED, client will never throw request. if saved_model["state"] != State.STOPPED.value: storage.update_model(id, state=State.STOPPED.value, running_state=RunningState.STOPPING.value) saved_model = storage.fetch_model(id) return {"best_result": saved_model['best_epoch_valid_result']} else: thread.returned_best_result2client() return {"best_result": thread.best_epoch_valid_result}
def sync_result(self): storage.update_model(self.model_id, last_prediction_result=self.prediction_result)
def sync_state(self): storage.update_model(self.model_id, state=self.state.value, running_state=self.running_state.value)
def sync_best_valid_result(self): storage.update_model( self.model_id, best_epoch_valid_result=self.best_epoch_valid_result)
def sync_valid_loss(self): storage.update_model(self.model_id, valid_loss_list=self.valid_loss_list)
def sync_train_loss(self): storage.update_model(self.model_id, train_loss_list=self.train_loss_list)
def polling_train(id): """ Cations: This function is possible to return empty dictionary. """ threads = TrainThread.jobs active_train_thread = threads.get(id, None) if active_train_thread is None: saved_model = storage.fetch_model(id) if saved_model is None: return # If the state == STOPPED, client will never throw request. if saved_model["state"] != State.STOPPED.value: storage.update_model(id, state=State.STOPPED.value, running_state=RunningState.STOPPING.value) saved_model = storage.fetch_model(id) return { "state": saved_model["state"], "running_state": saved_model["running_state"], "total_epoch": saved_model["total_epoch"], "nth_epoch": saved_model["nth_epoch"], "total_batch": saved_model["total_batch"], "nth_batch": saved_model["nth_batch"], "last_batch_loss": saved_model["last_batch_loss"], "total_valid_batch": 0, "nth_valid_batch": 0, "best_result_changed": False, "train_loss_list": saved_model["train_loss_list"], "valid_loss_list": saved_model["valid_loss_list"], } elif active_train_thread.state == State.RESERVED or \ active_train_thread.state == State.CREATED: for _ in range(60): if active_train_thread.state == State.RESERVED or \ active_train_thread.state == State.CREATED: time.sleep(1) if active_train_thread.updated: active_train_thread.returned2client() break else: time.sleep(1) break active_train_thread.consume_error() return { "state": active_train_thread.state.value, "running_state": active_train_thread.running_state.value, "total_epoch": 0, "nth_epoch": 0, "total_batch": 0, "nth_batch": 0, "last_batch_loss": 0, "total_valid_batch": 0, "nth_valid_batch": 0, "best_result_changed": False, "train_loss_list": [], "valid_loss_list": [], } else: for _ in range(10): time.sleep(0.5) # Avoid many request. if active_train_thread.updated: break active_train_thread.consume_error() active_train_thread.returned2client() return { "state": active_train_thread.state.value, "running_state": active_train_thread.running_state.value, "total_epoch": active_train_thread.total_epoch, "nth_epoch": active_train_thread.nth_epoch, "total_batch": active_train_thread.total_batch, "nth_batch": active_train_thread.nth_batch, "last_batch_loss": active_train_thread.last_batch_loss, "total_valid_batch": 0, "nth_valid_batch": 0, "best_result_changed": active_train_thread.best_valid_changed, "train_loss_list": active_train_thread.train_loss_list, "valid_loss_list": active_train_thread.valid_loss_list, }