def terminate(self):
if self._task:
# trigger pytorch lighting training graceful shutdown via a ^C
self._task.set_exception(KeyboardInterrupt())
model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.FAIL))
model_bo = ModelService.get_model_by_id(self.model_id)
model_bo.model_status.remove(ModelStatus.TRAINING)
model_bo.model_status.append(ModelStatus.DRAFT)
ModelService.update_model(model_bo)
def start(self):
def training_done_callback(future):
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.PASS))
# TODO: save to database
print(self.export_model())
self._task = self._executor.submit(self.trainer_engine.fit, self.model,
**self._data_loader_kwargs)
self._task.add_done_callback(training_done_callback)
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.RUNNING))
def start(self):
def training_done_callback(future):
model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.PASS))
# TODO: save to database and update model_status, engine
print(self.export_model())
self._task = self._executor.submit(self.trainer_engine.fit, self.model, **self._data_loader_kwargs)
self._task.add_done_callback(training_done_callback)
model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.RUNNING))
model_bo = ModelService.get_model_by_id(self.model_id)
model_bo.model_status.remove(ModelStatus.DRAFT)
model_bo.model_status.append(ModelStatus.TRAINING)
ModelService.update_model(model_bo)
def terminate(self):
if self._task:
# trigger pytorch lighting training graceful shutdown via a ^C
self._task.set_exception(KeyboardInterrupt())
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.FAIL))
def training_done_callback(future):
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.PASS))
# TODO: save to database
print(self.export_model())