def terminate(self):
if self._task:
# trigger pytorch lighting training graceful shutdown via a ^C
self._task.set_exception(KeyboardInterrupt())
model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.FAIL))
model_bo = ModelService.get_model_by_id(self.model_id)
model_bo.model_status.remove(ModelStatus.TRAINING)
model_bo.model_status.append(ModelStatus.DRAFT)
ModelService.update_model(model_bo)
def start(self):
def training_done_callback(future):
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.PASS))
# TODO: save to database
print(self.export_model())
self._task = self._executor.submit(self.trainer_engine.fit, self.model,
**self._data_loader_kwargs)
self._task.add_done_callback(training_done_callback)
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.RUNNING))
def start(self):
def training_done_callback(future):
model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.PASS))
# TODO: save to database and update model_status, engine
print(self.export_model())
self._task = self._executor.submit(self.trainer_engine.fit, self.model, **self._data_loader_kwargs)
self._task.add_done_callback(training_done_callback)
model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.RUNNING))
model_bo = ModelService.get_model_by_id(self.model_id)
model_bo.model_status.remove(ModelStatus.DRAFT)
model_bo.model_status.append(ModelStatus.TRAINING)
ModelService.update_model(model_bo)
def update(training_job: TrainingJobUpdate) -> int:
# exists by ID
if not bool(
_collection.count_documents(
filter={'_id': ObjectId(training_job.id)}, limit=1)):
raise ValueError(f'id {training_job.id} not found.')
# check model ID
if training_job.model and not ModelDAO.exists_by_id(
ObjectId(training_job.model)):
raise ServiceException(
f'Model with ID {training_job.model} not exist.')
# save update
update_data = training_job.dict(exclude_unset=True)
result = _collection.update_one(filter={'_id': ObjectId(training_job.id)},
update={'$set': update_data})
return result.modified_count
def terminate(self):
if self._task:
# trigger pytorch lighting training graceful shutdown via a ^C
self._task.set_exception(KeyboardInterrupt())
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.FAIL))
def training_done_callback(future):
model_train_curd.update(
TrainingJobUpdate(_id=self._id, status=Status.PASS))
# TODO: save to database
print(self.export_model())