def run(self) -> None: from modelci.hub.deployer.dispatcher import serve for job in iter(self.job_queue.get, None): # exit the queue if job is self._queue_finish_flag: break # start a new container if container not started if job.container_name is None: container = serve(save_path=job.model.saved_path, device=job.device) container_name = container.name # remember to clean-up the created container self._hold_container.put(container) else: container_name = job.container_name # change model status job.model.status = Status.RUNNING ModelService.update_model(job.model) profiler = Profiler(model_info=job.model, server_name=container_name, inspector=job.client) dpr = profiler.diagnose(device=job.device) ModelService.append_dynamic_profiling_result(job.model.id, dynamic_result=dpr) # set model status to pass job.model.status = Status.PASS ModelService.update_model(job.model) if job.container_name is None: # get holding container self._hold_container.get().stop()
def serve_by_task(args): model_bo = retrieve_model_by_task(task=args.task) serve(model_bo[0].saved_path, device=args.device, name=args.name, batch_size=args.bs) # TODO: check if the service is dispatched sucessfully new_status = [item for item in model_bo[0].model_status if item is not (ModelStatus.CONVERTED or ModelStatus.PUBLISHED)] new_status.append(ModelStatus.IN_SERVICE) model_bo[0].model_status = new_status ModelService.update_model(model_bo[0])
def terminate(self): if self._task: # trigger pytorch lighting training graceful shutdown via a ^C self._task.set_exception(KeyboardInterrupt()) model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.FAIL)) model_bo = ModelService.get_model_by_id(self.model_id) model_bo.model_status.remove(ModelStatus.TRAINING) model_bo.model_status.append(ModelStatus.DRAFT) ModelService.update_model(model_bo)
def serve_by_name(args): model = args.model framework = Framework[args.framework.upper()] engine = Engine[args.engine.upper()] model_bo = retrieve_model(architecture=model, framework=framework, engine=engine) serve(model_bo[0].saved_path, device=args.device, name=args.name, batch_size=args.bs) # TODO: check if the service is dispatched sucessfully new_status = [item for item in model_bo[0].model_status if item is not (ModelStatus.CONVERTED or ModelStatus.PUBLISHED)] new_status.append(ModelStatus.IN_SERVICE) model_bo[0].model_status = new_status ModelService.update_model(model_bo[0])
def start(self): def training_done_callback(future): model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.PASS)) # TODO: save to database and update model_status, engine print(self.export_model()) self._task = self._executor.submit(self.trainer_engine.fit, self.model, **self._data_loader_kwargs) self._task.add_done_callback(training_done_callback) model_train_curd.update(TrainingJobUpdate(_id=self._id, status=Status.RUNNING)) model_bo = ModelService.get_model_by_id(self.model_id) model_bo.model_status.remove(ModelStatus.DRAFT) model_bo.model_status.append(ModelStatus.TRAINING) ModelService.update_model(model_bo)
def test_update_model(): model = ModelService.get_models('ResNet50')[0] model.acc = 0.9 model.weight.weight = bytes([123, 255]) # check if update success assert ModelService.update_model(model) model_ = ModelService.get_models('ResNet50')[0] # check updated model assert abs(model_.acc - 0.9) < 1e-6 assert model_.weight.weight == model.weight.weight
def register_model(origin_model, dataset: str, metric: Dict[Metric, float], task: Task, inputs: List[IOShape], outputs: List[IOShape], model_input: Optional[List] = None, architecture: str = None, framework: Framework = None, engine: Engine = None, version: ModelVersion = None, parent_model_id: Optional[str] = None, convert: bool = True, profile: bool = True, model_status: List[ModelStatus] = None): """Upload a model to ModelDB. This function will upload the given model into the database with some variation. It may optionally generate a branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be scheduled for each generated model, in order to gain profiling results for model selection strategies. In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine` and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced. TODO: This function has a super comprehensive logic, need to be simplified. Arguments: origin_model: The uploaded model without optimization. When `no_generate` flag is set, this parameter should be a str indicating model file path. architecture (str): Model architecture name. Default to None. framework (Framework): Framework name. Default to None. version (ModelVersion): Model version. Default to None. dataset (str): Model testing dataset. metric (Dict[Metric,float]): Scoring metric and its corresponding score used for model evaluation task (Task): Model task type. inputs (Iterable[IOShape]): Model input tensors. outputs (Iterable[IOShape]): Model output tensors. model_input: specify sample model input data TODO: specify more model conversion related params engine (Engine): Model optimization engine. Default to `Engine.NONE`. parent_model_id (Optional[str]): the parent model id of current model if this model is derived from a pre-existing one model_status (List[ModelStatus]): Indicate the status of current model in its lifecycle convert (bool): Flag for generation of model family. When set, `origin_model` should be a path to model saving file. Default to `True`. profile (bool): Flag for profiling uploaded (including converted) models. Default to `False`. """ from modelci.controller import job_executor from modelci.controller.executor import Job model_dir_list = list() # type and existence check if isinstance(origin_model, str): model_dir = Path(origin_model).absolute() assert model_dir.exists( ), f'model weight does not exist at {origin_model}' if all([architecture, task, framework, engine, version]): # from explicit architecture, framework, engine and version ext = model_dir.suffix path = generate_path(architecture, task, framework, engine, version).with_suffix(ext) # if already in the destination folder if path == model_dir: pass # create destination folder else: if ext: path.parent.mkdir(parents=True, exist_ok=True) else: path.mkdir(parents=True, exist_ok=True) # copy to cached folder subprocess.call(['cp', model_dir, path]) else: # from implicit extracted from path, check validity of the path later at registration path = model_dir model_dir_list.append(path) elif framework == Framework.PYTORCH and engine in [ Engine.PYTORCH, Engine.NONE ]: # save original pytorch model pytorch_dir = generate_path( task=task, model_name=architecture, framework=framework, engine=engine, version=str(version), ) pytorch_dir.parent.mkdir(parents=True, exist_ok=True) save_path_with_ext = pytorch_dir.with_suffix('.pth') torch.save(origin_model, str(save_path_with_ext)) model_dir_list.append(pytorch_dir.with_suffix('.pth')) if convert: # TODO: generate from path name # generate model variant model_dir_list.extend( _generate_model_family(origin_model, architecture, task, framework, filename=str(version), inputs=inputs, outputs=outputs, model_input=model_input)) # register for model_dir in model_dir_list: parse_result = parse_path(model_dir) architecture = parse_result['architecture'] task = parse_result['task'] framework = parse_result['framework'] engine = parse_result['engine'] version = parse_result['version'] filename = parse_result['filename'] if model_status is not None: model_bo_status = model_status elif engine == Engine.PYTORCH: model_bo_status = [ModelStatus.PUBLISHED] else: model_bo_status = [ModelStatus.CONVERTED] with open(str(model_dir), 'rb') as f: model = ModelBO(name=architecture, task=task, framework=framework, engine=engine, version=version, dataset=dataset, metric=metric, parent_model_id=parent_model_id, inputs=inputs, outputs=outputs, model_status=model_bo_status, weight=Weight(f, filename=filename)) ModelService.post_model(model) # TODO refresh model = ModelService.get_models(name=architecture, task=task, framework=framework, engine=engine, version=version)[0] if model.engine == Engine.PYTORCH or model.engine == Engine.TFS: parent_model_id = model.id # profile registered model if profile and engine != Engine.PYTORCH: file = tf.keras.utils.get_file( "grace_hopper.jpg", "https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg" ) test_img_bytes = cv2.imread(file) kwargs = { 'repeat_data': test_img_bytes, 'batch_size': 32, 'batch_num': 100, 'asynchronous': False, 'model_info': model, } new_status = [ item for item in model.model_status if item is not (ModelStatus.CONVERTED or ModelStatus.PUBLISHED) ] new_status.append(ModelStatus.PROFILING) model.model_status = new_status ModelService.update_model(model) if engine == Engine.TORCHSCRIPT: client = CVTorchClient(**kwargs) elif engine == Engine.TFS: client = CVTFSClient(**kwargs) elif engine == Engine.ONNX: client = CVONNXClient(**kwargs) elif engine == Engine.TRT: client = CVTRTClient(**kwargs) else: raise ValueError(f'No such serving engine: {engine}') job_cuda = Job(client=client, device='cuda:0', model_info=model) # job_cpu = Job(client=client, device='cpu', model_info=model) job_executor.submit(job_cuda)
def register_model( model: MLModel, convert: bool = True, profile: bool = True, ) -> List[MLModel]: """Upload a model to ModelDB. This function will upload the given model into the database with some variation. It may optionally generate a branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be scheduled for each generated model, in order to gain profiling results for model selection strategies. In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine` and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced. TODO: This function has a super comprehensive logic, need to be simplified. Arguments: model: Required inputs for register a model. All information is wrapped in such model. convert (bool): Flag for generation of model family. Default to True. profile (bool): Flag for profiling uploaded (including converted) models. Default to True. """ models = list() model_dir_list = list() model.model_status = [ModelStatus.PUBLISHED] models.append(save(model)) # generate model family if convert: model_dir_list.extend(converter.generate_model_family(model)) # register model_data = model.dict(exclude={'weight', 'id', 'model_status', 'engine'}) for model_dir in model_dir_list: parse_result = parse_path_plain(model_dir) engine = parse_result['engine'] model_cvt = MLModel(**model_data, weight=model_dir, engine=engine, model_status=[ModelStatus.CONVERTED]) models.append(save(model_cvt)) # profile registered model if profile: from modelci.controller import job_executor from modelci.controller.executor import Job file = tf.keras.utils.get_file( "grace_hopper.jpg", "https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg" ) test_img_bytes = cv2.imread(file) kwargs = { 'repeat_data': test_img_bytes, 'batch_size': 32, 'batch_num': 100, 'asynchronous': False, } for model in models: model.model_status = [ModelStatus.PROFILING] ModelService.update_model(model) kwargs['model_info'] = model engine = model.engine if engine == Engine.TORCHSCRIPT: client = CVTorchClient(**kwargs) elif engine == Engine.TFS: client = CVTFSClient(**kwargs) elif engine == Engine.ONNX: client = CVONNXClient(**kwargs) elif engine == Engine.TRT: client = CVTRTClient(**kwargs) else: raise ValueError(f'No such serving engine: {engine}') job_cuda = Job(client=client, device='cuda:0', model_info=model) # job_cpu = Job(client=client, device='cpu', model_info=model) job_executor.submit(job_cuda) # job_executor.submit(job_cpu) return models