def register_model_from_yaml(file_path: Union[Path, str]): # check if file exist file_path = Path(file_path) assert file_path.exists( ), f'Model definition file at {str(file_path)} does not exist' # read yaml with open(file_path) as f: model_config = yaml.safe_load(f) model_yaml = MLModelFromYaml.parse_obj(model_config) # copy model weight to cache directory model_in_saved_path = model_yaml.saved_path if model_in_saved_path != model_yaml.weight: copy2(model_yaml.weight, model_in_saved_path) # zip weight folder if model_yaml.engine == Engine.TFS: weight_dir = model_yaml.weight make_archive(weight_dir.with_suffix('.zip'), 'zip', weight_dir) model_data = model_yaml.dict(exclude_none=True, exclude={'convert', 'profile'}) model = MLModel.parse_obj(model_data) register_model(model, convert=model_yaml.convert, profile=model_yaml.profile)
def detail(model_id: str = typer.Argument(..., help='Model ID')): """Show a single model.""" with requests.get( f'http://{SERVER_HOST}:{SERVER_PORT}/api/v1/model/{model_id}' ) as r: data = r.json() model_detailed_view(MLModel.parse_obj(data))
def get_by_id(id: str) -> MLModel: """Get a MLModel object by its ID. """ model_data = _collection.find_one(filter={'_id': ObjectId(id)}) if model_data is not None: return MLModel.parse_obj(model_data) else: raise ServiceException(f'Model with id={id} does not exist.')
def save(model_in: MLModel): """Register a model into ModelDB and GridFS. `model.id` should be set as `None`, otherwise, the function will raise a `ValueError`. Args: model_in (MLModelIn): model object to be registered Return: MLModel: Saved ML model object. Raises: BadRequestValueException: If `model.id` is not None. ServiceException: If model has exists with the same primary keys (name, framework, engine and version). """ if _collection.count_documents( filter=model_in.dict( use_enum_values=True, include={'architecture', 'framework', 'engine', 'version', 'task', 'dataset'} ), limit=1 ): raise ServiceException( f'Model with primary keys architecture={model_in.architecture}, ' f'framework={model_in.framework}, engine={model_in.engine}, version={model_in.version},' f'task={model_in.task}, and dataset={model_in.dataset} has exists.' ) # TODO: update weight ID in the MLModelIn weight_id = _fs.put(bytes(model_in.weight), filename=model_in.weight.filename) model = MLModel(**model_in.dict(exclude={'weight'}), weight=weight_id) model.id = _collection.insert_one(model.dict(exclude_none=True, by_alias=True, use_enum_values=True)).inserted_id return model
def _generate_model_family( model: MLModel, max_batch_size: int = -1 ): net = load(model.saved_path) build_saved_dir_from_engine = partial( generate_path_plain, **model.dict(include={'architecture', 'framework', 'task', 'version'}), ) inputs = model.inputs outputs = model.outputs model_input = model.model_input generated_dir_list = list() torchscript_dir = build_saved_dir_from_engine(engine=Engine.TORCHSCRIPT) tfs_dir = build_saved_dir_from_engine(engine=Engine.TFS) onnx_dir = build_saved_dir_from_engine(engine=Engine.ONNX) trt_dir = build_saved_dir_from_engine(engine=Engine.TRT) if isinstance(net, torch.nn.Module): # to TorchScript if converter.convert(net, 'pytorch', 'torchscript', save_path=torchscript_dir): generated_dir_list.append(torchscript_dir.with_suffix('.zip')) # to ONNX, TODO(lym): batch cache, input shape, opset version if converter.convert(net, 'pytorch', 'onnx', save_path=onnx_dir, inputs=inputs, outputs=outputs, model_input=model_input, optimize=False): generated_dir_list.append(onnx_dir.with_suffix('.onnx')) # to TRT # TRTConverter.from_onnx( # onnx_path=onnx_dir.with_suffix('.onnx'), save_path=trt_dir, inputs=inputs, outputs=outputs # ) elif isinstance(net, tf.keras.Model): # to TFS converter.convert(net, 'tensorflow', 'tfs', save_path=tfs_dir) generated_dir_list.append(tfs_dir.with_suffix('.zip')) # to TRT converter.convert(net, 'tfs', 'trt', tf_path=tfs_dir, save_path=trt_dir, inputs=inputs, outputs=outputs, max_batch_size=32) generated_dir_list.append(trt_dir.with_suffix('.zip')) return generated_dir_list
def list_models( architecture: Optional[str] = typer.Option(None, '-n', '--name', help='Model architecture name'), framework: Optional[Framework] = typer.Option(None, '-fw', '--framework', case_sensitive=False, help='Framework'), engine: Optional[Engine] = typer.Option(None, '-e', '--engine', case_sensitive=False, help='Serving engine'), version: Optional[int] = typer.Option(None, '-v', '--version', help='Version'), list_all: Optional[bool] = typer.Option( False, '-a', '--all', is_flag=True, help= 'Display queried models. otherwise, only partial result will be shown.' ), ): """Show a table that lists all models published in MLModelCI""" payload = remove_dict_null({ 'architecture': architecture, 'framework': framework, 'engine': engine, 'version': version }) with requests.get(f'{app_settings.api_v1_prefix}/model', params=payload) as r: model_list = r.json() model_view([MLModel.parse_obj(model) for model in model_list], list_all=list_all)
def generate_model_family(model: MLModel, max_batch_size: int = -1): model_weight_path = model.saved_path if not Path(model.saved_path).exists(): (filepath, filename) = os.path.split(model.saved_path) os.makedirs(filepath) with open(model.saved_path, 'wb') as f: f.write(model.weight.__bytes__()) net = load(model_weight_path) build_saved_dir_from_engine = partial( generate_path_plain, **model.dict(include={'architecture', 'framework', 'task', 'version'}), ) inputs = model.inputs outputs = model.outputs model_input = model.model_input generated_dir_list = list() torchscript_dir = build_saved_dir_from_engine(engine=Engine.TORCHSCRIPT) tfs_dir = build_saved_dir_from_engine(engine=Engine.TFS) onnx_dir = build_saved_dir_from_engine(engine=Engine.ONNX) trt_dir = build_saved_dir_from_engine(engine=Engine.TRT) if isinstance(net, torch.nn.Module): _torchfamily(net, False, torchscript_dir, onnx_dir, generated_dir_list, inputs, outputs, model_input) elif isinstance(net, tf.keras.Model): _tffamily(net, tfs_dir, generated_dir_list, trt_dir, inputs, outputs) elif isinstance(net, xgb.XGBModel): _xgbfamily(net, inputs, onnx_dir, generated_dir_list, torchscript_dir, outputs, model_input) elif isinstance(net, lgb.LGBMModel): _lgbfamily(net, inputs, onnx_dir, generated_dir_list, torchscript_dir, outputs, model_input) elif isinstance(net, skl.base.BaseEstimator): _sklfamily(net, inputs, onnx_dir, generated_dir_list, torchscript_dir, outputs, model_input) return generated_dir_list
async def publish_model( model: BaseMLModel = Depends(BaseMLModel.as_form), files: List[UploadFile] = File( [], description= 'This field can be set with empty value. In such settings, the publish is a dry run to' 'validate the `ml_model_in_form` field. You are recommend to try a dry run to find input' 'errors before you send the wight file(s) to the server.'), convert: bool = True, profile: bool = False): """Publish model to the model hub. The model weight file(s) as well as its meta information (e.g. architecture, framework, and serving engine) will be stored into the model hub. The publish API will also automatically convert the published model into other deployable format such as TorchScript and ONNX. After successfully converted, original model and its generated models will be profiled on the underlying devices in the clusters, and collects, aggregates, and processes running model performance. Args: model (MLModel): Model meta information. files (List[UploadFile]): A list of model weight files. The files are organized accordingly. Their file name contains relative path to their common parent directory. If the files is empty value, a dry-run to this API is conducted for parameter checks. No information will be saved into model hub in this case. convert (bool): Flag for auto configuration. profile (bool): Flag for auto profiling. Returns: A message response, with IDs of all published model. The format of the return is: ``` { "data": {"id": ["603e6a1f5a62b08bc0a2a7f2", "603e6a383b00cbe9bfee7277"]}, "status": true } ``` Specially, if the dry-run test passed, it will return a status True: ``` { "status": true } ``` """ # save the posted files as local cache loop = asyncio.get_event_loop() saved_path = model.saved_path if len(files) == 0: # conduct dry run for parameter check only. return {'status': True} if len(files) == 1: file = files[0] suffix = Path(file.filename).suffix try: # create directory if len(suffix) == 0: error = ErrorWrapper(ValueError( f'Expect a suffix for file {file.filename}, got None.'), loc='files[0]') raise RequestValidationError([error]) saved_path = saved_path.with_suffix(suffix) saved_path.parent.mkdir(exist_ok=True, parents=True) # save file await file.seek(0) with open(saved_path, 'wb') as buffer: await loop.run_in_executor(None, shutil.copyfileobj, file.file, buffer) finally: await file.close() else: raise NotImplementedError( '`publish_model` not implemented for multiple files upload.') # zip the files model = MLModel(**model.dict(), weight=saved_path) models = register_model(model=model, convert=convert, profile=profile) return { 'data': { 'id': [str(model.id) for model in models], }, 'status': True }
def convert( id: str = typer.Option(None, '-i', '--id', help='ID of model.'), yaml_file: Optional[Path] = typer. Option( None, '-f', '--yaml-file', exists=True, file_okay=True, help= 'Path to configuration YAML file. You should either set the `yaml_file` field or fields ' '(`FILE_OR_DIR`, `--name`, `--framework`, `--engine`, `--version`, `--task`, `--dataset`,' '`--metric`, `--input`, `--output`).'), register: bool = typer.Option( False, '-r', '--register', is_flag=True, help='register the converted models to modelhub, default false')): model = None if id is None and yaml_file is None: typer.echo( "WARNING: Please assign a way to find the target model! details refer to --help" ) raise RuntimeError if id is not None and yaml_file is not None: typer.echo("WARNING: Do not use -id and -path at the same time!") raise RuntimeError elif id is not None and yaml_file is None: if ModelDB.exists_by_id(id): model = ModelDB.get_by_id(id) else: typer.echo(f"model id: {id} does not exist in modelhub") elif id is None and yaml_file is not None: # get MLModel from yaml file with open(yaml_file) as f: model_config = yaml.safe_load(f) model_yaml = MLModelFromYaml.parse_obj(model_config) model_in_saved_path = model_yaml.saved_path if model_in_saved_path != model_yaml.weight: copy2(model_yaml.weight, model_in_saved_path) if model_yaml.engine == Engine.TFS: weight_dir = model_yaml.weight make_archive(weight_dir.with_suffix('.zip'), 'zip', weight_dir) model_data = model_yaml.dict(exclude_none=True, exclude={'convert', 'profile'}) model = MLModel.parse_obj(model_data) # auto execute all possible convert and return a list of save paths of every converted model generated_dir_list = generate_model_family(model) typer.echo(f"Converted models are save in: {generated_dir_list}") if register: model_data = model.dict( exclude={'weight', 'id', 'model_status', 'engine'}) for model_dir in generated_dir_list: parse_result = parse_path_plain(model_dir) engine = parse_result['engine'] model_cvt = MLModel(**model_data, weight=model_dir, engine=engine, model_status=[ModelStatus.CONVERTED]) ModelDB.save(model_cvt) typer.echo( f"converted {engine} are successfully registered in Modelhub")
def update( model_id: str = typer.Argument(..., help='Model ID'), architecture: Optional[str] = typer.Option(None, '-n', '--name', help='Architecture'), framework: Optional[Framework] = typer.Option(None, '-fw', '--framework', help='Framework'), engine: Optional[Engine] = typer.Option(None, '-e', '--engine', help='Engine'), version: Optional[int] = typer.Option(None, '-v', '--version', min=1, help='Version number'), task: Optional[Task] = typer.Option(None, '-t', '--task', help='Task'), dataset: Optional[str] = typer.Option(None, '-d', '--dataset', help='Dataset name'), metric: Optional[Dict[Metric, float]] = typer. Option( None, help='Metrics in the form of mapping JSON string. The map type is ' '`Dict[types.models.mlmodel.Metric, float]`. An example is \'{"acc": 0.76}.\'', ), inputs: Optional[List[IOShape]] = typer. Option( [], '-i', '--input', help= 'List of shape definitions for input tensors. An example of one shape definition is ' '\'{"name": "input", "shape": [-1, 3, 224, 224], "dtype": "TYPE_FP32", "format": "FORMAT_NCHW"}\'', ), outputs: Optional[List[IOShape]] = typer. Option( [], '-o', '--output', help= 'List of shape definitions for output tensors. An example of one shape definition is ' '\'{"name": "output", "shape": [-1, 1000], "dtype": "TYPE_FP32"}\'', )): model = ModelUpdateSchema( architecture=architecture, framework=framework, engine=engine, version=version, # noqa dataset=dataset, metric=metric, task=task, inputs=inputs, outputs=outputs) with requests.patch(f'{app_settings.api_v1_prefix}/model/{model_id}', data=model.json(exclude_defaults=True)) as r: data = r.json() model_detailed_view(MLModel.parse_obj(data))
def detail(model_id: str = typer.Argument(..., help='Model ID')): """Show a single model.""" with requests.get(f'{app_settings.api_v1_prefix}/model/{model_id}') as r: data = r.json() model_detailed_view(MLModel.parse_obj(data))
def update_finetune_model_as_new(id: str, updated_layer: Structure, dry_run: bool = False): # noqa """ Temporary function for finetune CV models. The function's functionality is overlapped with `update_model_structure_as_new`. Please use the `update_model_structure_as_new` in next release. Examples: Fine-tune the model by modify the layer with name 'fc' (last layer). The layer has a changed argument out_features = 10. op_='M' indicates the operation to this layer ('fc') is 'Modify'. There is no changes in layer connections. Therefore, the structure change summary is [M] fc: (...) out_features=10 >>> from collections import OrderedDict >>> structure_data = { ... 'layer': OrderedDict({'fc': {'out_features': 10, 'op_': 'M', 'type_': 'torch.nn.Linear'}}) ... } >>> update_finetune_model_as_new(id=..., updated_layer=Structure.parse_obj(structure_data)) Args: id (str): ID of the model to be updated. updated_layer (Structure): Contains layers to be fine-tuned. dry_run (bool): Test run for verify if the provided parameter (i.e. model specified in `id` and updated layers) is valid. Returns: """ if len(updated_layer.layer.items()) == 0: return True model = ModelService.get_model_by_id(id) if model.engine != Engine.PYTORCH: raise ValueError(f'model {id} is not supported for editing. ' f'Currently only support model with engine=PYTORCH') # download model as local cache cache_path = get_remote_model_weight(model=model) net = torch.load(cache_path) for layer_name, layer_param in updated_layer.layer.items(): layer_op = getattr(layer_param, 'op_') # update layer if layer_op == Operation.MODIFY: # check if the layer name exists # TODO check if layer path exists eg."layer1.0.conv1" if not hasattr(net, layer_name): raise ModelStructureError( f'Structure layer name `{layer_name}` not found in model {id}.' ) net_layer = getattr(net, layer_name) # check if the provided type matches the original type layer_type = type(net_layer) layer_type_provided = eval(layer_param.type_.value) # nosec if layer_type is not layer_type_provided: raise ModelStructureError( f'Expect `{layer_name}.type_` to be {layer_type}, ' f'but got {layer_type_provided}') # get layer parameters layer_param_old = layer_param.parse_layer_obj(net_layer) layer_param_data = layer_param_old.dict(exclude_none=True, exclude={'type_', 'op_'}) layer_param_update_data = layer_param.dict( exclude_none=True, exclude={'type_', 'op_'}) # replace 'null' with None. See reason :class:`ModelLayer`. for k, v in layer_param_update_data.items(): if v == 'null': layer_param_update_data[k] = None # update the layer parameters layer_param_data.update(layer_param_update_data) layer = layer_type(**layer_param_data) setattr(net, layer_name, layer) else: # if layer_op is Operation.ADD, # 1. check if the layer name not exists # 2. add a layer # 3. change the `forward` function according to the connections # if layer_op is Operation.DELETE, # 1. check if the layer exists # 2. delete the layer # 3. change the `forward` function raise ValueError( 'Operation not permitted. Please use `update_model_structure_as_new`.' ) input_tensors = list() bs = 1 for input_ in model.inputs: input_tensor = torch.rand(bs, *input_.shape[1:]).type( model_data_type_to_torch(input_.dtype)) input_tensors.append(input_tensor) # parse output tensors output_shapes = list() output_tensors = net(*input_tensors) if not isinstance(output_tensors, (list, tuple)): output_tensors = (output_tensors, ) for output_tensor in output_tensors: output_shape = IOShape(shape=[bs, *output_tensor.shape[1:]], dtype=type_to_data_type(output_tensor.dtype)) output_shapes.append(output_shape) if not dry_run: # TODO return validation result for dry_run mode # TODO apply Semantic Versioning https://semver.org/ # TODO reslove duplicate model version problem in a more efficient way version = ModelVersion(model.version.ver + 1) previous_models = ModelService.get_models( architecture=model.architecture, task=model.task, framework=model.framework, engine=Engine.NONE) if len(previous_models): last_version = max(previous_models, key=lambda k: k.version.ver).version.ver version = ModelVersion(last_version + 1) saved_path = generate_path_plain(architecture=model.architecture, task=model.task, framework=model.framework, engine=Engine.NONE, version=version) saved_path.parent.mkdir(parents=True, exist_ok=True) torch.save(model, saved_path.with_suffix('.pt')) mlmodelin = MLModel(dataset='', metric={key: 0 for key in model.metric.keys()}, task=model.task, inputs=model.inputs, outputs=output_shapes, architecture=model.name, framework=model.framework, engine=Engine.NONE, model_status=[ModelStatus.DRAFT], parent_model_id=model.id, version=version, weight=saved_path) register_model(mlmodelin, convert=False, profile=False) model_bo = ModelService.get_models(architecture=model.architecture, task=model.task, framework=model.framework, engine=Engine.NONE, version=version)[0] return {'id': model_bo.id}
def models(name, framework, engine, version, list_all, quiet): payload = remove_dict_null({'name': name, 'framework': framework, 'engine': engine, 'version': version}) with requests.get(f'{app_settings.api_v1_prefix}/model', params=payload) as r: model_list = r.json() model_view([MLModel.parse_obj(model) for model in model_list], list_all=list_all, quiet=quiet)
def register_model( model: MLModel, convert: bool = True, profile: bool = True, ) -> List[MLModel]: """Upload a model to ModelDB. This function will upload the given model into the database with some variation. It may optionally generate a branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be scheduled for each generated model, in order to gain profiling results for model selection strategies. In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine` and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced. TODO: This function has a super comprehensive logic, need to be simplified. Arguments: model: Required inputs for register a model. All information is wrapped in such model. convert (bool): Flag for generation of model family. Default to True. profile (bool): Flag for profiling uploaded (including converted) models. Default to True. """ models = list() model_dir_list = list() model.model_status = [ModelStatus.PUBLISHED] models.append(save(model)) # generate model family if convert: model_dir_list.extend(converter.generate_model_family(model)) # register model_data = model.dict(exclude={'weight', 'id', 'model_status', 'engine'}) for model_dir in model_dir_list: parse_result = parse_path_plain(model_dir) engine = parse_result['engine'] model_cvt = MLModel(**model_data, weight=model_dir, engine=engine, model_status=[ModelStatus.CONVERTED]) models.append(save(model_cvt)) # profile registered model if profile: from modelci.controller import job_executor from modelci.controller.executor import Job file = tf.keras.utils.get_file( "grace_hopper.jpg", "https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg" ) test_img_bytes = cv2.imread(file) kwargs = { 'repeat_data': test_img_bytes, 'batch_size': 32, 'batch_num': 100, 'asynchronous': False, } for model in models: model.model_status = [ModelStatus.PROFILING] ModelService.update_model(model) kwargs['model_info'] = model engine = model.engine if engine == Engine.TORCHSCRIPT: client = CVTorchClient(**kwargs) elif engine == Engine.TFS: client = CVTFSClient(**kwargs) elif engine == Engine.ONNX: client = CVONNXClient(**kwargs) elif engine == Engine.TRT: client = CVTRTClient(**kwargs) else: raise ValueError(f'No such serving engine: {engine}') job_cuda = Job(client=client, device='cuda:0', model_info=model) # job_cpu = Job(client=client, device='cpu', model_info=model) job_executor.submit(job_cuda) # job_executor.submit(job_cpu) return models