def test_delete_model(): model = ModelService.get_models('ResNet50')[0] assert ModelService.delete_model_by_id(model.id)
def register_model( origin_model, dataset: str, metric: Dict[Metric, float], task: Task, inputs: List[IOShape], outputs: List[IOShape], model_input: Optional[List] = None, architecture: str = None, framework: Framework = None, engine: Engine = None, version: ModelVersion = None, convert=True, profile=True, ): """Upload a model to ModelDB. This function will upload the given model into the database with some variation. It may optionally generate a branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be scheduled for each generated model, in order to gain profiling results for model selection strategies. In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine` and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced. Arguments: origin_model: The uploaded model without optimization. When `no_generate` flag is set, this parameter should be a str indicating model file path. architecture (str): Model architecture name. Default to None. framework (Framework): Framework name. Default to None. version (ModelVersion): Model version. Default to None. dataset (str): Model testing dataset. metric (Dict[Metric,float]): Scoring metric and its corresponding score used for model evaluation task (Task): Model task type. inputs (Iterable[IOShape]): Model input tensors. outputs (Iterable[IOShape]): Model output tensors. model_input: specify sample model input data TODO: specify more model conversion related params engine (Engine): Model optimization engine. Default to `Engine.NONE`. convert (bool): Flag for generation of model family. When set, `origin_model` should be a path to model saving file. Default to `True`. profile (bool): Flag for profiling uploaded (including converted) models. Default to `False`. """ from modelci.controller import job_executor from modelci.controller.executor import Job model_dir_list = list() if not convert: # type and existence check assert isinstance(origin_model, str) model_dir = Path(origin_model).absolute() assert model_dir.exists( ), f'model weight does not exist at {origin_model}' if all([architecture, task, framework, engine, version]): # from explicit architecture, framework, engine and version ext = model_dir.suffix path = generate_path(architecture, task, framework, engine, version).with_suffix(ext) # if already in the destination folder if path == model_dir: pass # create destination folder else: if ext: path.parent.mkdir(parents=True, exist_ok=True) else: path.mkdir(parents=True, exist_ok=True) # copy to cached folder subprocess.call(['cp', model_dir, path]) else: # from implicit extracted from path, check validity of the path later at registration path = model_dir model_dir_list.append(path) else: # TODO: generate from path name # generate model variant model_dir_list.extend( _generate_model_family(origin_model, architecture, task, framework, filename=str(version), inputs=inputs, outputs=outputs, model_input=model_input)) # register for model_dir in model_dir_list: parse_result = parse_path(model_dir) architecture = parse_result['architecture'] task = parse_result['task'] framework = parse_result['framework'] engine = parse_result['engine'] version = parse_result['version'] filename = parse_result['filename'] with open(str(model_dir), 'rb') as f: model = ModelBO(name=architecture, task=task, framework=framework, engine=engine, version=version, dataset=dataset, metric=metric, inputs=inputs, outputs=outputs, weight=Weight(f, filename=filename)) ModelService.post_model(model) # TODO refresh model = ModelService.get_models(name=architecture, task=task, framework=framework, engine=engine, version=version)[0] # profile registered model if profile and engine != Engine.PYTORCH: file = tf.keras.utils.get_file( "grace_hopper.jpg", "https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg" ) test_img_bytes = cv2.imread(file) kwargs = { 'repeat_data': test_img_bytes, 'batch_size': 32, 'batch_num': 100, 'asynchronous': False, 'model_info': model, } if engine == Engine.TORCHSCRIPT: client = CVTorchClient(**kwargs) elif engine == Engine.TFS: client = CVTFSClient(**kwargs) elif engine == Engine.ONNX: client = CVONNXClient(**kwargs) elif engine == Engine.TRT: client = CVTRTClient(**kwargs) else: raise ValueError(f'No such serving engine: {engine}') job_cuda = Job(client=client, device='cuda:0', model_info=model) # job_cpu = Job(client=client, device='cpu', model_info=model) job_executor.submit(job_cuda)
def test_register_static_profiling_result(): model = ModelService.get_models('ResNet50')[0] spr = StaticProfileResultBO(5000, 200000, 200000, 10000, 10000, 10000) assert ModelService.register_static_profiling_result(model.id, spr)
def update_finetune_model_as_new(id: str, updated_layer: Structure, dry_run: bool = False): # noqa """ Temporary function for finetune CV models. The function's functionality is overlapped with `update_model_structure_as_new`. Please use the `update_model_structure_as_new` in next release. Examples: Fine-tune the model by modify the layer with name 'fc' (last layer). The layer has a changed argument out_features = 10. op_='M' indicates the operation to this layer ('fc') is 'Modify'. There is no changes in layer connections. Therefore, the structure change summary is [M] fc: (...) out_features=10 >>> from collections import OrderedDict >>> structure_data = { ... 'layer': OrderedDict({'fc': {'out_features': 10, 'op_': 'M', 'type_': 'torch.nn.Linear'}}) ... } >>> update_finetune_model_as_new(id=..., updated_layer=Structure.parse_obj(structure_data)) Args: id (str): ID of the model to be updated. updated_layer (Structure): Contains layers to be fine-tuned. dry_run (bool): Test run for verify if the provided parameter (i.e. model specified in `id` and updated layers) is valid. Returns: """ if len(updated_layer.layer.items()) == 0: return True model = ModelService.get_model_by_id(id) if model.engine != Engine.PYTORCH: raise ValueError(f'model {id} is not supported for editing. ' f'Currently only support model with engine=PYTORCH') # download model as local cache cache_path = get_remote_model_weight(model=model) net = torch.load(cache_path) for layer_name, layer_param in updated_layer.layer.items(): layer_op = getattr(layer_param, 'op_') # update layer if layer_op == Operation.MODIFY: # check if the layer name exists # TODO check if layer path exists eg."layer1.0.conv1" if not hasattr(net, layer_name): raise ModelStructureError( f'Structure layer name `{layer_name}` not found in model {id}.' ) net_layer = getattr(net, layer_name) # check if the provided type matches the original type layer_type = type(net_layer) layer_type_provided = eval(layer_param.type_.value) # nosec if layer_type is not layer_type_provided: raise ModelStructureError( f'Expect `{layer_name}.type_` to be {layer_type}, ' f'but got {layer_type_provided}') # get layer parameters layer_param_old = layer_param.parse_layer_obj(net_layer) layer_param_data = layer_param_old.dict(exclude_none=True, exclude={'type_', 'op_'}) layer_param_update_data = layer_param.dict( exclude_none=True, exclude={'type_', 'op_'}) # replace 'null' with None. See reason :class:`ModelLayer`. for k, v in layer_param_update_data.items(): if v == 'null': layer_param_update_data[k] = None # update the layer parameters layer_param_data.update(layer_param_update_data) layer = layer_type(**layer_param_data) setattr(net, layer_name, layer) else: # if layer_op is Operation.ADD, # 1. check if the layer name not exists # 2. add a layer # 3. change the `forward` function according to the connections # if layer_op is Operation.DELETE, # 1. check if the layer exists # 2. delete the layer # 3. change the `forward` function raise ValueError( 'Operation not permitted. Please use `update_model_structure_as_new`.' ) input_tensors = list() bs = 1 for input_ in model.inputs: input_tensor = torch.rand(bs, *input_.shape[1:]).type( model_data_type_to_torch(input_.dtype)) input_tensors.append(input_tensor) # parse output tensors output_shapes = list() output_tensors = net(*input_tensors) if not isinstance(output_tensors, (list, tuple)): output_tensors = (output_tensors, ) for output_tensor in output_tensors: output_shape = IOShape(shape=[bs, *output_tensor.shape[1:]], dtype=type_to_data_type(output_tensor.dtype)) output_shapes.append(output_shape) if not dry_run: # TODO return validation result for dry_run mode # TODO apply Semantic Versioning https://semver.org/ # TODO reslove duplicate model version problem in a more efficient way version = ModelVersion(model.version.ver + 1) previous_models = ModelService.get_models( architecture=model.architecture, task=model.task, framework=model.framework, engine=Engine.NONE) if len(previous_models): last_version = max(previous_models, key=lambda k: k.version.ver).version.ver version = ModelVersion(last_version + 1) saved_path = generate_path_plain(architecture=model.architecture, task=model.task, framework=model.framework, engine=Engine.NONE, version=version) saved_path.parent.mkdir(parents=True, exist_ok=True) torch.save(model, saved_path.with_suffix('.pt')) mlmodelin = MLModel(dataset='', metric={key: 0 for key in model.metric.keys()}, task=model.task, inputs=model.inputs, outputs=output_shapes, architecture=model.name, framework=model.framework, engine=Engine.NONE, model_status=[ModelStatus.DRAFT], parent_model_id=model.id, version=version, weight=saved_path) register_model(mlmodelin, convert=False, profile=False) model_bo = ModelService.get_models(architecture=model.architecture, task=model.task, framework=model.framework, engine=Engine.NONE, version=version)[0] return {'id': model_bo.id}
def test_get_model_by_id(): model_bo = ModelService.get_models('ResNet50')[0] model = ModelService.get_model_by_id(model_bo.id) # check model id assert model.id == model_bo.id