def test_delete_model():
model = ModelService.get_models('ResNet50')[0]
assert ModelService.delete_model_by_id(model.id)
def register_model(
origin_model,
dataset: str,
metric: Dict[Metric, float],
task: Task,
inputs: List[IOShape],
outputs: List[IOShape],
model_input: Optional[List] = None,
architecture: str = None,
framework: Framework = None,
engine: Engine = None,
version: ModelVersion = None,
convert=True,
profile=True,
):
"""Upload a model to ModelDB.
This function will upload the given model into the database with some variation. It may optionally generate a
branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be
scheduled for each generated model, in order to gain profiling results for model selection strategies.
In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine`
and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced
from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced.
Arguments:
origin_model: The uploaded model without optimization. When `no_generate` flag is set, this parameter should
be a str indicating model file path.
architecture (str): Model architecture name. Default to None.
framework (Framework): Framework name. Default to None.
version (ModelVersion): Model version. Default to None.
dataset (str): Model testing dataset.
metric (Dict[Metric,float]): Scoring metric and its corresponding score used for model evaluation
task (Task): Model task type.
inputs (Iterable[IOShape]): Model input tensors.
outputs (Iterable[IOShape]): Model output tensors.
model_input: specify sample model input data
TODO: specify more model conversion related params
engine (Engine): Model optimization engine. Default to `Engine.NONE`.
convert (bool): Flag for generation of model family. When set, `origin_model` should be a path to model saving
file. Default to `True`.
profile (bool): Flag for profiling uploaded (including converted) models. Default to `False`.
"""
from modelci.controller import job_executor
from modelci.controller.executor import Job
model_dir_list = list()
if not convert:
# type and existence check
assert isinstance(origin_model, str)
model_dir = Path(origin_model).absolute()
assert model_dir.exists(
), f'model weight does not exist at {origin_model}'
if all([architecture, task, framework, engine, version]):
# from explicit architecture, framework, engine and version
ext = model_dir.suffix
path = generate_path(architecture, task, framework, engine,
version).with_suffix(ext)
# if already in the destination folder
if path == model_dir:
pass
# create destination folder
else:
if ext:
path.parent.mkdir(parents=True, exist_ok=True)
else:
path.mkdir(parents=True, exist_ok=True)
# copy to cached folder
subprocess.call(['cp', model_dir, path])
else: # from implicit extracted from path, check validity of the path later at registration
path = model_dir
model_dir_list.append(path)
else:
# TODO: generate from path name
# generate model variant
model_dir_list.extend(
_generate_model_family(origin_model,
architecture,
task,
framework,
filename=str(version),
inputs=inputs,
outputs=outputs,
model_input=model_input))
# register
for model_dir in model_dir_list:
parse_result = parse_path(model_dir)
architecture = parse_result['architecture']
task = parse_result['task']
framework = parse_result['framework']
engine = parse_result['engine']
version = parse_result['version']
filename = parse_result['filename']
with open(str(model_dir), 'rb') as f:
model = ModelBO(name=architecture,
task=task,
framework=framework,
engine=engine,
version=version,
dataset=dataset,
metric=metric,
inputs=inputs,
outputs=outputs,
weight=Weight(f, filename=filename))
ModelService.post_model(model)
# TODO refresh
model = ModelService.get_models(name=architecture,
task=task,
framework=framework,
engine=engine,
version=version)[0]
# profile registered model
if profile and engine != Engine.PYTORCH:
file = tf.keras.utils.get_file(
"grace_hopper.jpg",
"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg"
)
test_img_bytes = cv2.imread(file)
kwargs = {
'repeat_data': test_img_bytes,
'batch_size': 32,
'batch_num': 100,
'asynchronous': False,
'model_info': model,
}
if engine == Engine.TORCHSCRIPT:
client = CVTorchClient(**kwargs)
elif engine == Engine.TFS:
client = CVTFSClient(**kwargs)
elif engine == Engine.ONNX:
client = CVONNXClient(**kwargs)
elif engine == Engine.TRT:
client = CVTRTClient(**kwargs)
else:
raise ValueError(f'No such serving engine: {engine}')
job_cuda = Job(client=client, device='cuda:0', model_info=model)
# job_cpu = Job(client=client, device='cpu', model_info=model)
job_executor.submit(job_cuda)
def test_register_static_profiling_result():
model = ModelService.get_models('ResNet50')[0]
spr = StaticProfileResultBO(5000, 200000, 200000, 10000, 10000, 10000)
assert ModelService.register_static_profiling_result(model.id, spr)
def update_finetune_model_as_new(id: str,
updated_layer: Structure,
dry_run: bool = False): # noqa
"""
Temporary function for finetune CV models. The function's functionality is overlapped with
`update_model_structure_as_new`. Please use the `update_model_structure_as_new` in next release.
Examples:
Fine-tune the model by modify the layer with name 'fc' (last layer). The layer
has a changed argument out_features = 10. op_='M' indicates the operation to this layer ('fc')
is 'Modify'. There is no changes in layer connections.
Therefore, the structure change summary is
[M] fc: (...) out_features=10
>>> from collections import OrderedDict
>>> structure_data = {
... 'layer': OrderedDict({'fc': {'out_features': 10, 'op_': 'M', 'type_': 'torch.nn.Linear'}})
... }
>>> update_finetune_model_as_new(id=..., updated_layer=Structure.parse_obj(structure_data))
Args:
id (str): ID of the model to be updated.
updated_layer (Structure): Contains layers to be fine-tuned.
dry_run (bool): Test run for verify if the provided parameter (i.e. model specified in `id`
and updated layers) is valid.
Returns:
"""
if len(updated_layer.layer.items()) == 0:
return True
model = ModelService.get_model_by_id(id)
if model.engine != Engine.PYTORCH:
raise ValueError(f'model {id} is not supported for editing. '
f'Currently only support model with engine=PYTORCH')
# download model as local cache
cache_path = get_remote_model_weight(model=model)
net = torch.load(cache_path)
for layer_name, layer_param in updated_layer.layer.items():
layer_op = getattr(layer_param, 'op_')
# update layer
if layer_op == Operation.MODIFY:
# check if the layer name exists
# TODO check if layer path exists eg."layer1.0.conv1"
if not hasattr(net, layer_name):
raise ModelStructureError(
f'Structure layer name `{layer_name}` not found in model {id}.'
)
net_layer = getattr(net, layer_name)
# check if the provided type matches the original type
layer_type = type(net_layer)
layer_type_provided = eval(layer_param.type_.value) # nosec
if layer_type is not layer_type_provided:
raise ModelStructureError(
f'Expect `{layer_name}.type_` to be {layer_type}, '
f'but got {layer_type_provided}')
# get layer parameters
layer_param_old = layer_param.parse_layer_obj(net_layer)
layer_param_data = layer_param_old.dict(exclude_none=True,
exclude={'type_', 'op_'})
layer_param_update_data = layer_param.dict(
exclude_none=True, exclude={'type_', 'op_'})
# replace 'null' with None. See reason :class:`ModelLayer`.
for k, v in layer_param_update_data.items():
if v == 'null':
layer_param_update_data[k] = None
# update the layer parameters
layer_param_data.update(layer_param_update_data)
layer = layer_type(**layer_param_data)
setattr(net, layer_name, layer)
else:
# if layer_op is Operation.ADD,
# 1. check if the layer name not exists
# 2. add a layer
# 3. change the `forward` function according to the connections
# if layer_op is Operation.DELETE,
# 1. check if the layer exists
# 2. delete the layer
# 3. change the `forward` function
raise ValueError(
'Operation not permitted. Please use `update_model_structure_as_new`.'
)
input_tensors = list()
bs = 1
for input_ in model.inputs:
input_tensor = torch.rand(bs, *input_.shape[1:]).type(
model_data_type_to_torch(input_.dtype))
input_tensors.append(input_tensor)
# parse output tensors
output_shapes = list()
output_tensors = net(*input_tensors)
if not isinstance(output_tensors, (list, tuple)):
output_tensors = (output_tensors, )
for output_tensor in output_tensors:
output_shape = IOShape(shape=[bs, *output_tensor.shape[1:]],
dtype=type_to_data_type(output_tensor.dtype))
output_shapes.append(output_shape)
if not dry_run:
# TODO return validation result for dry_run mode
# TODO apply Semantic Versioning https://semver.org/
# TODO reslove duplicate model version problem in a more efficient way
version = ModelVersion(model.version.ver + 1)
previous_models = ModelService.get_models(
architecture=model.architecture,
task=model.task,
framework=model.framework,
engine=Engine.NONE)
if len(previous_models):
last_version = max(previous_models,
key=lambda k: k.version.ver).version.ver
version = ModelVersion(last_version + 1)
saved_path = generate_path_plain(architecture=model.architecture,
task=model.task,
framework=model.framework,
engine=Engine.NONE,
version=version)
saved_path.parent.mkdir(parents=True, exist_ok=True)
torch.save(model, saved_path.with_suffix('.pt'))
mlmodelin = MLModel(dataset='',
metric={key: 0
for key in model.metric.keys()},
task=model.task,
inputs=model.inputs,
outputs=output_shapes,
architecture=model.name,
framework=model.framework,
engine=Engine.NONE,
model_status=[ModelStatus.DRAFT],
parent_model_id=model.id,
version=version,
weight=saved_path)
register_model(mlmodelin, convert=False, profile=False)
model_bo = ModelService.get_models(architecture=model.architecture,
task=model.task,
framework=model.framework,
engine=Engine.NONE,
version=version)[0]
return {'id': model_bo.id}
def test_get_model_by_id():
model_bo = ModelService.get_models('ResNet50')[0]
model = ModelService.get_model_by_id(model_bo.id)
# check model id
assert model.id == model_bo.id