def register_model_from_yaml(file_path: Union[Path, str]):
# check if file exist
file_path = Path(file_path)
assert file_path.exists(
), f'Model definition file at {str(file_path)} does not exist'
# read yaml
with open(file_path) as f:
model_config = yaml.safe_load(f)
model_yaml = MLModelFromYaml.parse_obj(model_config)
# copy model weight to cache directory
model_in_saved_path = model_yaml.saved_path
if model_in_saved_path != model_yaml.weight:
copy2(model_yaml.weight, model_in_saved_path)
# zip weight folder
if model_yaml.engine == Engine.TFS:
weight_dir = model_yaml.weight
make_archive(weight_dir.with_suffix('.zip'), 'zip', weight_dir)
model_data = model_yaml.dict(exclude_none=True,
exclude={'convert', 'profile'})
model = MLModel.parse_obj(model_data)
register_model(model,
convert=model_yaml.convert,
profile=model_yaml.profile)
def detail(model_id: str = typer.Argument(..., help='Model ID')):
"""Show a single model."""
with requests.get(
f'http://{SERVER_HOST}:{SERVER_PORT}/api/v1/model/{model_id}'
) as r:
data = r.json()
model_detailed_view(MLModel.parse_obj(data))
def get_by_id(id: str) -> MLModel:
"""Get a MLModel object by its ID.
"""
model_data = _collection.find_one(filter={'_id': ObjectId(id)})
if model_data is not None:
return MLModel.parse_obj(model_data)
else:
raise ServiceException(f'Model with id={id} does not exist.')
def save(model_in: MLModel):
"""Register a model into ModelDB and GridFS. `model.id` should be set as `None`, otherwise, the function will
raise a `ValueError`.
Args:
model_in (MLModelIn): model object to be registered
Return:
MLModel: Saved ML model object.
Raises:
BadRequestValueException: If `model.id` is not None.
ServiceException: If model has exists with the same primary keys (name, framework, engine and version).
"""
if _collection.count_documents(
filter=model_in.dict(
use_enum_values=True,
include={'architecture', 'framework', 'engine', 'version', 'task', 'dataset'}
),
limit=1
):
raise ServiceException(
f'Model with primary keys architecture={model_in.architecture}, '
f'framework={model_in.framework}, engine={model_in.engine}, version={model_in.version},'
f'task={model_in.task}, and dataset={model_in.dataset} has exists.'
)
# TODO: update weight ID in the MLModelIn
weight_id = _fs.put(bytes(model_in.weight), filename=model_in.weight.filename)
model = MLModel(**model_in.dict(exclude={'weight'}), weight=weight_id)
model.id = _collection.insert_one(model.dict(exclude_none=True, by_alias=True, use_enum_values=True)).inserted_id
return model
def _generate_model_family(
model: MLModel,
max_batch_size: int = -1
):
net = load(model.saved_path)
build_saved_dir_from_engine = partial(
generate_path_plain,
**model.dict(include={'architecture', 'framework', 'task', 'version'}),
)
inputs = model.inputs
outputs = model.outputs
model_input = model.model_input
generated_dir_list = list()
torchscript_dir = build_saved_dir_from_engine(engine=Engine.TORCHSCRIPT)
tfs_dir = build_saved_dir_from_engine(engine=Engine.TFS)
onnx_dir = build_saved_dir_from_engine(engine=Engine.ONNX)
trt_dir = build_saved_dir_from_engine(engine=Engine.TRT)
if isinstance(net, torch.nn.Module):
# to TorchScript
if converter.convert(net, 'pytorch', 'torchscript', save_path=torchscript_dir):
generated_dir_list.append(torchscript_dir.with_suffix('.zip'))
# to ONNX, TODO(lym): batch cache, input shape, opset version
if converter.convert(net, 'pytorch', 'onnx', save_path=onnx_dir, inputs=inputs, outputs=outputs, model_input=model_input, optimize=False):
generated_dir_list.append(onnx_dir.with_suffix('.onnx'))
# to TRT
# TRTConverter.from_onnx(
# onnx_path=onnx_dir.with_suffix('.onnx'), save_path=trt_dir, inputs=inputs, outputs=outputs
# )
elif isinstance(net, tf.keras.Model):
# to TFS
converter.convert(net, 'tensorflow', 'tfs', save_path=tfs_dir)
generated_dir_list.append(tfs_dir.with_suffix('.zip'))
# to TRT
converter.convert(net, 'tfs', 'trt', tf_path=tfs_dir, save_path=trt_dir, inputs=inputs, outputs=outputs, max_batch_size=32)
generated_dir_list.append(trt_dir.with_suffix('.zip'))
return generated_dir_list
def list_models(
architecture: Optional[str] = typer.Option(None,
'-n',
'--name',
help='Model architecture name'),
framework: Optional[Framework] = typer.Option(None,
'-fw',
'--framework',
case_sensitive=False,
help='Framework'),
engine: Optional[Engine] = typer.Option(None,
'-e',
'--engine',
case_sensitive=False,
help='Serving engine'),
version: Optional[int] = typer.Option(None,
'-v',
'--version',
help='Version'),
list_all: Optional[bool] = typer.Option(
False,
'-a',
'--all',
is_flag=True,
help=
'Display queried models. otherwise, only partial result will be shown.'
),
):
"""Show a table that lists all models published in MLModelCI"""
payload = remove_dict_null({
'architecture': architecture,
'framework': framework,
'engine': engine,
'version': version
})
with requests.get(f'{app_settings.api_v1_prefix}/model',
params=payload) as r:
model_list = r.json()
model_view([MLModel.parse_obj(model) for model in model_list],
list_all=list_all)
def generate_model_family(model: MLModel, max_batch_size: int = -1):
model_weight_path = model.saved_path
if not Path(model.saved_path).exists():
(filepath, filename) = os.path.split(model.saved_path)
os.makedirs(filepath)
with open(model.saved_path, 'wb') as f:
f.write(model.weight.__bytes__())
net = load(model_weight_path)
build_saved_dir_from_engine = partial(
generate_path_plain,
**model.dict(include={'architecture', 'framework', 'task', 'version'}),
)
inputs = model.inputs
outputs = model.outputs
model_input = model.model_input
generated_dir_list = list()
torchscript_dir = build_saved_dir_from_engine(engine=Engine.TORCHSCRIPT)
tfs_dir = build_saved_dir_from_engine(engine=Engine.TFS)
onnx_dir = build_saved_dir_from_engine(engine=Engine.ONNX)
trt_dir = build_saved_dir_from_engine(engine=Engine.TRT)
if isinstance(net, torch.nn.Module):
_torchfamily(net, False, torchscript_dir, onnx_dir, generated_dir_list,
inputs, outputs, model_input)
elif isinstance(net, tf.keras.Model):
_tffamily(net, tfs_dir, generated_dir_list, trt_dir, inputs, outputs)
elif isinstance(net, xgb.XGBModel):
_xgbfamily(net, inputs, onnx_dir, generated_dir_list, torchscript_dir,
outputs, model_input)
elif isinstance(net, lgb.LGBMModel):
_lgbfamily(net, inputs, onnx_dir, generated_dir_list, torchscript_dir,
outputs, model_input)
elif isinstance(net, skl.base.BaseEstimator):
_sklfamily(net, inputs, onnx_dir, generated_dir_list, torchscript_dir,
outputs, model_input)
return generated_dir_list
async def publish_model(
model: BaseMLModel = Depends(BaseMLModel.as_form),
files:
List[UploadFile] = File(
[],
description=
'This field can be set with empty value. In such settings, the publish is a dry run to'
'validate the `ml_model_in_form` field. You are recommend to try a dry run to find input'
'errors before you send the wight file(s) to the server.'),
convert: bool = True,
profile: bool = False):
"""Publish model to the model hub. The model weight file(s) as well as its meta information (e.g.
architecture, framework, and serving engine) will be stored into the model hub.
The publish API will also automatically convert the published model into other deployable format such as
TorchScript and ONNX. After successfully converted, original model and its generated models will be profiled
on the underlying devices in the clusters, and collects, aggregates, and processes running model performance.
Args:
model (MLModel): Model meta information.
files (List[UploadFile]): A list of model weight files. The files are organized accordingly. Their file name
contains relative path to their common parent directory.
If the files is empty value, a dry-run to this API is conducted for parameter checks. No information
will be saved into model hub in this case.
convert (bool): Flag for auto configuration.
profile (bool): Flag for auto profiling.
Returns:
A message response, with IDs of all published model. The format of the return is:
```
{
"data": {"id": ["603e6a1f5a62b08bc0a2a7f2", "603e6a383b00cbe9bfee7277"]},
"status": true
}
```
Specially, if the dry-run test passed, it will return a status True:
```
{
"status": true
}
```
"""
# save the posted files as local cache
loop = asyncio.get_event_loop()
saved_path = model.saved_path
if len(files) == 0:
# conduct dry run for parameter check only.
return {'status': True}
if len(files) == 1:
file = files[0]
suffix = Path(file.filename).suffix
try:
# create directory
if len(suffix) == 0:
error = ErrorWrapper(ValueError(
f'Expect a suffix for file {file.filename}, got None.'),
loc='files[0]')
raise RequestValidationError([error])
saved_path = saved_path.with_suffix(suffix)
saved_path.parent.mkdir(exist_ok=True, parents=True)
# save file
await file.seek(0)
with open(saved_path, 'wb') as buffer:
await loop.run_in_executor(None, shutil.copyfileobj, file.file,
buffer)
finally:
await file.close()
else:
raise NotImplementedError(
'`publish_model` not implemented for multiple files upload.')
# zip the files
model = MLModel(**model.dict(), weight=saved_path)
models = register_model(model=model, convert=convert, profile=profile)
return {
'data': {
'id': [str(model.id) for model in models],
},
'status': True
}
def convert(
id: str = typer.Option(None, '-i', '--id', help='ID of model.'),
yaml_file: Optional[Path] = typer.
Option(
None,
'-f',
'--yaml-file',
exists=True,
file_okay=True,
help=
'Path to configuration YAML file. You should either set the `yaml_file` field or fields '
'(`FILE_OR_DIR`, `--name`, `--framework`, `--engine`, `--version`, `--task`, `--dataset`,'
'`--metric`, `--input`, `--output`).'),
register: bool = typer.Option(
False,
'-r',
'--register',
is_flag=True,
help='register the converted models to modelhub, default false')):
model = None
if id is None and yaml_file is None:
typer.echo(
"WARNING: Please assign a way to find the target model! details refer to --help"
)
raise RuntimeError
if id is not None and yaml_file is not None:
typer.echo("WARNING: Do not use -id and -path at the same time!")
raise RuntimeError
elif id is not None and yaml_file is None:
if ModelDB.exists_by_id(id):
model = ModelDB.get_by_id(id)
else:
typer.echo(f"model id: {id} does not exist in modelhub")
elif id is None and yaml_file is not None:
# get MLModel from yaml file
with open(yaml_file) as f:
model_config = yaml.safe_load(f)
model_yaml = MLModelFromYaml.parse_obj(model_config)
model_in_saved_path = model_yaml.saved_path
if model_in_saved_path != model_yaml.weight:
copy2(model_yaml.weight, model_in_saved_path)
if model_yaml.engine == Engine.TFS:
weight_dir = model_yaml.weight
make_archive(weight_dir.with_suffix('.zip'), 'zip', weight_dir)
model_data = model_yaml.dict(exclude_none=True,
exclude={'convert', 'profile'})
model = MLModel.parse_obj(model_data)
# auto execute all possible convert and return a list of save paths of every converted model
generated_dir_list = generate_model_family(model)
typer.echo(f"Converted models are save in: {generated_dir_list}")
if register:
model_data = model.dict(
exclude={'weight', 'id', 'model_status', 'engine'})
for model_dir in generated_dir_list:
parse_result = parse_path_plain(model_dir)
engine = parse_result['engine']
model_cvt = MLModel(**model_data,
weight=model_dir,
engine=engine,
model_status=[ModelStatus.CONVERTED])
ModelDB.save(model_cvt)
typer.echo(
f"converted {engine} are successfully registered in Modelhub")
def update(
model_id: str = typer.Argument(..., help='Model ID'),
architecture: Optional[str] = typer.Option(None,
'-n',
'--name',
help='Architecture'),
framework: Optional[Framework] = typer.Option(None,
'-fw',
'--framework',
help='Framework'),
engine: Optional[Engine] = typer.Option(None,
'-e',
'--engine',
help='Engine'),
version: Optional[int] = typer.Option(None,
'-v',
'--version',
min=1,
help='Version number'),
task: Optional[Task] = typer.Option(None, '-t', '--task', help='Task'),
dataset: Optional[str] = typer.Option(None,
'-d',
'--dataset',
help='Dataset name'),
metric: Optional[Dict[Metric, float]] = typer.
Option(
None,
help='Metrics in the form of mapping JSON string. The map type is '
'`Dict[types.models.mlmodel.Metric, float]`. An example is \'{"acc": 0.76}.\'',
),
inputs: Optional[List[IOShape]] = typer.
Option(
[],
'-i',
'--input',
help=
'List of shape definitions for input tensors. An example of one shape definition is '
'\'{"name": "input", "shape": [-1, 3, 224, 224], "dtype": "TYPE_FP32", "format": "FORMAT_NCHW"}\'',
),
outputs: Optional[List[IOShape]] = typer.
Option(
[],
'-o',
'--output',
help=
'List of shape definitions for output tensors. An example of one shape definition is '
'\'{"name": "output", "shape": [-1, 1000], "dtype": "TYPE_FP32"}\'',
)):
model = ModelUpdateSchema(
architecture=architecture,
framework=framework,
engine=engine,
version=version, # noqa
dataset=dataset,
metric=metric,
task=task,
inputs=inputs,
outputs=outputs)
with requests.patch(f'{app_settings.api_v1_prefix}/model/{model_id}',
data=model.json(exclude_defaults=True)) as r:
data = r.json()
model_detailed_view(MLModel.parse_obj(data))
def detail(model_id: str = typer.Argument(..., help='Model ID')):
"""Show a single model."""
with requests.get(f'{app_settings.api_v1_prefix}/model/{model_id}') as r:
data = r.json()
model_detailed_view(MLModel.parse_obj(data))
def update_finetune_model_as_new(id: str,
updated_layer: Structure,
dry_run: bool = False): # noqa
"""
Temporary function for finetune CV models. The function's functionality is overlapped with
`update_model_structure_as_new`. Please use the `update_model_structure_as_new` in next release.
Examples:
Fine-tune the model by modify the layer with name 'fc' (last layer). The layer
has a changed argument out_features = 10. op_='M' indicates the operation to this layer ('fc')
is 'Modify'. There is no changes in layer connections.
Therefore, the structure change summary is
[M] fc: (...) out_features=10
>>> from collections import OrderedDict
>>> structure_data = {
... 'layer': OrderedDict({'fc': {'out_features': 10, 'op_': 'M', 'type_': 'torch.nn.Linear'}})
... }
>>> update_finetune_model_as_new(id=..., updated_layer=Structure.parse_obj(structure_data))
Args:
id (str): ID of the model to be updated.
updated_layer (Structure): Contains layers to be fine-tuned.
dry_run (bool): Test run for verify if the provided parameter (i.e. model specified in `id`
and updated layers) is valid.
Returns:
"""
if len(updated_layer.layer.items()) == 0:
return True
model = ModelService.get_model_by_id(id)
if model.engine != Engine.PYTORCH:
raise ValueError(f'model {id} is not supported for editing. '
f'Currently only support model with engine=PYTORCH')
# download model as local cache
cache_path = get_remote_model_weight(model=model)
net = torch.load(cache_path)
for layer_name, layer_param in updated_layer.layer.items():
layer_op = getattr(layer_param, 'op_')
# update layer
if layer_op == Operation.MODIFY:
# check if the layer name exists
# TODO check if layer path exists eg."layer1.0.conv1"
if not hasattr(net, layer_name):
raise ModelStructureError(
f'Structure layer name `{layer_name}` not found in model {id}.'
)
net_layer = getattr(net, layer_name)
# check if the provided type matches the original type
layer_type = type(net_layer)
layer_type_provided = eval(layer_param.type_.value) # nosec
if layer_type is not layer_type_provided:
raise ModelStructureError(
f'Expect `{layer_name}.type_` to be {layer_type}, '
f'but got {layer_type_provided}')
# get layer parameters
layer_param_old = layer_param.parse_layer_obj(net_layer)
layer_param_data = layer_param_old.dict(exclude_none=True,
exclude={'type_', 'op_'})
layer_param_update_data = layer_param.dict(
exclude_none=True, exclude={'type_', 'op_'})
# replace 'null' with None. See reason :class:`ModelLayer`.
for k, v in layer_param_update_data.items():
if v == 'null':
layer_param_update_data[k] = None
# update the layer parameters
layer_param_data.update(layer_param_update_data)
layer = layer_type(**layer_param_data)
setattr(net, layer_name, layer)
else:
# if layer_op is Operation.ADD,
# 1. check if the layer name not exists
# 2. add a layer
# 3. change the `forward` function according to the connections
# if layer_op is Operation.DELETE,
# 1. check if the layer exists
# 2. delete the layer
# 3. change the `forward` function
raise ValueError(
'Operation not permitted. Please use `update_model_structure_as_new`.'
)
input_tensors = list()
bs = 1
for input_ in model.inputs:
input_tensor = torch.rand(bs, *input_.shape[1:]).type(
model_data_type_to_torch(input_.dtype))
input_tensors.append(input_tensor)
# parse output tensors
output_shapes = list()
output_tensors = net(*input_tensors)
if not isinstance(output_tensors, (list, tuple)):
output_tensors = (output_tensors, )
for output_tensor in output_tensors:
output_shape = IOShape(shape=[bs, *output_tensor.shape[1:]],
dtype=type_to_data_type(output_tensor.dtype))
output_shapes.append(output_shape)
if not dry_run:
# TODO return validation result for dry_run mode
# TODO apply Semantic Versioning https://semver.org/
# TODO reslove duplicate model version problem in a more efficient way
version = ModelVersion(model.version.ver + 1)
previous_models = ModelService.get_models(
architecture=model.architecture,
task=model.task,
framework=model.framework,
engine=Engine.NONE)
if len(previous_models):
last_version = max(previous_models,
key=lambda k: k.version.ver).version.ver
version = ModelVersion(last_version + 1)
saved_path = generate_path_plain(architecture=model.architecture,
task=model.task,
framework=model.framework,
engine=Engine.NONE,
version=version)
saved_path.parent.mkdir(parents=True, exist_ok=True)
torch.save(model, saved_path.with_suffix('.pt'))
mlmodelin = MLModel(dataset='',
metric={key: 0
for key in model.metric.keys()},
task=model.task,
inputs=model.inputs,
outputs=output_shapes,
architecture=model.name,
framework=model.framework,
engine=Engine.NONE,
model_status=[ModelStatus.DRAFT],
parent_model_id=model.id,
version=version,
weight=saved_path)
register_model(mlmodelin, convert=False, profile=False)
model_bo = ModelService.get_models(architecture=model.architecture,
task=model.task,
framework=model.framework,
engine=Engine.NONE,
version=version)[0]
return {'id': model_bo.id}
def models(name, framework, engine, version, list_all, quiet):
payload = remove_dict_null({'name': name, 'framework': framework, 'engine': engine, 'version': version})
with requests.get(f'{app_settings.api_v1_prefix}/model', params=payload) as r:
model_list = r.json()
model_view([MLModel.parse_obj(model) for model in model_list], list_all=list_all, quiet=quiet)
def register_model(
model: MLModel,
convert: bool = True,
profile: bool = True,
) -> List[MLModel]:
"""Upload a model to ModelDB.
This function will upload the given model into the database with some variation. It may optionally generate a
branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be
scheduled for each generated model, in order to gain profiling results for model selection strategies.
In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine`
and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced
from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced.
TODO:
This function has a super comprehensive logic, need to be simplified.
Arguments:
model: Required inputs for register a model. All information is wrapped in such model.
convert (bool): Flag for generation of model family. Default to True.
profile (bool): Flag for profiling uploaded (including converted) models. Default to True.
"""
models = list()
model_dir_list = list()
model.model_status = [ModelStatus.PUBLISHED]
models.append(save(model))
# generate model family
if convert:
model_dir_list.extend(converter.generate_model_family(model))
# register
model_data = model.dict(exclude={'weight', 'id', 'model_status', 'engine'})
for model_dir in model_dir_list:
parse_result = parse_path_plain(model_dir)
engine = parse_result['engine']
model_cvt = MLModel(**model_data,
weight=model_dir,
engine=engine,
model_status=[ModelStatus.CONVERTED])
models.append(save(model_cvt))
# profile registered model
if profile:
from modelci.controller import job_executor
from modelci.controller.executor import Job
file = tf.keras.utils.get_file(
"grace_hopper.jpg",
"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg"
)
test_img_bytes = cv2.imread(file)
kwargs = {
'repeat_data': test_img_bytes,
'batch_size': 32,
'batch_num': 100,
'asynchronous': False,
}
for model in models:
model.model_status = [ModelStatus.PROFILING]
ModelService.update_model(model)
kwargs['model_info'] = model
engine = model.engine
if engine == Engine.TORCHSCRIPT:
client = CVTorchClient(**kwargs)
elif engine == Engine.TFS:
client = CVTFSClient(**kwargs)
elif engine == Engine.ONNX:
client = CVONNXClient(**kwargs)
elif engine == Engine.TRT:
client = CVTRTClient(**kwargs)
else:
raise ValueError(f'No such serving engine: {engine}')
job_cuda = Job(client=client, device='cuda:0', model_info=model)
# job_cpu = Job(client=client, device='cpu', model_info=model)
job_executor.submit(job_cuda)
# job_executor.submit(job_cpu)
return models
