def __auto_select_client(self):
# according to the serving engine, select the right testing client.
# TODO: replace the input None data in each client with self-generated data.
serving_engine = self.model_info.engine
if serving_engine == Framework.NONE:
raise Exception('please choose a serving engine for the model')
# TODO How can we deploy to all available platforms if we don't know the engine?
elif serving_engine == Framework.TFS:
return CVTFSClient(None,
batch_num=DEFAULT_BATCH_NUM,
asynchronous=False)
elif serving_engine == Framework.TORCHSCRIPT:
return CVTorchClient(None,
batch_num=DEFAULT_BATCH_NUM,
asynchronous=False)
elif serving_engine == Framework.ONNX:
return CVONNXClient(None,
batch_num=DEFAULT_BATCH_NUM,
asynchronous=False)
elif serving_engine == Framework.TRT:
return CVTRTClient(None,
batch_num=DEFAULT_BATCH_NUM,
asynchronous=False)
elif serving_engine == Framework.TVM:
raise NotImplementedError
elif serving_engine == Framework.CUSTOMIZED:
raise Exception(
'please pass a custom client to the Profiler.__init__.')
else:
return None
def register_model(origin_model,
dataset: str,
metric: Dict[Metric, float],
task: Task,
inputs: List[IOShape],
outputs: List[IOShape],
model_input: Optional[List] = None,
architecture: str = None,
framework: Framework = None,
engine: Engine = None,
version: ModelVersion = None,
parent_model_id: Optional[str] = None,
convert: bool = True,
profile: bool = True,
model_status: List[ModelStatus] = None):
"""Upload a model to ModelDB.
This function will upload the given model into the database with some variation. It may optionally generate a
branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be
scheduled for each generated model, in order to gain profiling results for model selection strategies.
In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine`
and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced
from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced.
TODO:
This function has a super comprehensive logic, need to be simplified.
Arguments:
origin_model: The uploaded model without optimization. When `no_generate` flag is set, this parameter should
be a str indicating model file path.
architecture (str): Model architecture name. Default to None.
framework (Framework): Framework name. Default to None.
version (ModelVersion): Model version. Default to None.
dataset (str): Model testing dataset.
metric (Dict[Metric,float]): Scoring metric and its corresponding score used for model evaluation
task (Task): Model task type.
inputs (Iterable[IOShape]): Model input tensors.
outputs (Iterable[IOShape]): Model output tensors.
model_input: specify sample model input data
TODO: specify more model conversion related params
engine (Engine): Model optimization engine. Default to `Engine.NONE`.
parent_model_id (Optional[str]): the parent model id of current model if this model is derived from a pre-existing one
model_status (List[ModelStatus]): Indicate the status of current model in its lifecycle
convert (bool): Flag for generation of model family. When set, `origin_model` should be a path to model saving
file. Default to `True`.
profile (bool): Flag for profiling uploaded (including converted) models. Default to `False`.
"""
from modelci.controller import job_executor
from modelci.controller.executor import Job
model_dir_list = list()
# type and existence check
if isinstance(origin_model, str):
model_dir = Path(origin_model).absolute()
assert model_dir.exists(
), f'model weight does not exist at {origin_model}'
if all([architecture, task, framework, engine, version]):
# from explicit architecture, framework, engine and version
ext = model_dir.suffix
path = generate_path(architecture, task, framework, engine,
version).with_suffix(ext)
# if already in the destination folder
if path == model_dir:
pass
# create destination folder
else:
if ext:
path.parent.mkdir(parents=True, exist_ok=True)
else:
path.mkdir(parents=True, exist_ok=True)
# copy to cached folder
subprocess.call(['cp', model_dir, path])
else: # from implicit extracted from path, check validity of the path later at registration
path = model_dir
model_dir_list.append(path)
elif framework == Framework.PYTORCH and engine in [
Engine.PYTORCH, Engine.NONE
]:
# save original pytorch model
pytorch_dir = generate_path(
task=task,
model_name=architecture,
framework=framework,
engine=engine,
version=str(version),
)
pytorch_dir.parent.mkdir(parents=True, exist_ok=True)
save_path_with_ext = pytorch_dir.with_suffix('.pth')
torch.save(origin_model, str(save_path_with_ext))
model_dir_list.append(pytorch_dir.with_suffix('.pth'))
if convert:
# TODO: generate from path name
# generate model variant
model_dir_list.extend(
_generate_model_family(origin_model,
architecture,
task,
framework,
filename=str(version),
inputs=inputs,
outputs=outputs,
model_input=model_input))
# register
for model_dir in model_dir_list:
parse_result = parse_path(model_dir)
architecture = parse_result['architecture']
task = parse_result['task']
framework = parse_result['framework']
engine = parse_result['engine']
version = parse_result['version']
filename = parse_result['filename']
if model_status is not None:
model_bo_status = model_status
elif engine == Engine.PYTORCH:
model_bo_status = [ModelStatus.PUBLISHED]
else:
model_bo_status = [ModelStatus.CONVERTED]
with open(str(model_dir), 'rb') as f:
model = ModelBO(name=architecture,
task=task,
framework=framework,
engine=engine,
version=version,
dataset=dataset,
metric=metric,
parent_model_id=parent_model_id,
inputs=inputs,
outputs=outputs,
model_status=model_bo_status,
weight=Weight(f, filename=filename))
ModelService.post_model(model)
# TODO refresh
model = ModelService.get_models(name=architecture,
task=task,
framework=framework,
engine=engine,
version=version)[0]
if model.engine == Engine.PYTORCH or model.engine == Engine.TFS:
parent_model_id = model.id
# profile registered model
if profile and engine != Engine.PYTORCH:
file = tf.keras.utils.get_file(
"grace_hopper.jpg",
"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg"
)
test_img_bytes = cv2.imread(file)
kwargs = {
'repeat_data': test_img_bytes,
'batch_size': 32,
'batch_num': 100,
'asynchronous': False,
'model_info': model,
}
new_status = [
item for item in model.model_status
if item is not (ModelStatus.CONVERTED or ModelStatus.PUBLISHED)
]
new_status.append(ModelStatus.PROFILING)
model.model_status = new_status
ModelService.update_model(model)
if engine == Engine.TORCHSCRIPT:
client = CVTorchClient(**kwargs)
elif engine == Engine.TFS:
client = CVTFSClient(**kwargs)
elif engine == Engine.ONNX:
client = CVONNXClient(**kwargs)
elif engine == Engine.TRT:
client = CVTRTClient(**kwargs)
else:
raise ValueError(f'No such serving engine: {engine}')
job_cuda = Job(client=client, device='cuda:0', model_info=model)
# job_cpu = Job(client=client, device='cpu', model_info=model)
job_executor.submit(job_cuda)
Mask R-CNN registering, managing, serving and profiling code demo using ModelCI.
"""
from PIL import Image
from modelci.hub.client.tfs_client import CVTFSClient
from modelci.hub.manager import retrieve_model, register_model_from_yaml
from modelci.hub.profiler import Profiler
from modelci.types.bo import Engine, Framework
if __name__ == "__main__":
test_img = Image.open("path to the test data")
register_model_from_yaml(
"path to your yaml file") # register your model in the database
model_info = retrieve_model( # retrieve model information
architecture_name='MRCNN',
framework=Framework.TENSORFLOW,
engine=Engine.TFS)[0]
tfs_client = CVTFSClient(test_img,
batch_num=100,
batch_size=32,
asynchronous=True,
model_info=model_info)
profiler = Profiler(model_info=model_info,
server_name='tfs',
inspector=tfs_client)
profiler.diagnose(device='cuda:0') # profile batch size 32
def register_model(
origin_model,
dataset: str,
acc: float,
task: str,
inputs: List[IOShape],
outputs: List[IOShape],
architecture: str = None,
framework: Framework = None,
engine: Engine = None,
version: ModelVersion = None,
convert=True,
profile=True,
):
"""Upload a model to ModelDB.
This function will upload the given model into the database with some variation. It may optionally generate a
branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be
scheduled for each generated model, in order to gain profiling results for model selection strategies.
In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine`
and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced
from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced.
Arguments:
origin_model: The uploaded model without optimization. When `no_generate` flag is set, this parameter should
be a str indicating model file path.
architecture (str): Model architecture name. Default to None.
framework (Framework): Framework name. Default to None.
version (ModelVersion): Model version. Default to None.
dataset (str): Model testing dataset.
acc (float): Model accuracy on the testing dataset.
task (str): Model task type.
inputs (Iterable[IOShape]): Model input tensors.
outputs (Iterable[IOShape]): Model output tensors.
engine (Engine): Model optimization engine. Default to `Engine.NONE`.
convert (bool): Flag for generation of model family. When set, `origin_model` should be a path to model saving
file. Default to `True`.
profile (bool): Flag for profiling uploaded (including converted) models. Default to `False`.
"""
from modelci.controller import job_executor
from modelci.controller.executor import Job
model_dir_list = list()
if not convert:
# type and existence check
assert isinstance(origin_model, str)
model_dir = Path(origin_model).absolute()
assert model_dir.exists(
), f'model weight does not exist at {origin_model}'
if all([
architecture, framework, engine, version
]): # from explicit architecture, framework, engine and version
ext = model_dir.suffix
path = generate_path(architecture, framework, engine,
version).with_suffix(ext)
# if already in the destination folder
if path == model_dir:
pass
# create destination folder
else:
if ext:
path.parent.mkdir(parents=True, exist_ok=True)
else:
path.mkdir(parents=True, exist_ok=True)
# copy to cached folder
subprocess.call(['cp', model_dir, path])
else: # from implicit extracted from path, check validity of the path later at registration
path = model_dir
model_dir_list.append(path)
else:
# TODO: generate from path name
# generate model variant
model_dir_list.extend(
_generate_model_family(origin_model,
architecture,
framework,
filename=str(version),
inputs=inputs,
outputs=outputs))
# register
for model_dir in model_dir_list:
parse_result = parse_path(model_dir)
architecture = parse_result['architecture']
framework = parse_result['framework']
engine = parse_result['engine']
version = parse_result['version']
filename = parse_result['filename']
with open(str(model_dir), 'rb') as f:
model = ModelBO(name=architecture,
framework=framework,
engine=engine,
version=version,
dataset=dataset,
acc=acc,
task=task,
inputs=inputs,
outputs=outputs,
weight=Weight(f, filename=filename))
ModelService.post_model(model)
# TODO refresh
model = ModelService.get_models(name=architecture,
framework=framework,
engine=engine,
version=version)[0]
# profile registered model
if profile:
file = tf.keras.utils.get_file(
"grace_hopper.jpg",
"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg"
)
test_img_bytes = cv2.imread(file)
kwargs = {
'repeat_data': test_img_bytes,
'batch_size': 32,
'batch_num': 100,
'asynchronous': False,
'model_info': model,
}
if engine == Engine.TORCHSCRIPT:
client = CVTorchClient(**kwargs)
elif engine == Engine.TFS:
client = CVTFSClient(**kwargs)
elif engine == Engine.ONNX:
client = CVONNXClient(**kwargs)
elif engine == Engine.TRT:
client = CVTRTClient(**kwargs)
else:
raise ValueError(f'No such serving engine: {engine}')
job_cuda = Job(client=client, device='cuda:0', model_info=model)
# job_cpu = Job(client=client, device='cpu', model_info=model)
job_executor.submit(job_cuda)
def register_model(
model: MLModel,
convert: bool = True,
profile: bool = True,
) -> List[MLModel]:
"""Upload a model to ModelDB.
This function will upload the given model into the database with some variation. It may optionally generate a
branch of models (i.e. model family) with different optimization techniques. Besides, a benchmark will be
scheduled for each generated model, in order to gain profiling results for model selection strategies.
In the `no_generate` model(i.e. `no_generate` flag is set to be `True`), `architecture`, `framework`, `engine`
and `version` could be None. If any of the above arguments is `None`, all of them will be auto induced
from the origin_model path. An `ValueError` will be raised if the mata info cannot be induced.
TODO:
This function has a super comprehensive logic, need to be simplified.
Arguments:
model: Required inputs for register a model. All information is wrapped in such model.
convert (bool): Flag for generation of model family. Default to True.
profile (bool): Flag for profiling uploaded (including converted) models. Default to True.
"""
models = list()
model_dir_list = list()
model.model_status = [ModelStatus.PUBLISHED]
models.append(save(model))
# generate model family
if convert:
model_dir_list.extend(converter.generate_model_family(model))
# register
model_data = model.dict(exclude={'weight', 'id', 'model_status', 'engine'})
for model_dir in model_dir_list:
parse_result = parse_path_plain(model_dir)
engine = parse_result['engine']
model_cvt = MLModel(**model_data,
weight=model_dir,
engine=engine,
model_status=[ModelStatus.CONVERTED])
models.append(save(model_cvt))
# profile registered model
if profile:
from modelci.controller import job_executor
from modelci.controller.executor import Job
file = tf.keras.utils.get_file(
"grace_hopper.jpg",
"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg"
)
test_img_bytes = cv2.imread(file)
kwargs = {
'repeat_data': test_img_bytes,
'batch_size': 32,
'batch_num': 100,
'asynchronous': False,
}
for model in models:
model.model_status = [ModelStatus.PROFILING]
ModelService.update_model(model)
kwargs['model_info'] = model
engine = model.engine
if engine == Engine.TORCHSCRIPT:
client = CVTorchClient(**kwargs)
elif engine == Engine.TFS:
client = CVTFSClient(**kwargs)
elif engine == Engine.ONNX:
client = CVONNXClient(**kwargs)
elif engine == Engine.TRT:
client = CVTRTClient(**kwargs)
else:
raise ValueError(f'No such serving engine: {engine}')
job_cuda = Job(client=client, device='cuda:0', model_info=model)
# job_cpu = Job(client=client, device='cpu', model_info=model)
job_executor.submit(job_cuda)
# job_executor.submit(job_cpu)
return models
# for TensorRT Serving
test_img = Image.open(data_path)
# for TorchScript and ONNX
test_img_ndarray: np.ndarray = cv2.imread(data_path)
# input = torch.randn(1, 3, 224, 224)
# init clients for different serving platforms, you can custom a client by implementing the BaseModelInspector class.
model_bo = retrieve_model(architecture_name='ResNet50',
framework=Framework.PYTORCH,
engine=Engine.TORCHSCRIPT)[0]
tfs_client = CVTFSClient(
test_img_bytes,
batch_num=100,
batch_size=32,
asynchronous=False,
model_info=model_bo,
)
# trt_client = CVTRTClient(test_img, batch_num=100, batch_size=32, asynchronous=False)
# torch_client = CVTorchClient(test_img_ndarray, batch_num=100, batch_size=32, asynchronous=False)
# onnx_client = CVONNXClient(test_img_ndarray, batch_num=100, batch_size=32, asynchronous=False)
# model_path = '../resnet50_explicit_path.yml'
# register_model_from_yaml(model_path)
profiler = Profiler(model_info=model_bo,
server_name='tfs',
inspector=tfs_client)
profiler.diagnose(device='cuda:0')
# profiler.diagnose(batch_size=1) # you can use a new batch_size to overwrite the client's.
# profiler.diagnose_all_batches([1, 2, 4, 8, 16, 32]) # run all 1, 2, 4, 8, 16, 32 batch size