def __init__(self, app_name='mms', args=None):
"""Initialize mxnet model server application.
Parameters
----------
app_name : str
App name to initialize mms service.
args : List of str
Arguments for starting service. By default it is None
and commandline arguments will be used. It should follow
the format recognized by python argparse parse_args method:
https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser.parse_args.
An example for mms arguments:
['--models', 'resnet-18=path1', 'inception_v3=path2',
'--gen-api', 'java', '--port', '8080']
"""
# Initialize serving frontend and arg parser
try:
parser = ArgParser.mms_parser()
self.args = parser.parse_args(
args) if args else parser.parse_args()
self.serving_frontend = ServingFrontend(app_name)
self.gpu = self.args.gpu
# Setup root logger handler and level.
log_file = self.args.log_file
log_level = self.args.log_level or "INFO"
log_rotation_time = self.args.log_rotation_time or "1 H"
_set_root_logger(log_file, log_level, log_rotation_time)
logger.info('Initialized model serving.')
except Exception as e:
print('Failed to initialize model serving: ' + str(e))
exit(1)
def __init__(self, app_name='mms'):
# Initialize serving frontend and arg parser
try:
self.args = ArgParser.parse_args()
self.serving_frontend = ServingFrontend(app_name)
logger.info('Initialized model serving.')
except Exception as e:
logger.error('Failed to initialize model serving: ' + str(e))
exit(1)
def __init__(self, app_name='mms', args=None):
"""Initialize mxnet model server application.
Parameters
----------
app_name : str
App name to initialize mms service.
args : List of str
Arguments for starting service. By default it is None
and commandline arguments will be used. It should follow
the format recognized by python argparse parse_args method:
https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser.parse_args.
An example for mms arguments:
['--models', 'resnet-18=path1', 'inception_v3=path2',
'--gen-api', 'java', '--port', '8080']
"""
# Initialize serving frontend and arg parser
try:
parser = ArgParser.mms_parser()
self.args = parser.parse_args(args) if args else parser.parse_args()
self.serving_frontend = ServingFrontend(app_name)
self.gpu = self.args.gpu
# Setup root logger handler and level.
log_file = self.args.log_file
log_level = self.args.log_level or "INFO"
log_rotation_time = self.args.log_rotation_time or "1 H"
_set_root_logger(log_file, log_level, log_rotation_time)
logger.info('Initialized model serving.')
except Exception as e:
print ('Failed to initialize model serving: ' + str(e))
exit(1)
class TestServingFrontend(unittest.TestCase):
def setUp(self):
self.test_frontend = ServingFrontend('test')
def test_register_module(self):
# Mock
ret = [MXNetVisionService]
self.test_frontend.service_manager.parse_modelservices_from_module = mock.Mock(
return_value=ret)
self.test_frontend.service_manager.add_modelservice_to_registry = mock.Mock(
)
self.assertEqual(
self.test_frontend.register_module('mx_vision_service'), ret)
def test_register_gluon_service_module(self):
# Mock
ret = [GluonVisionService]
self.test_frontend.service_manager.parse_modelservices_from_module = mock.Mock(
return_value=ret)
self.test_frontend.service_manager.add_modelservice_to_registry = mock.Mock(
)
self.assertEqual(
self.test_frontend.register_module('mx_vision_service'), ret)
def test_get_registered_modelservices(self):
# Mock
all_model_services = {
MXNetBaseService.__name__: MXNetBaseService,
MXNetVisionService.__name__: MXNetVisionService
}
self.test_frontend.service_manager.get_modelservices_registry = mock.Mock(
return_value=all_model_services)
self.assertEqual(self.test_frontend.get_registered_modelservices(),
all_model_services)
def runTest(self):
self.test_register_module()
self.test_get_registered_modelservices()
class MMS(object):
'''MXNet Model Serving
'''
def __init__(self, app_name='mms'):
# Initialize serving frontend and arg parser
try:
self.args = ArgParser.parse_args()
self.serving_frontend = ServingFrontend(app_name)
logger.info('Initialized model serving.')
except Exception as e:
logger.error('Failed to initialize model serving: ' + str(e))
exit(1)
def start_model_serving(self):
'''Start model serving server
'''
try:
# Port
self.port = self.args.port or 8080
self.host = '127.0.0.1'
# Register user defined model service or default mxnet_vision_service
class_defs = self.serving_frontend.register_module(self.args.process)
if len(class_defs) < 2:
raise Exception('User defined module must derive base ModelService.')
# First class is the base ModelService class
mode_class_name = class_defs[1].__name__
# Load models using registered model definitions
registered_models = self.serving_frontend.get_registered_modelservices()
ModelClassDef = registered_models[mode_class_name]
self.serving_frontend.load_models(self.args.models, ModelClassDef)
if len(self.args.models) > 5:
raise Exception('Model number exceeds our system limits: 5')
# Setup endpoint
openapi_endpoints = self.serving_frontend.setup_openapi_endpoints(self.host, self.port)
# Generate client SDK
if self.args.gen_api is not None:
ClientSDKGenerator.generate(openapi_endpoints, self.args.gen_api)
# Start model serving host
self.serving_frontend.start_model_serving(self.host, self.port)
logger.info('Host is started at ' + self.host + ':' + str(self.port))
except Exception as e:
logger.error('Failed to start model serving host: ' + str(e))
exit(1)
def setUp(self):
self.test_frontend = ServingFrontend('test')
class MMS(object):
"""MXNet Model Serving
"""
def __init__(self, app_name='mms', args=None):
"""Initialize mxnet model server application.
Parameters
----------
app_name : str
App name to initialize mms service.
args : List of str
Arguments for starting service. By default it is None
and commandline arguments will be used. It should follow
the format recognized by python argparse parse_args method:
https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser.parse_args.
An example for mms arguments:
['--models', 'resnet-18=path1', 'inception_v3=path2',
'--gen-api', 'java', '--port', '8080']
"""
# Initialize serving frontend and arg parser
try:
parser = ArgParser.mms_parser()
self.args = parser.parse_args(
args) if args else parser.parse_args()
self.serving_frontend = ServingFrontend(app_name)
self.gpu = self.args.gpu
# Setup root logger handler and level.
log_file = self.args.log_file
log_level = self.args.log_level or "INFO"
log_rotation_time = self.args.log_rotation_time or "1 H"
_set_root_logger(log_file, log_level, log_rotation_time)
logger.info('Initialized model serving.')
except Exception as e:
print('Failed to initialize model serving: ' + str(e))
exit(1)
def start_model_serving(self):
"""Start model serving server
"""
try:
# Process arguments
self._arg_process()
# Start model serving host
if self.args.gen_api is None:
logger.info('Service started successfully.')
logger.info('Service description endpoint: ' + self.host +
':' + str(self.port) + '/api-description')
logger.info('Service health endpoint: ' + self.host + ':' +
str(self.port) + '/ping')
self.serving_frontend.start_handler(self.host, self.port)
except SocketServer.socket.error as exc:
if exc.args[0] != 48:
raise
logger.error('Port: ' + str(self.port) +
" already in use. exiting...")
exit(1)
except Exception as e:
logger.error('Failed to start model serving host: ' + str(e))
exit(1)
def create_app(self):
"""Create a Flask app object.
"""
try:
# Process arguments
self._arg_process()
logger.info('Service started successfully.')
logger.info('Service description endpoint: ' + self.host + ':' +
str(self.port) + '/api-description')
logger.info('Service health endpoint: ' + self.host + ':' +
str(self.port) + '/ping')
# Create app
return self.serving_frontend.handler.app
except Exception as e:
logger.error('Failed to start model serving host: ' + str(e))
exit(1)
def _arg_process(self):
"""Process arguments before starting service or create application.
"""
try:
# Port
self.port = int(self.args.port) if self.args.port else 8080
self.host = self.args.host or '127.0.0.1'
# Load models
models = ModelLoader.load(self.args.models)
# Register user defined model service or default mxnet_vision_service
manifest = models[0][3]
service_file = os.path.join(models[0][2],
manifest['Model']['Service'])
class_defs = self.serving_frontend.register_module(
self.args.service or service_file)
if len(class_defs) < 1:
raise Exception(
'User defined module must derive base ModelService.')
# The overrided class is the last one in class_defs
mode_class_name = class_defs[-1].__name__
# Load models using registered model definitions
registered_models = self.serving_frontend.get_registered_modelservices(
)
ModelClassDef = registered_models[mode_class_name]
self.serving_frontend.load_models(models, ModelClassDef, self.gpu)
if len(self.args.models) > 5:
raise Exception('Model number exceeds our system limits: 5')
# Setup endpoint
openapi_endpoints = self.serving_frontend.setup_openapi_endpoints(
self.host, self.port)
# Generate client SDK
if self.args.gen_api is not None:
ClientSDKGenerator.generate(openapi_endpoints,
self.args.gen_api)
# Generate metrics to target location (log, csv ...), default to log
MetricsManager.start(self.args.metrics_write_to, mode_class_name,
Lock())
except Exception as e:
logger.error('Failed to process arguments: ' + str(e))
exit(1)
class MMS(object):
"""MXNet Model Serving
"""
def __init__(self, app_name='mms', args=None):
"""Initialize mxnet model server application.
Parameters
----------
app_name : str
App name to initialize mms service.
args : List of str
Arguments for starting service. By default it is None
and commandline arguments will be used. It should follow
the format recognized by python argparse parse_args method:
https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser.parse_args.
An example for mms arguments:
['--models', 'resnet-18=path1', 'inception_v3=path2',
'--gen-api', 'java', '--port', '8080']
"""
# Initialize serving frontend and arg parser
try:
parser = ArgParser.mms_parser()
self.args = parser.parse_args(args) if args else parser.parse_args()
self.serving_frontend = ServingFrontend(app_name)
self.gpu = self.args.gpu
# Setup root logger handler and level.
log_file = self.args.log_file
log_level = self.args.log_level or "INFO"
log_rotation_time = self.args.log_rotation_time or "1 H"
_set_root_logger(log_file, log_level, log_rotation_time)
logger.info('Initialized model serving.')
except Exception as e:
print ('Failed to initialize model serving: ' + str(e))
exit(1)
def start_model_serving(self):
"""Start model serving server
"""
try:
# Process arguments
self._arg_process()
# Start model serving host
if self.args.gen_api is None:
logger.info('Service started successfully.')
logger.info('Service description endpoint: ' + self.host + ':' + str(self.port) + '/api-description')
logger.info('Service health endpoint: ' + self.host + ':' + str(self.port) + '/ping')
self.serving_frontend.start_handler(self.host, self.port)
except SocketServer.socket.error as exc:
if exc.args[0] != 48:
raise
logger.error('Port: ' + str(self.port) + " already in use. exiting...")
exit(1)
except Exception as e:
logger.error('Failed to start model serving host: ' + str(e))
exit(1)
def create_app(self):
"""Create a Flask app object.
"""
try:
# Process arguments
self._arg_process()
logger.info('Service started successfully.')
logger.info('Service description endpoint: ' + self.host + ':' + str(self.port) + '/api-description')
logger.info('Service health endpoint: ' + self.host + ':' + str(self.port) + '/ping')
# Create app
return self.serving_frontend.handler.app
except Exception as e:
logger.error('Failed to start model serving host: ' + str(e))
exit(1)
def _arg_process(self):
"""Process arguments before starting service or create application.
"""
try:
# Port
self.port = int(self.args.port) if self.args.port else 8080
self.host = self.args.host or '127.0.0.1'
# Load models
models = ModelLoader.load(self.args.models)
# Register user defined model service or default mxnet_vision_service
manifest = models[0][3]
service_file = os.path.join(models[0][2], manifest['Model']['Service'])
class_defs = self.serving_frontend.register_module(self.args.service or service_file)
class_defs = list(filter(lambda c: len(c.__subclasses__()) == 0, class_defs))
if len(class_defs) != 1:
raise Exception('There should be one user defined service derived from ModelService.')
model_class_name = class_defs[0].__name__
# Load models using registered model definitions
registered_models = self.serving_frontend.get_registered_modelservices()
ModelClassDef = registered_models[model_class_name]
self.serving_frontend.load_models(models, ModelClassDef, self.gpu)
if len(self.args.models) > 5:
raise Exception('Model number exceeds our system limits: 5')
# Setup endpoint
openapi_endpoints = self.serving_frontend.setup_openapi_endpoints(self.host, self.port)
# Generate client SDK
if self.args.gen_api is not None:
ClientSDKGenerator.generate(openapi_endpoints, self.args.gen_api)
# Generate metrics to target location (log, csv ...), default to log
MetricsManager.start(self.args.metrics_write_to, self.args.models, Lock())
except Exception as e:
logger.error('Failed to process arguments: ' + str(e))
exit(1)
import logging
import os
import shutil
from mms.serving_frontend import ServingFrontend
from mms.model_loader import ModelLoader
logger = logging.getLogger()
logger.setLevel(logging.INFO)
serving_frontend = ServingFrontend(__name__)
model_path = 'squeezenet_v1.1.model'
if os.environ.get('LAMBDA_TASK_ROOT', False):
shutil.copyfile('/var/task/squeezenet_v1.1.model', '/tmp/squeezenet_v1.1.model')
model_path = '/tmp/squeezenet_v1.1.model'
models = ModelLoader.load({'squeezenet': model_path})
manifest = models[0][3]
service_file = os.path.join(models[0][2], manifest['Model']['Service'])
class_defs = serving_frontend.register_module(service_file)
if len(class_defs) < 1:
raise Exception('User defined module must derive base ModelService.')
# The overrided class is the last one in class_defs
mode_class_name = class_defs[-1].__name__
# Load models using registered model definitions