class ServingFrontend(object):
"""ServingFrontend warps up all internal services including
model service manager, request handler. It provides all public
apis for users to extend and use our system.
"""
def __init__(self, app_name):
"""
Initialize handler for FlaskHandler and ServiceManager.
Parameters
----------
app_name : str
App name to initialize request handler.
"""
try:
self.service_manager = ServiceManager()
self.handler = FlaskRequestHandler(app_name)
logger.info('Initialized serving frontend.')
except Exception as e:
raise Exception('Failed to initialize serving frontend: ' + str(e))
def start_handler(self, host, port):
"""
Start handler using given host and port.
Parameters
----------
host : str
Host that server will use.
port: int
Port that server will use.
"""
self.handler.start_handler(host, port)
def load_models(self, models, ModelServiceClassDef, gpu=None):
"""
Load models by using user passed Model Service Class Definitions.
Parameters
----------
models : List of model_name, model_path pairs
List of model_name, model_path pairs that will be initialized.
ModelServiceClassDef: python class
Model Service Class Definition which can initialize a model service.
gpu : int
Id of gpu device. If machine has two gpus, this number can be 0 or 1.
If it is not set, cpu will be used.
"""
for service_name, model_name, model_path, manifest in models:
self.service_manager.load_model(service_name, model_name, model_path, manifest, ModelServiceClassDef, gpu)
def register_module(self, user_defined_module_file_path):
"""
Register a python module according to user_defined_module_name
This module should contain a valid Model Service Class whose
pre-process and post-process can be derived and customized.
Parameters
----------
user_defined_module_file_path : Python module file path
A python module will be loaded according to this file path.
Returns
----------
List of model service class definitions.
Those python class can be used to initialize model service.
"""
model_class_definations = self.service_manager.parse_modelservices_from_module(user_defined_module_file_path)
assert len(model_class_definations) >= 1, 'No valid python class derived from Base Model Service is in module file: %s' % user_defined_module_file_path
for ModelServiceClassDef in model_class_definations:
self.service_manager.add_modelservice_to_registry(ModelServiceClassDef.__name__, ModelServiceClassDef)
return model_class_definations
def get_registered_modelservices(self, modelservice_names=None):
"""
Get all registered Model Service Class Definitions into a dictionary
according to name or list of names.
If nothing is passed, all registered model services will be returned.
Parameters
----------
modelservice_names : str or List, optional
Names to retrieve registered model services
Returns
----------
Dict of name, model service pairs
Registered model services according to given names.
"""
if not isinstance(modelservice_names, list) and modelservice_names is not None:
modelservice_names = [modelservice_names]
return self.service_manager.get_modelservices_registry(modelservice_names)
def get_loaded_modelservices(self, modelservice_names=None):
"""
Get all model services which are loaded in the system into a dictionary
according to name or list of names.
If nothing is passed, all loaded model services will be returned.
Parameters
----------
modelservice_names : str or List, optional
Names to retrieve loaded model services
Returns
----------
Dict of name, model service pairs
Loaded model services according to given names.
"""
if not isinstance(modelservice_names, list) and modelservice_names is not None:
modelservice_names = [modelservice_names]
return self.service_manager.get_loaded_modelservices(modelservice_names)
def get_query_string(self, field):
"""
Get field data in the query string from request.
Parameters
----------
field : str
Field in the query string from request.
Returns
----------
Object
Field data in query string.
"""
return self.handler.get_query_string(field)
def add_endpoint(self, api_definition, callback, **kwargs):
"""
Add an endpoint with OpenAPI compatible api definition and callback.
Parameters
----------
api_definition : dict(json)
OpenAPI compatible api definition.
callback: function
Callback function in the endpoint.
kwargs: dict
Arguments for callback functions.
"""
endpoint = list(api_definition.keys())[0]
method = list(api_definition[endpoint].keys())[0]
api_name = api_definition[endpoint][method]['operationId']
logger.info('Adding endpoint: %s to Flask' % api_name)
self.handler.add_endpoint(api_name, endpoint, partial(callback, **kwargs), [method.upper()])
def setup_openapi_endpoints(self, host, port):
"""
Firstly, construct Openapi compatible api definition for
1. Predict
2. Ping
3. API description
Then the api definition is used to setup web server endpoint.
Parameters
----------
host : str
Host that server will use
port: int
Host that server will use
"""
modelservices = self.service_manager.get_loaded_modelservices()
# TODO: not hardcode host:port
self.openapi_endpoints = {
'swagger': '2.0',
'info': {
'version': '1.0.0',
'title': 'Model Serving Apis'
},
'host': host + ':' + str(port),
'schemes': ['http'],
'paths': {},
}
# 1. Predict endpoints
for model_name, modelservice in modelservices.items():
input_type = modelservice.signature['input_type']
inputs = modelservice.signature['inputs']
output_type = modelservice.signature['output_type']
# Contruct predict openapi specs
endpoint = '/' + model_name + '/predict'
predict_api = {
endpoint: {
'post': {
'operationId': model_name + '_predict',
'consumes': ['multipart/form-data'],
'produces': [output_type],
'parameters': [],
'responses': {
'200': {}
}
}
}
}
input_names = []
# Setup endpoint for each modelservice
for idx in range(len(inputs)):
# Check input content type to set up proper openapi consumes field
input_names.append(inputs[idx]['data_name'])
if input_type == 'application/json':
parameter = {
'in': 'formData',
'name': inputs[idx]['data_name'],
'description': '%s should tensor with shape: %s' %
(inputs[idx]['data_name'], inputs[idx]['data_shape'][1:]),
'required': 'true',
'schema': {
'type': 'string'
}
}
elif input_type == 'image/jpeg':
parameter = {
'in': 'formData',
'name': inputs[idx]['data_name'],
'description': '%s should be image which will be resized to: %s' %
(inputs[idx]['data_name'], inputs[idx]['data_shape'][1:]),
'required': 'true',
'type': 'file'
}
else:
msg = '%s is not supported for input content-type' % (input_type)
logger.error(msg)
abort(500, "Service setting error. %s" % (msg))
predict_api[endpoint]['post']['parameters'].append(parameter)
# Contruct openapi response schema
if output_type == 'application/json':
responses = {
'description': 'OK',
'schema': {
'type': 'object',
'properties': {
'prediction': {
'type': 'string'
}
}
}
}
elif output_type == 'image/jpeg':
responses = {
'description': 'OK',
'schema': {
'type': 'file'
}
}
else:
msg = '%s is not supported for output content-type' % output_type
logger.error(msg)
abort(500, "Service setting error. %s" % (msg))
predict_api[endpoint]['post']['responses']['200'].update(responses)
self.openapi_endpoints['paths'].update(predict_api)
# Setup Flask endpoint for predict api
self.add_endpoint(predict_api,
self.predict_callback,
modelservice=modelservice,
input_names=input_names,
model_name=model_name)
# 2. Ping endpoints
ping_api = {
'/ping': {
'get': {
'operationId': 'ping',
'produces': ['application/json'],
'responses': {
'200': {
'description': 'OK',
'schema': {
'type': 'object',
'properties': {
'health': {
'type': 'string'
}
}
}
}
}
}
}
}
self.openapi_endpoints['paths'].update(ping_api)
self.add_endpoint(ping_api, self.ping_callback)
# 3. Describe apis endpoints
api_description_api = {
'/api-description': {
'get': {
'produces': ['application/json'],
'operationId': 'api-description',
'responses': {
'200': {
'description': 'OK',
'schema': {
'type': 'object',
'properties': {
'description': {
'type': 'string'
}
}
}
}
}
}
}
}
self.openapi_endpoints['paths'].update(api_description_api)
self.add_endpoint(api_description_api, self.api_description)
return self.openapi_endpoints
def ping_callback(self, **kwargs):
"""
Callback function for ping endpoint.
Returns
----------
Response
Http response for ping endpiont.
"""
if 'PingTotal' in MetricsManager.metrics:
MetricsManager.metrics['PingTotal'].update(metric=1)
try:
for model in self.service_manager.get_loaded_modelservices().values():
model.ping()
except Exception:
logger.warn('Model serving is unhealthy.')
return self.handler.jsonify({'health': 'unhealthy!'})
return self.handler.jsonify({'health': 'healthy!'})
def api_description(self, **kwargs):
"""
Callback function for api description endpoint.
Returns
----------
Response
Http response for api description endpiont.
"""
if 'APIDescriptionTotal' in MetricsManager.metrics:
MetricsManager.metrics['APIDescriptionTotal'].update(metric=1)
return self.handler.jsonify({'description': self.openapi_endpoints})
def predict_callback(self, **kwargs):
"""
Callback for predict endpoint
Parameters
----------
modelservice : ModelService
ModelService handler.
input_names: list
Input names in request form data.
Returns
----------
Response
Http response for predict endpiont.
"""
handler_start_time = time.time()
modelservice = kwargs['modelservice']
input_names = kwargs['input_names']
model_name = kwargs['model_name']
if model_name + '_PredictionTotal' in MetricsManager.metrics:
MetricsManager.metrics[model_name + '_PredictionTotal'].update(metric=1)
input_type = modelservice.signature['input_type']
output_type = modelservice.signature['output_type']
# Get data from request according to input type
input_data = []
if input_type == 'application/json':
try:
for name in input_names:
logger.info('Request input: ' + name + ' should be json tensor.')
form_data = self.handler.get_form_data(name)
form_data = ast.literal_eval(form_data)
assert isinstance(form_data, list), "Input data for request argument: %s is not correct. " \
"%s is expected but got %s instead of list" \
% (name, input_type, type(form_data))
input_data.append(form_data)
except Exception as e:
if model_name + '_Prediction4XX' in MetricsManager.metrics:
MetricsManager.metrics[model_name + '_Prediction4XX'].update(metric=1)
logger.error(str(e))
abort(400, str(e))
elif input_type == 'image/jpeg':
try:
for name in input_names:
logger.info('Request input: ' + name + ' should be image with jpeg format.')
input_file = self.handler.get_file_data(name)
if input_file:
mime_type = input_file.content_type
assert mime_type == input_type, 'Input data for request argument: %s is not correct. ' \
'%s is expected but %s is given.' % (name, input_type, mime_type)
file_data = input_file.read()
assert isinstance(file_data, (str, bytes)), 'Image file buffer should be type str or ' \
'bytes, but got %s' % (type(file_data))
else:
file_data = base64.decodestring(self.handler.get_form_data(name))
input_data.append(file_data)
except Exception as e:
if model_name + '_Prediction4XX' in MetricsManager.metrics:
MetricsManager.metrics[model_name + '_Prediction4XX'].update(metric=1)
logger.error(str(e))
abort(400, str(e))
else:
msg = '%s is not supported for input content-type' % (input_type)
if model_name + '_Prediction5XX' in MetricsManager.metrics:
MetricsManager.metrics[model_name + '_Prediction5XX'].update(metric=1)
logger.error(msg)
abort(500, "Service setting error. %s" % (msg))
# Doing prediciton on model
try:
response = modelservice.inference(input_data)
except Exception:
if model_name + '_Prediction5XX' in MetricsManager.metrics:
MetricsManager.metrics[model_name + '_Prediction5XX'].update(metric=1)
logger.error(str(traceback.format_exc()))
abort(500, "Error occurs while inference was executed on server.")
# Construct response according to output type
if output_type == 'application/json':
logger.info('Response is text.')
elif output_type == 'image/jpeg':
logger.info('Response is jpeg image encoded in base64 string.')
else:
msg = '%s is not supported for input content-type.' % (output_type)
if model_name + '_Prediction5XX' in MetricsManager.metrics:
MetricsManager.metrics[model_name + '_Prediction5XX'].update(metric=1)
logger.error(msg)
abort(500, "Service setting error. %s" % (msg))
logger.debug("Prediction request handling time is: %s ms" %
((time.time() - handler_start_time) * 1000))
return self.handler.jsonify({'prediction': response})
class ServingFrontend(object):
'''ServingFrontend warps up all internal services including
model service manager, request handler. It provides all public
apis for users to extend and use our system.
'''
def __init__(self, app_name):
'''
Initialize handler for FlaskHandler and ServiceManager.
Parameters
----------
app_name : string
App name to initialize request handler.
'''
try:
self.service_manager = ServiceManager()
self.handler = FlaskRequestHandler(app_name)
logger.info('Initialized serving frontend.')
except Exception as e:
raise Exception('Failed to initialize serving frontend: ' + str(e))
def start_model_serving(self, host, port):
'''
Start model serving using given host and port.
Parameters
----------
host : string
Host that server will use.
port: int
Port that server will use.
'''
self.handler.start_handler(host, port)
def load_models(self, models, ModelServiceClassDef):
'''
Load models by using user passed Model Service Class Definitions.
Parameters
----------
models : List of model_name, model_path pairs
List of model_name, model_path pairs that will be initialized.
ModelServiceClassDef: python class
Model Service Class Definition which can initialize a model service.
'''
for model_name, model_path in models.items():
self.service_manager.load_model(model_name, model_path,
ModelServiceClassDef)
def register_module(self, user_defined_module_file_path):
'''
Register a python module according to user_defined_module_name
This module should contain a valid Model Service Class whose
pre-process and post-process can be derived and customized.
Parameters
----------
user_defined_module_file_path : Python module file path
A python module will be loaded according to this file path.
Returns
----------
List of model service class definitions.
Those python class can be used to initialize model service.
'''
model_class_definations = self.service_manager.parse_modelservices_from_module(
user_defined_module_file_path)
assert len(
model_class_definations
) >= 1, 'No valid python class derived from Base Model Service is in module file: %s' % user_defined_module_file_path
for ModelServiceClassDef in model_class_definations:
self.service_manager.add_modelservice_to_registry(
ModelServiceClassDef.__name__, ModelServiceClassDef)
return model_class_definations
def get_registered_modelservices(self, modelservice_names=None):
'''
Get all registered Model Service Class Definitions into a dictionary
according to name or list of names.
If nothing is passed, all registered model services will be returned.
Parameters
----------
modelservice_names : string or List, optional
Names to retrieve registered model services
Returns
----------
Dict of name, model service pairs
Registered model services according to given names.
'''
if not isinstance(modelservice_names,
list) and modelservice_names is not None:
modelservice_names = [modelservice_names]
return self.service_manager.get_modelservices_registry(
modelservice_names)
def get_loaded_modelservices(self, modelservice_names=None):
'''
Get all model services which are loaded in the system into a dictionary
according to name or list of names.
If nothing is passed, all loaded model services will be returned.
Parameters
----------
modelservice_names : string or List, optional
Names to retrieve loaded model services
Returns
----------
Dict of name, model service pairs
Loaded model services according to given names.
'''
if not isinstance(modelservice_names,
list) and modelservice_names is not None:
modelservice_names = [modelservice_names]
return self.service_manager.get_loaded_modelservices(
modelservice_names)
def get_query_string(self, field):
'''
Get field data in the query string from request.
Parameters
----------
field : string
Field in the query string from request.
Returns
----------
Object
Field data in query string.
'''
return self.handler.get_query_string(field)
def add_endpoint(self, api_definition, callback, **kwargs):
'''
Add an endpoint with OpenAPI compatible api definition and callback.
Parameters
----------
api_definition : dict(json)
OpenAPI compatible api definition.
callback: function
Callback function in the endpoint.
kwargs: dict
Arguments for callback functions.
'''
endpoint = list(api_definition.keys())[0]
method = list(api_definition[endpoint].keys())[0]
api_name = api_definition[endpoint][method]['operationId']
logger.info('Adding endpoint: %s to Flask' % api_name)
self.handler.add_endpoint(api_name, endpoint,
partial(callback, **kwargs),
[method.upper()])
def setup_openapi_endpoints(self, host, port):
'''
Firstly, construct Openapi compatible api definition for
1. Predict
2. Ping
3. API description
Then the api definition is used to setup web server endpoint.
Parameters
----------
host : string
Host that server will use
port: int
Host that server will use
'''
modelservices = self.service_manager.get_loaded_modelservices()
# TODO: not hardcode host:port
self.openapi_endpoints = {
'swagger': '2.0',
'info': {
'version': '1.0.0',
'title': 'Model Serving Apis'
},
'host': host + ':' + str(port),
'schemes': ['http'],
'paths': {},
}
# 1. Predict endpoints
for model_name, modelservice in modelservices.items():
input_type = modelservice.signature['input_type']
inputs = modelservice.signature['inputs']
output_type = modelservice.signature['output_type']
# Contruct predict openapi specs
endpoint = '/' + model_name + '/predict'
predict_api = {
endpoint: {
'post': {
'operationId': model_name + '_predict',
'consumes': ['multipart/form-data'],
'produces': [output_type],
'parameters': [],
'responses': {
'200': {}
}
}
}
}
input_names = ['input' + str(idx) for idx in range(len(inputs))]
# Setup endpoint for each modelservice
for idx in range(len(inputs)):
# Check input content type to set up proper openapi consumes field
if input_type == 'application/json':
parameter = {
'in':
'formData',
'name':
input_names[idx],
'description':
'%s should tensor with shape: %s' %
(input_names[idx], inputs[idx]['data_shape'][1:]),
'required':
'true',
'schema': {
'type': 'string'
}
}
elif input_type == 'image/jpeg':
parameter = {
'in':
'formData',
'name':
input_names[idx],
'description':
'%s should be image with shape: %s' %
(input_names[idx], inputs[idx]['data_shape'][1:]),
'required':
'true',
'type':
'file'
}
else:
raise Exception(
'%s is not supported for input content-type' %
input_type)
predict_api[endpoint]['post']['parameters'].append(parameter)
# Contruct openapi response schema
if output_type == 'application/json':
responses = {
'description': 'OK',
'schema': {
'type': 'object',
'properties': {
'prediction': {
'type': 'string'
}
}
}
}
elif output_type == 'image/jpeg':
responses = {'description': 'OK', 'schema': {'type': 'file'}}
else:
raise Exception('%s is not supported for output content-type' %
output_type)
predict_api[endpoint]['post']['responses']['200'].update(responses)
self.openapi_endpoints['paths'].update(predict_api)
# Setup Flask endpoint for predict api
self.add_endpoint(predict_api,
self.predict_callback,
modelservice=modelservice,
input_names=input_names)
# 2. Ping endpoints
ping_api = {
'/ping': {
'get': {
'operationId': 'ping',
'produces': ['application/json'],
'responses': {
'200': {
'description': 'OK',
'schema': {
'type': 'object',
'properties': {
'health': {
'type': 'string'
}
}
}
}
}
}
}
}
self.openapi_endpoints['paths'].update(ping_api)
self.add_endpoint(ping_api, self.ping_callback)
# 3. Describe apis endpoints
api_description_api = {
'/api-description': {
'get': {
'produces': ['application/json'],
'operationId': 'apiDescription',
'responses': {
'200': {
'description': 'OK',
'schema': {
'type': 'object',
'properties': {
'description': {
'type': 'string'
}
}
}
}
}
}
}
}
self.openapi_endpoints['paths'].update(api_description_api)
self.add_endpoint(api_description_api, self.api_description)
return self.openapi_endpoints
def ping_callback(self, **kwargs):
'''
Callback function for ping endpoint.
Returns
----------
Response
Http response for ping endpiont.
'''
try:
for model in self.service_manager.get_loaded_modelservices(
).values():
model.ping()
except Exception:
logger.warn('Model serving is unhealthy.')
return self.handler.jsonify({'health': 'unhealthy!'})
return self.handler.jsonify({'health': 'healthy!'})
def api_description(self, **kwargs):
'''
Callback function for api description endpoint.
Returns
----------
Response
Http response for api description endpiont.
'''
return self.handler.jsonify({'description': self.openapi_endpoints})
def predict_callback(self, **kwargs):
'''
Callback for predict endpoint
Parameters
----------
modelservice : ModelService
ModelService handler.
input_names: list
Input names in request form data.
Returns
----------
Response
Http response for predict endpiont.
'''
modelservice = kwargs['modelservice']
input_names = kwargs['input_names']
input_type = modelservice.signature['input_type']
output_type = modelservice.signature['output_type']
# Get data from request according to input type
input_data = []
if input_type == 'application/json':
form_data = None
try:
for name in input_names:
logger.info('Request input: ' + name +
' should be json tensor.')
form_data = self.handler.get_form_data(name)
assert isinstance(form_data, dict)
input_data.append(form_data)
except:
raise Exception(
'Type for request argument %s is not correct. %s is expected but %s is given.'
% (name, input_type, type(form_data)))
elif input_type == 'image/jpeg':
file_data = None
try:
for name in input_names:
logger.info('Request input: ' + name +
' should be image with jpeg format.')
file_data = self.handler.get_file_data(name).read()
assert isinstance(file_data, (str, bytes))
input_data.append(file_data)
except Exception as e:
raise Exception(
'Input data for request argument: %s is not correct. %s is expected but %s '
'is given.' % (str(e), input_type, type(file_data)))
else:
logger.warn('%s is not supported for input content-type' %
input_type)
raise Exception('%s is not supported for input content-type' %
input_type)
# Doing prediciton on model
try:
response = modelservice.inference(input_data)
except Exception as e:
raise Exception('MXNet prediction run-time error')
# Construct response according to output type
if output_type == 'application/json':
logger.info('Response is text.')
return self.handler.jsonify({'prediction': response})
elif output_type == 'image/jpeg':
logger.info('Response is jpeg image encoded in base64 string.')
return self.handler.jsonify({'prediction': response})
else:
logger.warn('%s is not supported for input content-type' %
output_type)
raise Exception('%s is not supported for output content-type' %
output_type)