def test8(self):
"""testing predict() controller will handle exceptions in the model class correctly"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "tests.web_api.controllers_test",
"class_name": "MLModelMock"
}])
# act
exception_raised = False
try:
result = controllers.predict(qualified_name="qualified_name",
request_body='{}')
schema = ErrorSchema()
data = schema.loads(json_data=result.data)
except Exception as e:
exception_raised = True
# assert
self.assertFalse(exception_raised)
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 500)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(
json.loads(result.data) == {
"message": "Could not make a prediction.",
"type": "ERROR"
})
def test1(self):
""" testing the load_models() method """
# arrange
# instantiating the model manager class
model_manager = ModelManager()
# loading the MLModel objects from configuration
model_manager.load_models(configuration=[{
"module_name": "tests.model_manager_test",
"class_name": "MLModelMock"
}])
# act
exception_raised = False
model_object = None
# accessing the MLModelMock model object
try:
model_object = model_manager.get_model(
qualified_name="qualified_name")
except Exception as e:
exception_raised = True
print_tb(e)
# assert
self.assertFalse(exception_raised)
self.assertTrue(model_object is not None)
def test7(self):
"""testing predict() controller with data that does not meet the model schema"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.predict(
qualified_name="iris_model",
request_body=
'{"petal_length": "asdf", "petal_width": 1.0, "sepal_length": 1.0, "sepal_width": 1.0}'
)
schema = ErrorSchema()
data = schema.loads(json_data=result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 400)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(
json.loads(result.data) == {
"message":
"Failed to validate input data: Key 'petal_length' error:\n'asdf' should be instance of 'float'",
"type": "SCHEMA_ERROR"
})
def test5(self):
"""testing predict() controller with non-existing model"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.predict(
qualified_name="asdf",
request_body=
'{"petal_length": 1.0, "petal_width": 1.0, "sepal_length": 1.0, "sepal_width": 1.0}'
)
schema = ErrorSchema()
data = schema.loads(json_data=result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 404)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(
json.loads(result.data) == {
"type": "ERROR",
"message": "Model not found."
})
def test2(self):
"""test model manager is loaded with configuration when the lambda_function module is initiated"""
# arrange, act
from model_lambda.model_manager import ModelManager
model_manager = ModelManager()
# assert
self.assertTrue(model_manager.get_models() == [{'display_name': 'Iris Model', 'qualified_name': 'iris_model', 'description': 'A machine learning model for predicting the species of a flower based on its measurements.', 'major_version': 0, 'minor_version': 1}])
def test2(self):
"""testing that the ModelManager will return the same instance of an MLModel class from several different
instances of ModelManager"""
# arrange
# instantiating the model manager class
first_model_manager = ModelManager()
# loading the MLModel objects from configuration
first_model_manager.load_models(
configuration=[{
"module_name": "tests.model_manager_test",
"class_name": "MLModelMock"
}])
# act
first_model_object = first_model_manager.get_model(
qualified_name="iris_model")
# instantiating the ModelManager class again
second_model_manager = ModelManager()
second_model_object = second_model_manager.get_model(
qualified_name="iris_model")
# assert
self.assertTrue(str(first_model_object) == str(second_model_object))
def test1(self):
"""testing get_models() controller"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.get_models()
schema = ModelCollectionSchema()
data = schema.loads(json_data=result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 200)
self.assertTrue(result.mimetype == "application/json")
def test6(self):
"""testing predict() controller with good data"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.predict(
qualified_name="iris_model",
request_body=
'{"petal_length": 1.0, "petal_width": 1.0, "sepal_length": 1.0, "sepal_width": 1.0}'
)
data = json.loads(result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 200)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(json.loads(result.data) == {"species": "setosa"})
def get_models():
"""List of models available.
---
get:
responses:
200:
description: List of model available
content:
application/json:
schema:
$ref: '#/components/schemas/ModelCollection'
"""
# instantiating ModelManager singleton
model_manager = ModelManager()
# retrieving the model object from the model manager
models = model_manager.get_models()
response_data = model_collection_schema.dumps(dict(models=models))
return Response(data=response_data,
status=200,
mimetype="application/json")
def test3(self):
"""testing get_metadata() controller with non-existing model"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.get_metadata(qualified_name="asdf")
schema = ErrorSchema()
data = schema.loads(json_data=result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 400)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(
json.loads(result.data) == {
"message": "Model not found.",
"type": "ERROR"
})
def get_metadata(qualified_name):
"""Metadata about one model.
---
get:
parameters:
- in: path
name: qualified_name
schema:
type: string
required: true
description: The qualified name of the model for which metadata is being requested.
responses:
200:
description: Metadata about one model
content:
application/json:
schema:
$ref: '#/components/schemas/ModelMetadata'
404:
description: Model not found.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
"""
model_manager = ModelManager()
metadata = model_manager.get_model_metadata(qualified_name=qualified_name)
if metadata is not None:
response_data = model_metadata_schema.dumps(metadata)
return Response(response_data, status=200, mimetype='application/json')
else:
response = dict(type="ERROR", message="Model not found.")
response_data = error_schema.dumps(response)
return Response(data=response_data,
status=400,
mimetype='application/json')
def test4(self):
"""testing predict() controller with bad data"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.predict(qualified_name="iris_model",
request_body="")
schema = ErrorSchema()
data = schema.loads(json_data=result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 400)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(
json.loads(result.data) == {
"type": "DESERIALIZATION_ERROR",
"message": "Expecting value: line 1 column 1 (char 0)"
})
def test3(self):
""" testing that the ModelManager only allows MLModel objects to be stored """
# arrange
model_manager = ModelManager()
# act
exception_raised = False
exception_message = ""
try:
model_manager.load_models(configuration=[{
"module_name": "tests.model_manager_test",
"class_name":
"SomeClass" # using the class defined at the top of this file to test
}])
except Exception as e:
exception_raised = True
exception_message = str(e)
# assert
self.assertTrue(exception_raised)
self.assertTrue(
exception_message ==
"The ModelManager can only hold references to objects of type MLModel."
)
def test2(self):
"""testing get_metadata() controller with existing model"""
# arrange
model_manager = ModelManager()
model_manager.load_models(configuration=[{
"module_name": "iris_model.iris_predict",
"class_name": "IrisModel"
}])
# act
result = controllers.get_metadata(qualified_name="iris_model")
schema = ModelMetadataSchema()
data = schema.loads(json_data=result.data)
# assert
self.assertTrue(type(result) == controllers.Response)
self.assertTrue(result.status == 200)
self.assertTrue(result.mimetype == "application/json")
self.assertTrue(
json.loads(result.data) == {
'output_schema': {
'additionalProperties': False,
'required': ['species'],
'schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
'id': 'https://example.com/output_schema.json',
'properties': {
'species': {
'type': 'string'
}
}
},
'minor_version': 1,
'major_version': 0,
'qualified_name': 'iris_model',
'description':
'A machine learning model for predicting the species of a flower based on its measurements.',
'display_name': 'Iris Model',
'input_schema': {
'additionalProperties':
False,
'required': [
'sepal_length', 'sepal_width', 'petal_length',
'petal_width'
],
'schema':
'http://json-schema.org/draft-07/schema#',
'type':
'object',
'id':
'https://example.com/input_schema.json',
'properties': {
'sepal_length': {
'type': 'number'
},
'sepal_width': {
'type': 'number'
},
'petal_length': {
'type': 'number'
},
'petal_width': {
'type': 'number'
}
}
}
})
def predict(qualified_name, request_body):
"""Endpoint that uses a model to make a prediction.
---
post:
parameters:
- in: path
name: qualified_name
schema:
type: string
required: true
description: The qualified name of the model being used for prediction.
responses:
200:
description: Prediction is succesful. The schema of the body of the response is described by the model's
output schema.
400:
description: Input is not valid JSON or does not meet the model's input schema.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
404:
description: Model not found.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
500:
description: Server error.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
"""
# attempting to deserialize JSON in body of request
try:
data = json.loads(request_body)
except json.decoder.JSONDecodeError as e:
response = dict(type="DESERIALIZATION_ERROR", message=str(e))
response_data = error_schema.dumps(response)
return Response(data=response_data,
status=400,
mimetype='application/json')
# getting the model object from the Model Manager
model_manager = ModelManager()
model_object = model_manager.get_model(qualified_name=qualified_name)
# returning a 404 if model is not found
if model_object is None:
response = dict(type="ERROR", message="Model not found.")
response_data = error_schema.dumps(response)
return Response(data=response_data,
status=404,
mimetype='application/json')
try:
prediction = model_object.predict(data)
return Response(data=json.dumps(prediction),
status=200,
mimetype="application/json")
except MLModelSchemaValidationException as e:
# responding with a 400 if the schema does not meet the model's input schema
response = dict(type="SCHEMA_ERROR", message=str(e))
response_data = error_schema.dumps(response)
return Response(data=response_data,
status=400,
mimetype='application/json')
except Exception as e:
response = dict(type="ERROR", message="Could not make a prediction.")
response_data = error_schema.dumps(response)
return Response(data=response_data,
status=500,
mimetype='application/json')
"""Lambda function entry point."""
from model_lambda.model_manager import ModelManager
from model_lambda.config import Config
from model_lambda.web_api.controllers import get_models, get_metadata, predict
# instantiating the model manager class
model_manager = ModelManager()
# loading the MLModel objects from configuration
model_manager.load_models(configuration=Config.models)
def lambda_handler(event, context):
"""Lambda handler function."""
# detecting if the event came from an API Gateway
if event.get("resource") is not None \
and event.get("path") is not None \
and event.get("httpMethod") is not None:
if event["resource"] == "/api/models" and event["httpMethod"] == "GET":
# calling the get_models controller function
response = get_models()
elif event[
"resource"] == "/api/models/{qualified_name}/metadata" and event[
"httpMethod"] == "GET":
# calling the get_metadata controller function
response = get_metadata(
qualified_name=event["pathParameters"]["qualified_name"])