def test_request_schema_response_schema_uninitialized(self):
model = mock.Mock()
model_name = 'my-test-model-noschemas'
api_version = 'v1'
mock_additional_checks = mock.Mock()
prediction_schema = schemas.Object(properties=dict(
prediction=schemas.Number(),
confidence=schemas.Number(),
))
prediction_service = PredictionService(
model=model,
name=model_name,
api_version=api_version,
meta={},
preprocessor=None,
postprocessor=None,
)
# request_schema is None if feature_schema is None
self.assertIs(prediction_service.request_schema, None)
# response_schema has a default
response = dict(
model_context=dict(
api_version=api_version,
model_meta={},
model_name=model_name),
predictions=[ dict(id=1, prediction=3.14) ],
request_id='abcdefg')
prediction_service.response_schema.validate(response)
def test_request_schema(self):
model = mock.Mock()
model_name = api_version = mock.MagicMock()
mock_additional_checks = mock.Mock()
feature_schema = schemas.Object(properties=dict(
x=schemas.Integer(),
y=schemas.Number(),
z=schemas.String(),
))
with mock.patch('porter.services.BaseService.add_request_schema') as mock_add_request_schema:
prediction_service = PredictionService(
model=model,
name=model_name,
api_version=api_version,
meta={},
preprocessor=None,
postprocessor=None,
batch_prediction=False,
feature_schema=feature_schema,
)
request_schema = prediction_service.request_schema
request = dict(id=1, x=2, y=3.5, z='4')
request_schema.validate(request)
with self.assertRaisesRegex(ValueError, 'data must contain'):
request = dict(x=2, y=3.5, z='4')
request_schema.validate(request)
def test__add_prediction_schema_batch(self):
model = mock.Mock()
model_name = api_version = mock.MagicMock()
mock_additional_checks = mock.Mock()
prediction_schema = schemas.Object(properties=dict(
prediction=schemas.Number(),
confidence=schemas.Number(),
))
with mock.patch('porter.services.BaseService.add_response_schema') as mock_add_response_schema:
prediction_service = PredictionService(
model=model,
name=model_name,
api_version=api_version,
meta={},
preprocessor=None,
postprocessor=None,
batch_prediction=True,
prediction_schema=prediction_schema,
)
args = mock_add_response_schema.call_args_list[-1][0]
self.assertEqual(args[0].upper(), 'POST')
self.assertEqual(args[1], 200)
response_obj = args[2]
self.assertIsInstance(response_obj, schemas.Object)
self.assertIn('request_id', response_obj.properties)
self.assertIn('model_context', response_obj.properties)
self.assertIn('predictions', response_obj.properties)
pred_obj = response_obj.properties['predictions']
self.assertIsInstance(pred_obj, schemas.Array)
item_obj = pred_obj.item_type
self.assertIn('id', item_obj.properties)
self.assertIn('prediction', item_obj.properties)
pred_schema = item_obj.properties['prediction']
self.assertIn('prediction', pred_schema.properties)
self.assertIn('confidence', pred_schema.properties)
def test__add_feature_schema_batch(self):
model = mock.Mock()
model_name = api_version = mock.MagicMock()
mock_additional_checks = mock.Mock()
feature_schema = schemas.Object(properties=dict(
x=schemas.Integer(),
y=schemas.Number(),
z=schemas.String(),
))
with mock.patch('porter.services.BaseService.add_request_schema') as mock_add_request_schema:
prediction_service = PredictionService(
model=model,
name=model_name,
api_version=api_version,
meta={},
preprocessor=None,
postprocessor=None,
batch_prediction=True,
feature_schema=feature_schema,
)
args = mock_add_request_schema.call_args_list[0][0]
self.assertEqual(args[0].upper(), 'POST')
request_obj = args[1]
self.assertIsInstance(request_obj, schemas.Array)
item_obj = request_obj.item_type
self.assertIn('id', item_obj.properties)
self.assertIn('x', item_obj.properties)
self.assertIn('y', item_obj.properties)
self.assertIn('z', item_obj.properties)
def setUpClass(cls):
service1 = PredictionService(
name='service1',
api_version='2',
model=None, # we're not going to make calls for predictions here
feature_schema=sc.Object(properties={
'a': sc.Integer(),
'b': sc.Integer(),
'c': sc.Number()
}))
service1 = PredictionService(
namespace='ns',
name='service2',
api_version='1',
model=None, # we're not going to make calls for predictions here
feature_schema=sc.Object(properties={
'a': sc.Integer(),
'b': sc.Integer()
}))
def test_add_request_schema(self):
input_schema = schemas.Object(properties=dict(
x=schemas.Integer(),
y=schemas.Number(),
z=schemas.String()
))
service = mock.MagicMock()
service.request_schemas = {}
BaseService.add_request_schema(
service, 'post', input_schema, description='test')
request_schema = service.request_schemas['POST']
self.assertIsInstance(request_schema, schemas.RequestSchema)
self.assertEqual(request_schema.description, 'test')
self.assertIs(request_schema.api_obj, input_schema)
def test_response_schema(self):
model = mock.Mock()
model_name = 'my-test-model'
api_version = 'v1.2'
mock_additional_checks = mock.Mock()
prediction_schema = schemas.Object(properties=dict(
prediction=schemas.Number(),
confidence=schemas.Number(),
))
with mock.patch('porter.services.BaseService.add_response_schema') as mock_add_response_schema:
prediction_service = PredictionService(
model=model,
name=model_name,
api_version=api_version,
meta={},
preprocessor=None,
postprocessor=None,
batch_prediction=False,
prediction_schema=prediction_schema,
)
response_schema = prediction_service.response_schema
response = dict(
model_context=dict(
api_version=api_version,
model_meta={},
model_name=model_name),
predictions=dict(
id=1,
prediction=dict(prediction=3.14, confidence=2.72)),
request_id='abcdefg')
response_schema.validate(response)
response = dict(
model_context=dict(
api_version=1,
model_meta={},
model_name=model_name),
predictions=dict(
id=1,
prediction=dict(prediction=3.14, confidence=2.72)),
request_id='abcdefg')
with self.assertRaisesRegex(ValueError, 'data.model_context.api_version must be string'):
response_schema.validate(response)
def test_add_response_schema(self):
output_schema = schemas.Object(properties=dict(
prediction=schemas.Number(),
confidence=schemas.Number(),
))
service = mock.MagicMock()
service.response_schemas = {}
BaseService.add_response_schema(
service, 'post', 200, output_schema, description='test')
# make sure there's now one corresponding element
n = 0
for schema in service.response_schemas['POST']:
if schema.status_code == 200:
n += 1
response_schema = schema
self.assertEqual(n, 1)
# check properties
self.assertIsInstance(response_schema, schemas.ResponseSchema)
self.assertEqual(response_schema.description, 'test')
self.assertIs(response_schema.api_obj, output_schema)
def test_get_post_data_validation(self, mock_request_json):
# this test also implicitly covers BaseService.get_post_data
mock_model = mock.Mock()
mock_model.predict.return_value = []
mock_name = mock_version = mock.MagicMock()
feature_schema = schemas.Object(properties=dict(x=schemas.Integer()))
prediction_service = PredictionService(
model=mock_model,
name=mock_name,
api_version=mock_version,
meta={},
preprocessor=None,
postprocessor=None,
batch_prediction=True,
feature_schema=feature_schema,
additional_checks=None
)
# Succeed
mock_request_json.return_value = [{'id': 1, 'x': 37}]
prediction_service.get_post_data()
# Succeed
mock_request_json.return_value = [{'id': 1, 'x': 3.7}]
prediction_service.get_post_data()
# Fail
prediction_service = PredictionService(
model=mock_model,
name=mock_name,
api_version=mock_version + 1,
meta={},
preprocessor=None,
postprocessor=None,
batch_prediction=True,
feature_schema=feature_schema,
validate_request_data=True,
additional_checks=None)
with self.assertRaises(werkzeug_exc.UnprocessableEntity):
prediction_service.get_post_data()
def setUpClass(cls):
# define objects for model 1
class Preprocessor1(BasePreProcessor):
def process(self, X):
X = X.copy() # silence SettingWithCopyWarning
X['feature2'] = X.feature2.astype(str)
return X
class Model1(BaseModel):
feature2_map = {str(x + 1): x for x in range(5)}
def predict(self, X):
return X['feature1'] * X.feature2.map(self.feature2_map)
class Postprocessor1(BasePostProcessor):
def process(self, X_input, X_preprocessed, predictions):
return predictions * -1
feature_schema1 = sc.Object(properties={
'feature1': sc.Number(),
'feature2': sc.Number(),
})
# define objects for model 2
class Preprocessor2(BasePreProcessor):
def process(self, X):
X['feature3'] = range(len(X))
return X
class Model2(BaseModel):
def predict(self, X):
return X['feature1'] + X['feature3']
feature_schema2 = sc.Object(properties={'feature1': sc.Number()})
def user_check(X):
if (X.feature1 == 0).any():
raise exc.UnprocessableEntity
# define objects for model 3
class Model3(BaseModel):
def predict(self, X):
return X['feature1'] * -1
feature_schema3 = sc.Object(properties={'feature1': sc.Number()})
wrong_prediction_schema3 = sc.Number(additional_params=dict(minimum=0))
cls.prediction_service_error = E = Exception(
'this mock service failed during prediction')
class ModelFailing(BaseModel):
def predict(self, X):
raise E
# define configs and add services to app
prediction_service1 = PredictionService(model=Model1(),
name='a-model',
api_version='v0',
action='predict',
preprocessor=Preprocessor1(),
postprocessor=Postprocessor1(),
feature_schema=feature_schema1,
validate_request_data=True,
batch_prediction=True)
prediction_service2 = PredictionService(model=Model2(),
name='anotherModel',
api_version='v1',
namespace='n/s/',
preprocessor=Preprocessor2(),
postprocessor=None,
feature_schema=feature_schema2,
validate_request_data=True,
batch_prediction=True,
additional_checks=user_check)
with warnings.catch_warnings():
warnings.simplefilter('ignore')
prediction_service3 = PredictionService(
model=Model3(),
name='model-3',
api_version='v0.0-alpha',
preprocessor=None,
postprocessor=None,
feature_schema=feature_schema3,
validate_request_data=True,
validate_response_data=True,
batch_prediction=False,
meta={
'algorithm': 'randomforest',
'lasttrained': 1
})
prediction_service4 = PredictionService(
model=Model3(),
name='model-4',
api_version='v0.0-alpha',
preprocessor=None,
postprocessor=None,
feature_schema=feature_schema3,
validate_request_data=True,
validate_response_data=True,
batch_prediction=False,
meta={
'algorithm': 'randomforest',
'lasttrained': 1
})
prediction_service5 = PredictionService(
model=Model3(),
name='model-5',
api_version='v0.0-alpha',
preprocessor=None,
postprocessor=None,
feature_schema=feature_schema3,
prediction_schema=wrong_prediction_schema3,
validate_request_data=True,
validate_response_data=True,
batch_prediction=False,
meta={
'algorithm': 'randomforest',
'lasttrained': 1
})
prediction_service_failing = PredictionService(
model=ModelFailing(),
name='failing-model',
api_version='v1',
action='fail',
)
cls.model_app = ModelApp([
prediction_service1,
prediction_service2,
prediction_service3,
prediction_service4,
prediction_service5,
prediction_service_failing,
])
cls.app = cls.model_app.app.test_client()
