def test_request_schema_response_schema_uninitialized(self): model = mock.Mock() model_name = 'my-test-model-noschemas' api_version = 'v1' mock_additional_checks = mock.Mock() prediction_schema = schemas.Object(properties=dict( prediction=schemas.Number(), confidence=schemas.Number(), )) prediction_service = PredictionService( model=model, name=model_name, api_version=api_version, meta={}, preprocessor=None, postprocessor=None, ) # request_schema is None if feature_schema is None self.assertIs(prediction_service.request_schema, None) # response_schema has a default response = dict( model_context=dict( api_version=api_version, model_meta={}, model_name=model_name), predictions=[ dict(id=1, prediction=3.14) ], request_id='abcdefg') prediction_service.response_schema.validate(response)
def test__add_prediction_schema_batch(self): model = mock.Mock() model_name = api_version = mock.MagicMock() mock_additional_checks = mock.Mock() prediction_schema = schemas.Object(properties=dict( prediction=schemas.Number(), confidence=schemas.Number(), )) with mock.patch('porter.services.BaseService.add_response_schema') as mock_add_response_schema: prediction_service = PredictionService( model=model, name=model_name, api_version=api_version, meta={}, preprocessor=None, postprocessor=None, batch_prediction=True, prediction_schema=prediction_schema, ) args = mock_add_response_schema.call_args_list[-1][0] self.assertEqual(args[0].upper(), 'POST') self.assertEqual(args[1], 200) response_obj = args[2] self.assertIsInstance(response_obj, schemas.Object) self.assertIn('request_id', response_obj.properties) self.assertIn('model_context', response_obj.properties) self.assertIn('predictions', response_obj.properties) pred_obj = response_obj.properties['predictions'] self.assertIsInstance(pred_obj, schemas.Array) item_obj = pred_obj.item_type self.assertIn('id', item_obj.properties) self.assertIn('prediction', item_obj.properties) pred_schema = item_obj.properties['prediction'] self.assertIn('prediction', pred_schema.properties) self.assertIn('confidence', pred_schema.properties)
def test_request_schema(self): model = mock.Mock() model_name = api_version = mock.MagicMock() mock_additional_checks = mock.Mock() feature_schema = schemas.Object(properties=dict( x=schemas.Integer(), y=schemas.Number(), z=schemas.String(), )) with mock.patch('porter.services.BaseService.add_request_schema') as mock_add_request_schema: prediction_service = PredictionService( model=model, name=model_name, api_version=api_version, meta={}, preprocessor=None, postprocessor=None, batch_prediction=False, feature_schema=feature_schema, ) request_schema = prediction_service.request_schema request = dict(id=1, x=2, y=3.5, z='4') request_schema.validate(request) with self.assertRaisesRegex(ValueError, 'data must contain'): request = dict(x=2, y=3.5, z='4') request_schema.validate(request)
def test__add_feature_schema_batch(self): model = mock.Mock() model_name = api_version = mock.MagicMock() mock_additional_checks = mock.Mock() feature_schema = schemas.Object(properties=dict( x=schemas.Integer(), y=schemas.Number(), z=schemas.String(), )) with mock.patch('porter.services.BaseService.add_request_schema') as mock_add_request_schema: prediction_service = PredictionService( model=model, name=model_name, api_version=api_version, meta={}, preprocessor=None, postprocessor=None, batch_prediction=True, feature_schema=feature_schema, ) args = mock_add_request_schema.call_args_list[0][0] self.assertEqual(args[0].upper(), 'POST') request_obj = args[1] self.assertIsInstance(request_obj, schemas.Array) item_obj = request_obj.item_type self.assertIn('id', item_obj.properties) self.assertIn('x', item_obj.properties) self.assertIn('y', item_obj.properties) self.assertIn('z', item_obj.properties)
def test_response_schema(self): model = mock.Mock() model_name = 'my-test-model' api_version = 'v1.2' mock_additional_checks = mock.Mock() prediction_schema = schemas.Object(properties=dict( prediction=schemas.Number(), confidence=schemas.Number(), )) with mock.patch('porter.services.BaseService.add_response_schema') as mock_add_response_schema: prediction_service = PredictionService( model=model, name=model_name, api_version=api_version, meta={}, preprocessor=None, postprocessor=None, batch_prediction=False, prediction_schema=prediction_schema, ) response_schema = prediction_service.response_schema response = dict( model_context=dict( api_version=api_version, model_meta={}, model_name=model_name), predictions=dict( id=1, prediction=dict(prediction=3.14, confidence=2.72)), request_id='abcdefg') response_schema.validate(response) response = dict( model_context=dict( api_version=1, model_meta={}, model_name=model_name), predictions=dict( id=1, prediction=dict(prediction=3.14, confidence=2.72)), request_id='abcdefg') with self.assertRaisesRegex(ValueError, 'data.model_context.api_version must be string'): response_schema.validate(response)
def test_add_response_schema(self): output_schema = schemas.Object(properties=dict( prediction=schemas.Number(), confidence=schemas.Number(), )) service = mock.MagicMock() service.response_schemas = {} BaseService.add_response_schema( service, 'post', 200, output_schema, description='test') # make sure there's now one corresponding element n = 0 for schema in service.response_schemas['POST']: if schema.status_code == 200: n += 1 response_schema = schema self.assertEqual(n, 1) # check properties self.assertIsInstance(response_schema, schemas.ResponseSchema) self.assertEqual(response_schema.description, 'test') self.assertIs(response_schema.api_obj, output_schema)
def test_add_request_schema(self): input_schema = schemas.Object(properties=dict( x=schemas.Integer(), y=schemas.Number(), z=schemas.String() )) service = mock.MagicMock() service.request_schemas = {} BaseService.add_request_schema( service, 'post', input_schema, description='test') request_schema = service.request_schemas['POST'] self.assertIsInstance(request_schema, schemas.RequestSchema) self.assertEqual(request_schema.description, 'test') self.assertIs(request_schema.api_obj, input_schema)
def setUpClass(cls): service1 = PredictionService( name='service1', api_version='2', model=None, # we're not going to make calls for predictions here feature_schema=sc.Object(properties={ 'a': sc.Integer(), 'b': sc.Integer(), 'c': sc.Number() })) service1 = PredictionService( namespace='ns', name='service2', api_version='1', model=None, # we're not going to make calls for predictions here feature_schema=sc.Object(properties={ 'a': sc.Integer(), 'b': sc.Integer() }))
def sum(x): return np.sum(x).tolist() def prod(x): return np.prod(x).tolist() def check_for_zeros(x): if 0 in x: raise PorterException('input cannot include zeros', code=422) input_schema = sc.Array(item_type=sc.Number(), reference_name='InputSchema') output_schema = sc.Number(reference_name='OutputSchema') service_kw = dict(input_schema=input_schema, output_schema=output_schema, validate_request_data=True) sum_service = FunctionService('sum', sum, name='math', api_version='v1', **service_kw) prod_service = FunctionService('prod', prod, name='math', api_version='v1', additional_checks=check_for_zeros,
def setUpClass(cls): # define objects for model 1 class Preprocessor1(BasePreProcessor): def process(self, X): X = X.copy() # silence SettingWithCopyWarning X['feature2'] = X.feature2.astype(str) return X class Model1(BaseModel): feature2_map = {str(x + 1): x for x in range(5)} def predict(self, X): return X['feature1'] * X.feature2.map(self.feature2_map) class Postprocessor1(BasePostProcessor): def process(self, X_input, X_preprocessed, predictions): return predictions * -1 feature_schema1 = sc.Object(properties={ 'feature1': sc.Number(), 'feature2': sc.Number(), }) # define objects for model 2 class Preprocessor2(BasePreProcessor): def process(self, X): X['feature3'] = range(len(X)) return X class Model2(BaseModel): def predict(self, X): return X['feature1'] + X['feature3'] feature_schema2 = sc.Object(properties={'feature1': sc.Number()}) def user_check(X): if (X.feature1 == 0).any(): raise exc.UnprocessableEntity # define objects for model 3 class Model3(BaseModel): def predict(self, X): return X['feature1'] * -1 feature_schema3 = sc.Object(properties={'feature1': sc.Number()}) wrong_prediction_schema3 = sc.Number(additional_params=dict(minimum=0)) cls.prediction_service_error = E = Exception( 'this mock service failed during prediction') class ModelFailing(BaseModel): def predict(self, X): raise E # define configs and add services to app prediction_service1 = PredictionService(model=Model1(), name='a-model', api_version='v0', action='predict', preprocessor=Preprocessor1(), postprocessor=Postprocessor1(), feature_schema=feature_schema1, validate_request_data=True, batch_prediction=True) prediction_service2 = PredictionService(model=Model2(), name='anotherModel', api_version='v1', namespace='n/s/', preprocessor=Preprocessor2(), postprocessor=None, feature_schema=feature_schema2, validate_request_data=True, batch_prediction=True, additional_checks=user_check) with warnings.catch_warnings(): warnings.simplefilter('ignore') prediction_service3 = PredictionService( model=Model3(), name='model-3', api_version='v0.0-alpha', preprocessor=None, postprocessor=None, feature_schema=feature_schema3, validate_request_data=True, validate_response_data=True, batch_prediction=False, meta={ 'algorithm': 'randomforest', 'lasttrained': 1 }) prediction_service4 = PredictionService( model=Model3(), name='model-4', api_version='v0.0-alpha', preprocessor=None, postprocessor=None, feature_schema=feature_schema3, validate_request_data=True, validate_response_data=True, batch_prediction=False, meta={ 'algorithm': 'randomforest', 'lasttrained': 1 }) prediction_service5 = PredictionService( model=Model3(), name='model-5', api_version='v0.0-alpha', preprocessor=None, postprocessor=None, feature_schema=feature_schema3, prediction_schema=wrong_prediction_schema3, validate_request_data=True, validate_response_data=True, batch_prediction=False, meta={ 'algorithm': 'randomforest', 'lasttrained': 1 }) prediction_service_failing = PredictionService( model=ModelFailing(), name='failing-model', api_version='v1', action='fail', ) cls.model_app = ModelApp([ prediction_service1, prediction_service2, prediction_service3, prediction_service4, prediction_service5, prediction_service_failing, ]) cls.app = cls.model_app.app.test_client()