def testBuildRequests_TFServing(self): builder_cls = self._PrepareTFServingRequestBuilder() builder = builder_cls.return_value request_builder.build_requests( model_name='foo', model=self._model, examples=self._examples, request_spec=_make_request_spec({ 'tensorflow_serving': { 'signature_names': ['serving_default'] }, 'split_name': 'eval', 'num_examples': 1 }) ) builder_cls.assert_called_with( model_name='foo', signatures={'serving_default': mock.ANY}) builder.ReadExamplesArtifact.assert_called_with( self._examples, split_name='eval', num_examples=1) builder.BuildRequests.assert_called()
def testEmptyServingBinary(self): # Prepare empty request spec and examples. request_spec = _create_request_spec({}) examples = standard_artifacts.Examples() with self.assertRaisesRegexp(ValueError, 'Invalid RequestSpec'): request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec)
def testInvalidTensorFlowServingRpcKind(self): # Prepare arguments. request_spec = _create_request_spec({ 'tensorflow_serving': { 'rpc_kind': 'TF_SERVING_RPC_KIND_UNSPECIFIED' } }) examples = standard_artifacts.Examples() with self.assertRaisesRegexp(ValueError, 'Invalid TensorFlowServingRpcKind'): request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec)
def testTensorFlowServingRegress(self): # Prepare arguments. request_spec = _create_request_spec( {'tensorflow_serving': { 'rpc_kind': 'REGRESS' }}) examples = standard_artifacts.Examples() # Call build_requests. request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec) # Check RequestBuilder calls. self.builder_cls.assert_called_with(model_name='foo', max_examples=1) self.builder.BuildRegressionRequests.assert_called()
def testTensorFlowServingClassify(self): # Prepare arguments. request_spec = _create_request_spec( {'tensorflow_serving': { 'rpc_kind': 'CLASSIFY' }}) examples = mock.Mock() # Call build_requests. request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec) # Check RequestBuilder calls. self.builder_cls.assert_called_with(model_name='foo', max_examples=1) self.builder.BuildClassificationRequests.assert_called()
def testBuildRequests_NumberOfRequests(self): result = request_builder.build_requests( model_name='foo', model=self._model, examples=self._examples, request_spec=_make_request_spec({ 'tensorflow_serving': { 'signature_names': ['classification', 'regression'] }, 'split_name': 'eval', 'num_examples': 3 })) # Total 6 requests (3 requests for each signature) self.assertEqual(len(result), 6) self.assertEqual( len([ r for r in result if r.model_spec.signature_name == 'classification' ]), 3) self.assertEqual( len([ r for r in result if r.model_spec.signature_name == 'regression' ]), 3)
def testSignatureName(self): # Prepare arguments. request_spec = _create_request_spec({ 'tensorflow_serving': { 'rpc_kind': 'CLASSIFY', 'signature_name': 'my_signature_name' } }) examples = standard_artifacts.Examples() # Call build_requests. request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec) # Check RequestBuilder calls. self.builder.SetSignatureName.assert_called_with('my_signature_name')
def testMaxExamples(self): # Prepare arguments. request_spec = _create_request_spec({ 'tensorflow_serving': { 'rpc_kind': 'CLASSIFY' }, 'max_examples': 123 }) examples = standard_artifacts.Examples() # Call build_requests. request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec) # Check RequestBuilder calls. self.builder_cls.assert_called_with(model_name='foo', max_examples=123)
def testSplitNames(self): # Prepare arguments. request_spec = _create_request_spec({ 'tensorflow_serving': { 'rpc_kind': 'CLASSIFY' }, 'split_name': 'train' }) examples = standard_artifacts.Examples() # Call build_requests. request_builder.build_requests(model_name='foo', examples=examples, request_spec=request_spec) # Check RequestBuilder calls. self.builder.ReadFromExamplesArtifact.assert_called_with( examples, split_name='train')
def testBuildRequests_DefaultArgument(self): builder_cls = self._PrepareTFServingRequestBuilder() builder = builder_cls.return_value request_builder.build_requests( model_name='foo', model=self._model, examples=self._examples, request_spec=_make_request_spec({ 'tensorflow_serving': { # 'signature_names': ['serving_default'] }, # 'split_name': 'eval', # 'num_examples': 1 })) builder.ReadExamplesArtifact.assert_called_with( self._examples, split_name=None, # Without split_name (will choose any split). num_examples=1) # Default num_examples = 1.
def _Do( self, model: types.Artifact, examples: Optional[types.Artifact], blessing: types.Artifact, serving_spec: infra_validator_pb2.ServingSpec, validation_spec: infra_validator_pb2.ValidationSpec, request_spec: Optional[infra_validator_pb2.RequestSpec], ): if examples and request_spec: logging.info('InfraValidator will be run in LOAD_AND_QUERY mode.') requests = request_builder.build_requests( model_name=serving_spec.model_name, model=model, examples=examples, request_spec=request_spec) else: logging.info('InfraValidator will be run in LOAD_ONLY mode.') requests = [] model_path = self._PrepareModelPath(model, serving_spec) # TODO(jjong): Make logic parallel. all_passed = True for serving_binary in serving_bins.parse_serving_binaries( serving_spec): all_passed &= self._ValidateWithRetry( model_path=model_path, serving_binary=serving_binary, serving_spec=serving_spec, validation_spec=validation_spec, requests=requests) if all_passed: _mark_blessed(blessing) if requests and request_spec.make_warmup: self._CreateWarmupModel(blessing, model_path, warmup_requests=requests) else: _mark_not_blessed(blessing)
def Do(self, input_dict: Dict[Text, List[types.Artifact]], output_dict: Dict[Text, List[types.Artifact]], exec_properties: Dict[Text, Any]) -> None: """Contract for running InfraValidator Executor. Args: input_dict: - `model`: Single `Model` artifact that we're validating. - `examples`: `Examples` artifacts to be used for test requests. output_dict: - `blessing`: Single `InfraBlessing` artifact containing the validated result. It is an empty file with the name either of INFRA_BLESSED or INFRA_NOT_BLESSED. exec_properties: - `serving_spec`: Serialized `ServingSpec` configuration. - `validation_spec`: Serialized `ValidationSpec` configuration. - `request_spec`: Serialized `RequestSpec` configuration. """ self._log_startup(input_dict, output_dict, exec_properties) model = artifact_utils.get_single_instance(input_dict['model']) blessing = artifact_utils.get_single_instance(output_dict['blessing']) serving_spec = infra_validator_pb2.ServingSpec() json_format.Parse(exec_properties['serving_spec'], serving_spec) if not serving_spec.model_name: serving_spec.model_name = _DEFAULT_MODEL_NAME validation_spec = infra_validator_pb2.ValidationSpec() if 'validation_spec' in exec_properties: json_format.Parse(exec_properties['validation_spec'], validation_spec) if not validation_spec.num_tries: validation_spec.num_tries = _DEFAULT_NUM_TRIES if not validation_spec.max_loading_time_seconds: validation_spec.max_loading_time_seconds = _DEFAULT_MAX_LOADING_TIME_SEC if _is_query_mode(input_dict, exec_properties): logging.info('InfraValidator will be run in LOAD_AND_QUERY mode.') request_spec = infra_validator_pb2.RequestSpec() json_format.Parse(exec_properties['request_spec'], request_spec) examples = artifact_utils.get_single_instance( input_dict['examples']) requests = request_builder.build_requests( model_name=os.path.basename( os.path.dirname(path_utils.serving_model_path(model.uri))), examples=examples, request_spec=request_spec) else: logging.info('InfraValidator will be run in LOAD_ONLY mode.') requests = [] model_path = self._PrepareModelPath(model.uri, serving_spec) try: # TODO(jjong): Make logic parallel. all_passed = True for serving_binary in serving_bins.parse_serving_binaries( serving_spec): all_passed &= self._ValidateWithRetry( model_path=model_path, serving_binary=serving_binary, serving_spec=serving_spec, validation_spec=validation_spec, requests=requests) finally: io_utils.delete_dir(self._get_tmp_dir()) if all_passed: _mark_blessed(blessing) else: _mark_not_blessed(blessing)
def Do(self, input_dict: Dict[Text, List[types.Artifact]], output_dict: Dict[Text, List[types.Artifact]], exec_properties: Dict[Text, Any]) -> None: """Contract for running InfraValidator Executor. Args: input_dict: - `model`: Single `Model` artifact that we're validating. - `examples`: `Examples` artifacts to be used for test requests. output_dict: - `blessing`: Single `InfraBlessing` artifact containing the validated result. It is an empty file with the name either of INFRA_BLESSED or INFRA_NOT_BLESSED. exec_properties: - `serving_spec`: Serialized `ServingSpec` configuration. - `validation_spec`: Serialized `ValidationSpec` configuration. - `request_spec`: Serialized `RequestSpec` configuration. """ model = artifact_utils.get_single_instance(input_dict['model']) blessing = artifact_utils.get_single_instance(output_dict['blessing']) serving_spec = infra_validator_pb2.ServingSpec() json_format.Parse(exec_properties['serving_spec'], serving_spec) validation_spec = infra_validator_pb2.ValidationSpec() json_format.Parse(exec_properties['validation_spec'], validation_spec) if _is_query_mode(input_dict, exec_properties): logging.info('InfraValidator will be run in LOAD_AND_QUERY mode.') request_spec = infra_validator_pb2.RequestSpec() json_format.Parse(exec_properties['request_spec'], request_spec) examples = artifact_utils.get_single_instance( input_dict['examples']) requests = request_builder.build_requests( model_name=os.path.basename( os.path.dirname(path_utils.serving_model_path(model.uri))), examples=examples, request_spec=request_spec) else: logging.info('InfraValidator will be run in LOAD_ONLY mode.') requests = [] runners = factory.create_model_server_runners( model=cast(standard_artifacts.Model, model), serving_spec=serving_spec) # TODO(jjong): Make logic parallel. for runner in runners: with _defer_stop(runner): logging.info('Starting %s.', repr(runner)) runner.Start() # Check model is successfully loaded. if not runner.WaitUntilModelAvailable( timeout_secs=validation_spec.max_loading_time_seconds): logging.error( 'Failed to load model in %s; marking as not blessed.', repr(runner)) self._MarkNotBlessed(blessing) continue # Check model can be successfully queried. if requests: try: runner.client.IssueRequests(requests) except (grpc.RpcError, ValueError) as e: logging.error(e) logging.error( 'Failed to query model in %s; marking as not blessed.', repr(runner)) self._MarkNotBlessed(blessing) continue self._MarkBlessedIfSucceeded(blessing)