def test_get_model_status(self, resnet_multiple_batch_sizes, start_server_multi_model, create_grpc_channel): _, ports = start_server_multi_model print("Downloaded model files:", resnet_multiple_batch_sizes) stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), MODEL_SERVICE) request = get_model_status(model_name='resnet', version=1) response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] request = get_model_status(model_name='resnet_bs4') response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK]
def test_get_model_status(self, download_two_models, start_server_multi_model, create_grpc_channel): print("Downloaded model files:", download_two_models) stub = create_grpc_channel('localhost:9001', MODEL_SERVICE) request = get_model_status(model_name='resnet_V1_50', version=1) response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] request = get_model_status(model_name='pnasnet_large') response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK]
def test_get_model_status(self, download_two_model_versions, start_server_multi_model, create_grpc_channel): _, ports = start_server_multi_model print("Downloaded model files:", download_two_model_versions) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), MODEL_SERVICE) versions = [None, 1] for x in range(len(versions)): request = get_model_status(model_name=self.model_name, version=versions[x]) response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] if x == 0: assert len(versions_statuses) == 2 else: assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK]
def test_get_model_status(self, start_server_single_model_from_gc, create_grpc_channel): stub = create_grpc_channel('localhost:9006', MODEL_SERVICE) request = get_model_status(model_name='resnet') response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK]
def test_get_model_status(self, model_version_policy_models, start_server_model_ver_policy, create_grpc_channel, model_name, throw_error): _, ports = start_server_model_ver_policy print("Downloaded model files:", model_version_policy_models) # Connect to grpc service stub = create_grpc_channel('localhost:{}'.format(ports["grpc_port"]), MODEL_SERVICE) versions = [1, 2, 3] for x in range(len(versions)): request = get_model_status(model_name=model_name, version=versions[x]) if not throw_error[x]: response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status version_status = versions_statuses[0] assert version_status.version == versions[x] assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] else: with pytest.raises(Exception) as e: response = stub.GetModelStatus(request, 10) assert "Servable not found for request" in str(e.value) # aggregated results check if model_name == 'all': request = get_model_status(model_name=model_name) response = stub.GetModelStatus(request, 10) versions_statuses = response.model_version_status assert len(versions_statuses) == 3 for version_status in versions_statuses: assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK]
def test_specific_version(self, resnet_multiple_batch_sizes, get_test_dir, start_server_update_flow_specific, create_grpc_channel): resnet, resnet_bs4, resnet_bs8 = resnet_multiple_batch_sizes dir = get_test_dir + '/saved_models/' + 'update/' # ensure model dir is empty at the beginning shutil.rmtree(dir, ignore_errors=True) stub = create_grpc_channel('localhost:9008', PREDICTION_SERVICE) status_stub = create_grpc_channel('localhost:9008', MODEL_SERVICE) resnet_copy_dir = copy_model(resnet, 1, dir) resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir) # This could be replaced with status polling time.sleep(8) # Available versions: 1, 4 print("Getting info about resnet model") model_name = 'resnet' in_name = 'map/TensorArrayStack/TensorArrayGatherV3' out_name = 'softmax_tensor' expected_input_metadata_v1 = { in_name: { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata_v1 = { out_name: { 'dtype': 1, 'shape': [1, 1001] } } request = get_model_metadata(model_name=model_name, version=1) response = stub.GetModelMetadata(request, 10) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata_v1 == input_metadata assert expected_output_metadata_v1 == output_metadata request_latest = get_model_metadata(model_name=model_name) response_latest = stub.GetModelMetadata(request_latest, 10) print("response", response_latest) input_metadata_latest, output_metadata_latest = \ model_metadata_response(response=response_latest) request_v4 = get_model_metadata(model_name=model_name, version=4) response_v4 = stub.GetModelMetadata(request_v4, 10) print("response", response_v4) input_metadata_v4, output_metadata_v4 = model_metadata_response( response=response_latest) assert response_v4.model_spec.name == response_latest.model_spec.name assert input_metadata_v4 == input_metadata_latest assert output_metadata_v4 == output_metadata_latest # Model status check model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 2 for version_status in versions_statuses: assert version_status.version in [1, 4] assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_bs4_copy_dir) resnet_bs8_copy_dir = copy_model(resnet_bs8, 3, dir) time.sleep(10) # Available versions: 1, 3 request_latest = get_model_metadata(model_name=model_name) response_latest = stub.GetModelMetadata(request_latest, 10) print("response", response_latest) input_metadata_latest, output_metadata_latest = \ model_metadata_response(response=response_latest) request_v3 = get_model_metadata(model_name=model_name, version=3) response_v3 = stub.GetModelMetadata(request_v3, 10) print("response", response_v3) input_metadata_v3, output_metadata_v3 = model_metadata_response( response=response_v3) assert response_v3.model_spec.name == response_latest.model_spec.name assert input_metadata_v3 == input_metadata_latest assert output_metadata_v3 == output_metadata_latest # Model status check model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 3 for version_status in versions_statuses: assert version_status.version in [1, 3, 4] if version_status.version == 4: assert version_status.state == ModelVersionState.END assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.END][ErrorCode.OK] elif version_status.version == 1 or version_status.version == 3: assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### # Available versions: 1, 3, 4 resnet_bs4_copy_dir = copy_model(resnet_bs4, 4, dir) time.sleep(10) request_v1 = get_model_metadata(model_name=model_name, version=1) response_v1 = stub.GetModelMetadata(request_v1, 10) input_metadata_v1, output_metadata_v1 = model_metadata_response( response=response_v1) assert model_name == response_v1.model_spec.name assert expected_input_metadata_v1 == input_metadata_v1 assert expected_output_metadata_v1 == output_metadata_v1 expected_input_metadata_v3 = { in_name: { 'dtype': 1, 'shape': [8, 3, 224, 224] } } expected_output_metadata_v3 = { out_name: { 'dtype': 1, 'shape': [8, 1001] } } request_v3 = get_model_metadata(model_name=model_name, version=3) response_v3 = stub.GetModelMetadata(request_v3, 10) input_metadata_v3, output_metadata_v3 = model_metadata_response( response=response_v3) assert model_name == response_v3.model_spec.name assert expected_input_metadata_v3 == input_metadata_v3 assert expected_output_metadata_v3 == output_metadata_v3 expected_input_metadata_v4 = { in_name: { 'dtype': 1, 'shape': [4, 3, 224, 224] } } expected_output_metadata_v4 = { out_name: { 'dtype': 1, 'shape': [4, 1001] } } request_v4 = get_model_metadata(model_name=model_name) response_v4 = stub.GetModelMetadata(request_v4, 10) input_metadata_v4, output_metadata_v4 = model_metadata_response( response=response_v4) assert model_name == response_v4.model_spec.name assert expected_input_metadata_v4 == input_metadata_v4 assert expected_output_metadata_v4 == output_metadata_v4 # Model status check model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 3 for version_status in versions_statuses: assert version_status.version in [1, 3, 4] assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_copy_dir) shutil.rmtree(resnet_bs4_copy_dir) shutil.rmtree(resnet_bs8_copy_dir) time.sleep(10)
def test_latest_version(self, resnet_multiple_batch_sizes, get_test_dir, start_server_update_flow_latest, create_grpc_channel): resnet, resnet_bs4, resnet_bs8 = resnet_multiple_batch_sizes dir = get_test_dir + '/saved_models/' + 'update/' # ensure model dir is empty at the beginning shutil.rmtree(dir, ignore_errors=True) resnet_v1_copy_dir = copy_model(resnet, 1, dir) time.sleep(8) stub = create_grpc_channel('localhost:9007', PREDICTION_SERVICE) status_stub = create_grpc_channel('localhost:9007', MODEL_SERVICE) print("Getting info about resnet model") model_name = 'resnet' in_name = 'map/TensorArrayStack/TensorArrayGatherV3' out_name = 'softmax_tensor' expected_input_metadata_v1 = { in_name: { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata_v1 = { out_name: { 'dtype': 1, 'shape': [1, 1001] } } request = get_model_metadata(model_name=model_name) response = stub.GetModelMetadata(request, 10) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata_v1 == input_metadata assert expected_output_metadata_v1 == output_metadata # Model status check before update model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status version_status = versions_statuses[0] assert len(versions_statuses) == 1 assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### resnet_v2_copy_dir = copy_model(resnet_bs4, 2, dir) time.sleep(10) expected_input_metadata_v2 = { in_name: { 'dtype': 1, 'shape': [4, 3, 224, 224] } } expected_output_metadata_v2 = { out_name: { 'dtype': 1, 'shape': [4, 1001] } } request = get_model_metadata(model_name=model_name) response = stub.GetModelMetadata(request, 10) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata_v2 == input_metadata assert expected_output_metadata_v2 == output_metadata # Model status check after update model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 2 for version_status in versions_statuses: assert version_status.version in [1, 2] if version_status.version == 1: assert version_status.state == ModelVersionState.END assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.END][ErrorCode.OK] elif version_status.version == 2: assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_v1_copy_dir) shutil.rmtree(resnet_v2_copy_dir) time.sleep(10)
def test_specific_version(self, download_two_model_versions, resnet_2_out_model_downloader, get_test_dir, start_server_update_flow_specific, create_grpc_channel): resnet_v1, resnet_v2 = download_two_model_versions resnet_2_out = resnet_2_out_model_downloader dir = get_test_dir + '/saved_models/' + 'update/' stub = create_grpc_channel('localhost:9008', PREDICTION_SERVICE) status_stub = create_grpc_channel('localhost:9008', MODEL_SERVICE) resnet_v1_copy_dir = copy_model(resnet_v1, 1, dir) resnet_2_out_copy_dir = copy_model(resnet_2_out, 4, dir) time.sleep(8) # Available versions: 1, 4 print("Getting info about resnet model") model_name = 'resnet' out_name_v1 = 'resnet_v1_50/predictions/Reshape_1' expected_input_metadata_v1 = { 'input': { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata_v1 = { out_name_v1: { 'dtype': 1, 'shape': [1, 1000] } } request = get_model_metadata(model_name=model_name, version=1) response = stub.GetModelMetadata(request, 10) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata_v1 == input_metadata assert expected_output_metadata_v1 == output_metadata request_latest = get_model_metadata(model_name=model_name) response_latest = stub.GetModelMetadata(request_latest, 10) print("response", response_latest) input_metadata_latest, output_metadata_latest = \ model_metadata_response(response=response_latest) request_v4 = get_model_metadata(model_name=model_name, version=4) response_v4 = stub.GetModelMetadata(request_v4, 10) print("response", response_v4) input_metadata_v4, output_metadata_v4 = model_metadata_response( response=response_latest) assert response_v4.model_spec.name == response_latest.model_spec.name assert input_metadata_v4 == input_metadata_latest assert output_metadata_v4 == output_metadata_latest # Model status check model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 2 for version_status in versions_statuses: assert version_status.version in [1, 4] assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_2_out_copy_dir) resnet_v2_copy_dir = copy_model(resnet_v2, 3, dir) time.sleep(10) # Available versions: 1, 3 request_latest = get_model_metadata(model_name=model_name) response_latest = stub.GetModelMetadata(request_latest, 10) print("response", response_latest) input_metadata_latest, output_metadata_latest = \ model_metadata_response(response=response_latest) request_v3 = get_model_metadata(model_name=model_name, version=3) response_v3 = stub.GetModelMetadata(request_v3, 10) print("response", response_v3) input_metadata_v3, output_metadata_v3 = model_metadata_response( response=response_v3) assert response_v3.model_spec.name == response_latest.model_spec.name assert input_metadata_v3 == input_metadata_latest assert output_metadata_v3 == output_metadata_latest # Model status check model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 3 for version_status in versions_statuses: assert version_status.version in [1, 3, 4] if version_status.version == 4: assert version_status.state == ModelVersionState.END assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.END][ErrorCode.OK] elif version_status.version == 1 or version_status.version == 3: assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### # Available versions: 1, 3, 4 resnet_2_out_copy_dir = copy_model(resnet_2_out, 4, dir) time.sleep(10) request_v1 = get_model_metadata(model_name=model_name, version=1) response_v1 = stub.GetModelMetadata(request_v1, 10) input_metadata_v1, output_metadata_v1 = model_metadata_response( response=response_v1) assert model_name == response_v1.model_spec.name assert expected_input_metadata_v1 == input_metadata_v1 assert expected_output_metadata_v1 == output_metadata_v1 out_name_v3 = 'resnet_v2_50/predictions/Reshape_1' expected_input_metadata_v3 = { 'input': { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata_v3 = { out_name_v3: { 'dtype': 1, 'shape': [1, 1001] } } request_v3 = get_model_metadata(model_name=model_name, version=3) response_v3 = stub.GetModelMetadata(request_v3, 10) input_metadata_v3, output_metadata_v3 = model_metadata_response( response=response_v3) assert model_name == response_v3.model_spec.name assert expected_input_metadata_v3 == input_metadata_v3 assert expected_output_metadata_v3 == output_metadata_v3 expected_input_metadata_v4 = { 'input': { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata_v4 = { 'res5c_branch2c1': { 'dtype': 1, 'shape': [1, 2048, 7, 7] }, 'res5c_branch2c2': { 'dtype': 1, 'shape': [1, 2048, 7, 7] } } request_v4 = get_model_metadata(model_name=model_name) response_v4 = stub.GetModelMetadata(request_v4, 10) input_metadata_v4, output_metadata_v4 = model_metadata_response( response=response_v4) assert model_name == response_v4.model_spec.name assert expected_input_metadata_v4 == input_metadata_v4 assert expected_output_metadata_v4 == output_metadata_v4 # Model status check model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 3 for version_status in versions_statuses: assert version_status.version in [1, 3, 4] assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_v2_copy_dir) shutil.rmtree(resnet_v1_copy_dir) shutil.rmtree(resnet_2_out_copy_dir) time.sleep(10)
def test_latest_version(self, download_two_model_versions, get_test_dir, start_server_update_flow_latest, create_grpc_channel): resnet_v1, resnet_v2 = download_two_model_versions dir = get_test_dir + '/saved_models/' + 'update/' resnet_v1_copy_dir = copy_model(resnet_v1, 1, dir) time.sleep(8) stub = create_grpc_channel('localhost:9007', PREDICTION_SERVICE) status_stub = create_grpc_channel('localhost:9007', MODEL_SERVICE) print("Getting info about resnet model") model_name = 'resnet' out_name = 'resnet_v1_50/predictions/Reshape_1' expected_input_metadata = { 'input': { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata = {out_name: {'dtype': 1, 'shape': [1, 1000]}} request = get_model_metadata(model_name=model_name) response = stub.GetModelMetadata(request, 10) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata == input_metadata assert expected_output_metadata == output_metadata # Model status check before update model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status version_status = versions_statuses[0] assert len(versions_statuses) == 1 assert version_status.version == 1 assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_v1_copy_dir) resnet_v2_copy_dir = copy_model(resnet_v2, 2, dir) time.sleep(10) out_name = 'resnet_v2_50/predictions/Reshape_1' expected_input_metadata = { 'input': { 'dtype': 1, 'shape': [1, 3, 224, 224] } } expected_output_metadata = {out_name: {'dtype': 1, 'shape': [1, 1001]}} request = get_model_metadata(model_name=model_name) response = stub.GetModelMetadata(request, 10) input_metadata, output_metadata = model_metadata_response( response=response) print(output_metadata) assert model_name == response.model_spec.name assert expected_input_metadata == input_metadata assert expected_output_metadata == output_metadata # Model status check after update model_name = 'resnet' request = get_model_status(model_name=model_name) status_response = status_stub.GetModelStatus(request, 10) versions_statuses = status_response.model_version_status assert len(versions_statuses) == 2 for version_status in versions_statuses: assert version_status.version in [1, 2] if version_status.version == 1: assert version_status.state == ModelVersionState.END assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.END][ErrorCode.OK] elif version_status.version == 2: assert version_status.state == ModelVersionState.AVAILABLE assert version_status.status.error_code == ErrorCode.OK assert version_status.status.error_message == _ERROR_MESSAGE[ ModelVersionState.AVAILABLE][ErrorCode.OK] ### shutil.rmtree(resnet_v2_copy_dir) time.sleep(10)