def add(self, deployment_pb): try: deployment_spec = deployment_pb.spec deployment_spec.aws_lambda_operator_config.region = ( deployment_spec.aws_lambda_operator_config.region or get_default_aws_region()) if not deployment_spec.aws_lambda_operator_config.region: raise InvalidArgument('AWS region is missing') ensure_sam_available_or_raise() ensure_docker_available_or_raise() bento_pb = self.yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, )) if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3): raise BentoMLException( 'BentoML currently not support {} repository'.format( BentoUri.StorageType.Name(bento_pb.bento.uri.type))) return self._add(deployment_pb, bento_pb, bento_pb.bento.uri.uri) except BentoMLException as error: deployment_pb.state.state = DeploymentState.ERROR deployment_pb.state.error_message = f'Error: {str(error)}' return ApplyDeploymentResponse(status=error.status_proto, deployment=deployment_pb)
def temporary_docker_postgres_url(): ensure_docker_available_or_raise() container_name = f'e2e-test-yatai-service-postgres-db-{uuid.uuid4().hex[:6]}' db_url = 'postgresql://*****:*****@localhost:5432/bentoml' command = [ 'docker', 'run', '--rm', '--name', container_name, '-e', 'POSTGRES_PASSWORD=postgres', '-p', '5432:5432', 'postgres', ] docker_proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) wait_for_docker_container_ready( container_name, b'database system is ready to accept connections') from sqlalchemy_utils import create_database create_database(db_url) yield db_url docker_proc.terminate()
def add(self, deployment_pb): try: ensure_docker_available_or_raise() deployment_spec = deployment_pb.spec sagemaker_config = deployment_spec.sagemaker_operator_config if sagemaker_config is None: raise YataiDeploymentException('Sagemaker configuration is missing.') bento_pb = self.yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3): raise BentoMLException( 'BentoML currently not support {} repository'.format( BentoUri.StorageType.Name(bento_pb.bento.uri.type) ) ) return self._add(deployment_pb, bento_pb, bento_pb.bento.uri.uri) except BentoMLException as error: deployment_pb.state.state = DeploymentState.ERROR deployment_pb.state.error_message = ( f'Error creating SageMaker deployment: {str(error)}' ) return ApplyDeploymentResponse( status=error.status_proto, deployment=deployment_pb )
def apply(self, deployment_pb, yatai_service, prev_deployment=None): try: ensure_docker_available_or_raise() deployment_spec = deployment_pb.spec sagemaker_config = deployment_spec.sagemaker_operator_config if sagemaker_config is None: raise BentoMLDeploymentException('Sagemaker configuration is missing.') bento_pb = yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3): raise BentoMLException( 'BentoML currently not support {} repository'.format( bento_pb.bento.uri.type ) ) return self._apply( deployment_pb, bento_pb, yatai_service, bento_pb.bento.uri.uri, prev_deployment, ) except BentoMLException as error: return ApplyDeploymentResponse(status=exception_to_return_status(error))
def update(self, deployment_pb, previous_deployment): try: ensure_sam_available_or_raise() ensure_docker_available_or_raise() deployment_spec = deployment_pb.spec bento_pb = self.yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) if bento_pb.bento.uri.type not in (BentoUri.LOCAL, BentoUri.S3): raise BentoMLException( 'BentoML currently not support {} repository'.format( BentoUri.StorageType.Name(bento_pb.bento.uri.type) ) ) return self._update( deployment_pb, previous_deployment, bento_pb, bento_pb.bento.uri.uri ) except BentoMLException as error: deployment_pb.state.state = DeploymentState.ERROR deployment_pb.state.error_message = f'Error: {str(error)}' return ApplyDeploymentResponse( status=error.status_code, deployment_pb=deployment_pb )
def temporary_yatai_service_url(): ensure_docker_available_or_raise() docker_client = docker.from_env() local_bentoml_repo_path = os.path.abspath( os.path.join(__file__, '..', '..')) docker_tag = f'bentoml/yatai-service:e2e-test-{uuid.uuid4().hex[:6]}' # Note: When set both `custom_context` and `fileobj`, docker api will not use the # `path` provide... docker/api/build.py L138. The solution is create an actual # Dockerfile along with path, instead of fileobj and custom_context. with TempDirectory() as temp_dir: temp_docker_file_path = os.path.join(temp_dir, 'Dockerfile') with open(temp_docker_file_path, 'w') as f: f.write(f"""\ FROM bentoml/yatai-service:{PREV_PYPI_RELEASE_VERSION} ADD . /bentoml-local-repo RUN pip install /bentoml-local-repo """) logger.info('building docker image') docker_client.images.build( path=local_bentoml_repo_path, dockerfile=temp_docker_file_path, tag=docker_tag, ) logger.info('complete build docker image') container_name = f'e2e-test-yatai-service-container-{uuid.uuid4().hex[:6]}' yatai_service_url = 'localhost:50051' command = [ 'docker', 'run', '--rm', '--name', container_name, '-p', '50051:50051', '-p', '3000:3000', docker_tag, '--repo-base-url', '/tmp', ] logger.info(f'Running docker command {" ".join(command)}') docker_proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) wait_for_docker_container_ready( container_name, b'* Starting BentoML YataiService gRPC Server') yield yatai_service_url docker_proc.terminate()
def test_ensure_docker_available_or_raise(): if version_info.major < 3: not_found_error = OSError else: not_found_error = FileNotFoundError with patch('subprocess.check_output', new=lambda x: raise_(not_found_error())): with pytest.raises(BentoMLMissingDependencyException) as error: ensure_docker_available_or_raise() assert str(error.value).startswith('Docker is required')
def temporary_docker_postgres_url(): ensure_docker_available_or_raise() container_name = f'e2e-test-yatai-service-postgres-db-{uuid.uuid4().hex[:6]}' db_url = 'postgresql://*****:*****@localhost:5432/bentoml' command = [ 'docker', 'run', '--rm', '--name', container_name, '-e', 'POSTGRES_PASSWORD=postgres', '-p', '5432:5432', 'postgres', ] docker_proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # wait until postgres db is ready docker_client = docker.from_env() while True: time.sleep(1) container_list = docker_client.containers.list(filters={ 'name': container_name, 'status': 'running' }) logger.info("container_list: " + str(container_list)) if not container_list: continue assert len( container_list) == 1, "should be only one container with name" postgres_container = container_list[0] logger.info("container_log:" + str(postgres_container.logs())) if (b'database system is ready to accept connections' in postgres_container.logs()): break from sqlalchemy_utils import create_database create_database(db_url) yield db_url docker_proc.terminate()
def apply(self, deployment_pb, yatai_service, prev_deployment=None): try: ensure_docker_available_or_raise() deployment_spec = deployment_pb.spec sagemaker_config = deployment_spec.sagemaker_operator_config if sagemaker_config is None: raise BentoMLDeploymentException('Sagemaker configuration is missing.') bento_pb = yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) if bento_pb.bento.uri.type != BentoUri.LOCAL: raise BentoMLException( 'BentoML currently only support local repository' ) else: bento_path = bento_pb.bento.uri.uri ensure_deploy_api_name_exists_in_bento( [api.name for api in bento_pb.bento.bento_service_metadata.apis], [sagemaker_config.api_name], ) sagemaker_client = boto3.client('sagemaker', sagemaker_config.region) with TempDirectory() as temp_dir: sagemaker_project_dir = os.path.jon( temp_dir, deployment_spec.bento_name ) init_sagemaker_project(sagemaker_project_dir, bento_path) ecr_image_path = create_push_docker_image_to_ecr( deployment_spec.bento_name, deployment_spec.bento_version, sagemaker_project_dir, ) execution_role_arn = get_arn_role_from_current_aws_user() model_name = create_sagemaker_model_name( deployment_spec.bento_name, deployment_spec.bento_version ) sagemaker_model_info = { "ModelName": model_name, "PrimaryContainer": { "ContainerHostname": model_name, "Image": ecr_image_path, "Environment": { "API_NAME": sagemaker_config.api_name, "BENTO_SERVER_TIMEOUT": config().get( 'apiserver', 'default_timeout' ), "BENTO_SERVER_WORKERS": config().get( 'apiserver', 'default_gunicorn_workers_count' ), }, }, "ExecutionRoleArn": execution_role_arn, } logger.info("Creating sagemaker model %s", model_name) try: create_model_response = sagemaker_client.create_model( **sagemaker_model_info ) logger.debug("AWS create model response: %s", create_model_response) except ClientError as e: status = _parse_aws_client_exception_or_raise(e) status.error_message = ( 'Failed to create model for SageMaker Deployment: %s', status.error_message, ) return ApplyDeploymentResponse(status=status, deployment=deployment_pb) production_variants = [ { "VariantName": generate_aws_compatible_string( deployment_spec.bento_name ), "ModelName": model_name, "InitialInstanceCount": sagemaker_config.instance_count, "InstanceType": sagemaker_config.instance_type, } ] endpoint_config_name = create_sagemaker_endpoint_config_name( deployment_spec.bento_name, deployment_spec.bento_version ) logger.info( "Creating Sagemaker endpoint %s configuration", endpoint_config_name ) try: create_config_response = sagemaker_client.create_endpoint_config( EndpointConfigName=endpoint_config_name, ProductionVariants=production_variants, ) logger.debug( "AWS create endpoint config response: %s", create_config_response ) except ClientError as e: # create endpoint failed, will remove previously created model cleanup_model_error = _cleanup_sagemaker_model( sagemaker_client, deployment_spec.bento_name, deployment_spec.bento_version, ) if cleanup_model_error: cleanup_model_error.error_message = ( 'Failed to clean up model after unsuccessfully ' 'create endpoint config: %s', cleanup_model_error.error_message, ) return ApplyDeploymentResponse( status=cleanup_model_error, deployment=deployment_pb ) status = _parse_aws_client_exception_or_raise(e) status.error_message = ( 'Failed to create endpoint config for SageMaker deployment: %s', status.error_message, ) return ApplyDeploymentResponse(status=status, deployment=deployment_pb) endpoint_name = generate_aws_compatible_string( deployment_pb.namespace + '-' + deployment_spec.bento_name ) try: if prev_deployment: logger.debug("Updating sagemaker endpoint %s", endpoint_name) update_endpoint_response = sagemaker_client.update_endpoint( EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name, ) logger.debug( "AWS update endpoint response: %s", update_endpoint_response ) else: logger.debug("Creating sagemaker endpoint %s", endpoint_name) create_endpoint_response = sagemaker_client.create_endpoint( EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name, ) logger.debug( "AWS create endpoint response: %s", create_endpoint_response ) except ClientError as e: # create/update endpoint failed, will remove previously created config # and then remove the model cleanup_endpoint_config_error = _cleanup_sagemaker_endpoint_config( client=sagemaker_client, name=deployment_spec.bento_name, version=deployment_spec.bento_version, ) if cleanup_endpoint_config_error: cleanup_endpoint_config_error.error_message = ( 'Failed to clean up endpoint config after unsuccessfully ' 'apply SageMaker deployment: %s', cleanup_endpoint_config_error.error_message, ) return ApplyDeploymentResponse( status=cleanup_endpoint_config_error, deployment=deployment_pb ) cleanup_model_error = _cleanup_sagemaker_model( client=sagemaker_client, name=deployment_spec.bento_name, version=deployment_spec.bento_version, ) if cleanup_model_error: cleanup_model_error.error_message = ( 'Failed to clean up model after unsuccessfully ' 'apply SageMaker deployment: %s', cleanup_model_error.error_message, ) return ApplyDeploymentResponse( status=cleanup_model_error, deployment=deployment_pb ) status = _parse_aws_client_exception_or_raise(e) status.error_message = ( 'Failed to apply SageMaker deployment: %s', status.error_message, ) return ApplyDeploymentResponse(status=status, deployment=deployment_pb) res_deployment_pb = Deployment(state=DeploymentState()) res_deployment_pb.CopyFrom(deployment_pb) return ApplyDeploymentResponse( status=Status.OK(), deployment=res_deployment_pb ) except BentoMLException as error: return ApplyDeploymentResponse(status=exception_to_return_status(error))
def deploy_bentoml( clipper_conn, bundle_path, api_name, model_name=None, labels=None, build_envs=None ): """Deploy bentoml bundle to clipper cluster Args: clipper_conn(clipper_admin.ClipperConnection): Clipper connection instance bundle_path(str): Path to the saved BentomlService bundle. api_name(str): name of the api that will be used as prediction function for clipper cluster model_name(str): Model's name for clipper cluster labels(:obj:`list(str)`, optional): labels for clipper model Returns: tuple: Model name and model version that deployed to clipper """ track("clipper-deploy", {'bento_service_bundle_path': bundle_path}) build_envs = {} if build_envs is None else build_envs # docker is required to build clipper model image ensure_docker_available_or_raise() if not clipper_conn.connected: raise BentoMLException( "No connection to Clipper cluster. CallClipperConnection.connect to " "connect to an existing cluster or ClipperConnnection.start_clipper to " "create a new one" ) bento_service_metadata = load_bento_service_metadata(bundle_path) try: api_metadata = next( (api for api in bento_service_metadata.apis if api.name == api_name) ) except StopIteration: raise BentoMLException( "Can't find API '{}' in BentoService bundle {}".format( api_name, bento_service_metadata.name ) ) if api_metadata.handler_type not in HANDLER_TYPE_TO_INPUT_TYPE: raise BentoMLException( "Only BentoService APIs using ClipperHandler can be deployed to Clipper" ) input_type = HANDLER_TYPE_TO_INPUT_TYPE[api_metadata.handler_type] model_name = model_name or get_clipper_compatiable_string( bento_service_metadata.name + "-" + api_metadata.name ) model_version = get_clipper_compatiable_string(bento_service_metadata.version) with TempDirectory() as tempdir: entry_py_content = CLIPPER_ENTRY.format(api_name=api_name) model_path = os.path.join(tempdir, "bento") shutil.copytree(bundle_path, model_path) with open(os.path.join(tempdir, "clipper_entry.py"), "w") as f: f.write(entry_py_content) if bento_service_metadata.env.python_version.startswith("3.6"): base_image = "clipper/python36-closure-container:0.4.1" elif bento_service_metadata.env.python_version.startswith("2.7"): base_image = "clipper/python-closure-container:0.4.1" else: raise BentoMLException( "Python version {} is not supported in Clipper".format( bento_service_metadata.env.python_version ) ) docker_content = CLIPPER_DOCKERFILE.format( model_name=model_name, model_version=model_version, base_image=base_image, pip_index_url=build_envs.get("PIP_INDEX_URL", ""), pip_trusted_url=build_envs.get("PIP_TRUSTED_HOST", ""), ) with open(os.path.join(tempdir, "Dockerfile-clipper"), "w") as f: f.write(docker_content) docker_api = docker.APIClient() clipper_model_docker_image_tag = "clipper-model-{}:{}".format( bento_service_metadata.name.lower(), bento_service_metadata.version ) for line in docker_api.build( path=tempdir, dockerfile="Dockerfile-clipper", tag=clipper_model_docker_image_tag, ): process_docker_api_line(line) logger.info( "Successfully built docker image %s for Clipper deployment", clipper_model_docker_image_tag, ) clipper_conn.deploy_model( name=model_name, version=model_version, input_type=input_type, image=clipper_model_docker_image_tag, labels=labels, ) track("clipper-deploy-success", {'bento_service_bundle_path': bundle_path}) return model_name, model_version
def apply(self, deployment_pb, yatai_service, prev_deployment=None): try: ensure_docker_available_or_raise() deployment_spec = deployment_pb.spec aws_config = deployment_spec.aws_lambda_operator_config bento_pb = yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) if bento_pb.bento.uri.type != BentoUri.LOCAL: raise BentoMLException( 'BentoML currently only support local repository' ) else: bento_path = bento_pb.bento.uri.uri bento_service_metadata = bento_pb.bento.bento_service_metadata template = 'aws-python3' if version.parse(bento_service_metadata.env.python_version) < version.parse( '3.0.0' ): template = 'aws-python' api_names = ( [aws_config.api_name] if aws_config.api_name else [api.name for api in bento_service_metadata.apis] ) ensure_deploy_api_name_exists_in_bento( [api.name for api in bento_service_metadata.apis], api_names ) with TempDirectory() as serverless_project_dir: init_serverless_project_dir( serverless_project_dir, bento_path, deployment_pb.name, deployment_spec.bento_name, template, ) generate_aws_lambda_handler_py( deployment_spec.bento_name, api_names, serverless_project_dir ) generate_aws_lambda_serverless_config( bento_service_metadata.env.python_version, deployment_pb.name, api_names, serverless_project_dir, aws_config.region, # BentoML deployment namespace is mapping to serverless `stage` # concept stage=deployment_pb.namespace, ) logger.info( 'Installing additional packages: serverless-python-requirements' ) install_serverless_plugin( "serverless-python-requirements", serverless_project_dir ) logger.info('Deploying to AWS Lambda') call_serverless_command(["deploy"], serverless_project_dir) res_deployment_pb = Deployment(state=DeploymentState()) res_deployment_pb.CopyFrom(deployment_pb) state = self.describe(res_deployment_pb, yatai_service).state res_deployment_pb.state.CopyFrom(state) return ApplyDeploymentResponse( status=Status.OK(), deployment=res_deployment_pb ) except BentoMLException as error: return ApplyDeploymentResponse(status=exception_to_return_status(error))