Esempio n. 1
0
    def ListDeployments(self, request, context=None):
        try:
            namespace = request.namespace or self.default_namespace
            deployment_pb_list = self.deployment_store.list(
                namespace=namespace,
                labels_query=request.labels_query,
                offset=request.offset,
                limit=request.limit,
                operator=request.operator,
                order_by=request.order_by,
                ascending_order=request.ascending_order,
            )

            return ListDeploymentsResponse(status=Status.OK(),
                                           deployments=deployment_pb_list)
        except BentoMLException as e:
            logger.error("RPC ERROR ListDeployments: %s", e)
            return DeleteDeploymentResponse(status=e.status_proto)
        except Exception as e:  # pylint: disable=broad-except
            logger.error("RPC ERROR ListDeployments: %s", e)
            return DeleteDeploymentResponse(status=Status.INTERNAL())
Esempio n. 2
0
        def ListBento(self, request, context=None):
            try:
                # TODO: validate request
                bento_metadata_pb_list = self.bento_metadata_store.list(
                    bento_name=request.bento_name,
                    offset=request.offset,
                    limit=request.limit,
                    order_by=request.order_by,
                    label_selectors=request.label_selectors,
                    ascending_order=request.ascending_order,
                )

                return ListBentoResponse(
                    status=Status.OK(), bentos=bento_metadata_pb_list
                )
            except BentoMLException as e:
                logger.error("RPC ERROR ListBento: %s", e)
                return ListBentoResponse(status=e.status_proto)
            except Exception as e:  # pylint: disable=broad-except
                logger.error("RPC ERROR ListBento: %s", e)
                return ListBentoResponse(status=Status.INTERNAL())
Esempio n. 3
0
 def GetDeployment(self, request, context=None):
     try:
         request.namespace = request.namespace or self.default_namespace
         deployment_pb = self.deployment_store.get(
             request.deployment_name, request.namespace
         )
         if deployment_pb:
             return GetDeploymentResponse(
                 status=Status.OK(), deployment=deployment_pb
             )
         else:
             return GetDeploymentResponse(
                 status=Status.NOT_FOUND(
                     'Deployment "{}" in namespace "{}" not found'.format(
                         request.deployment_name, request.namespace
                     )
                 )
             )
     except BentoMLException as e:
         logger.error("INTERNAL ERROR: %s", e)
         return GetDeploymentResponse(status=Status.INTERNAL(str(e)))
 def GetBento(self, request, context=None):
     with self.db.create_session() as sess:
         try:
             # TODO: validate request
             bento_pb = self.db.metadata_store.get(
                 sess, request.bento_name, request.bento_version
             )
             if bento_pb:
                 if request.bento_version.lower() == 'latest':
                     logger.info(
                         'Getting latest version %s:%s',
                         request.bento_name,
                         bento_pb.version,
                     )
                 if bento_pb.uri.type == BentoUri.S3:
                     bento_pb.uri.s3_presigned_url = self.repo.get(
                         bento_pb.name, bento_pb.version
                     )
                 elif bento_pb.uri.type == BentoUri.GCS:
                     bento_pb.uri.gcs_presigned_url = self.repo.get(
                         bento_pb.name, bento_pb.version
                     )
                 return GetBentoResponse(status=Status.OK(), bento=bento_pb)
             else:
                 return GetBentoResponse(
                     status=Status.NOT_FOUND(
                         "BentoService `{}:{}` is not found".format(
                             request.bento_name, request.bento_version
                         )
                     )
                 )
         except BentoMLException as e:
             logger.error("RPC ERROR GetBento: %s", e)
             return GetBentoResponse(status=e.status_proto)
         except Exception as e:  # pylint: disable=broad-except
             logger.error("RPC ERROR GetBento: %s", e)
             return GetBentoResponse(status=Status.INTERNAL())
Esempio n. 5
0
    def DescribeDeployment(self, request, context=None):
        try:
            request.namespace = request.namespace or self.default_namespace
            deployment_pb = self.deployment_store.get(request.deployment_name,
                                                      request.namespace)

            if deployment_pb:
                operator = get_deployment_operator(deployment_pb)
                response = operator.describe(deployment_pb, self.repo)

                if response.status.status_code == status_pb2.Status.OK:
                    with self.deployment_store.update_deployment(
                            request.deployment_name,
                            request.namespace) as deployment:
                        deployment.state = ProtoMessageToDict(response.state)

                return response
            else:
                return DescribeDeploymentResponse(status=Status.NOT_FOUND(
                    'Deployment "{}" in namespace "{}" not found'.format(
                        request.deployment_name, request.namespace)))
        except BentoMLException as e:
            logger.error("INTERNAL ERROR: %s", e)
            return DescribeDeploymentResponse(Status.INTERNAL(str(e)))
Esempio n. 6
0
        def DescribeDeployment(self, request, context=None):
            deployment_id = f"{request.deployment_name}_{request.namespace}"
            with lock(self.db, [(deployment_id, LockType.READ)]) as (
                    sess,
                    _,
            ):
                try:
                    request.namespace = request.namespace or self.default_namespace
                    deployment_pb = self.db.deployment_store.get(
                        sess, request.deployment_name, request.namespace)

                    if deployment_pb:
                        operator = get_deployment_operator(self, deployment_pb)

                        response = operator.describe(deployment_pb)

                        if response.status.status_code == status_pb2.Status.OK:
                            with self.db.deployment_store.update_deployment(
                                    sess, request.deployment_name,
                                    request.namespace) as deployment:
                                deployment.state = ProtoMessageToDict(
                                    response.state)

                        return response
                    else:
                        return DescribeDeploymentResponse(
                            status=Status.NOT_FOUND(
                                'Deployment "{}" in namespace "{}" not found'.
                                format(request.deployment_name,
                                       request.namespace)))
                except BentoMLException as e:
                    logger.error("RPC ERROR DescribeDeployment: %s", e)
                    return DeleteDeploymentResponse(status=e.status_proto)
                except Exception as e:  # pylint: disable=broad-except
                    logger.error("RPC ERROR DescribeDeployment: %s", e)
                    return DeleteDeploymentResponse(status=Status.INTERNAL())
Esempio n. 7
0
 def GetDeployment(self, request, context=None):
     deployment_id = f"{request.deployment_name}_{request.namespace}"
     with lock(self.db, [(deployment_id, LockType.READ)]) as (
             sess,
             _,
     ):
         try:
             request.namespace = request.namespace or self.default_namespace
             deployment_pb = self.db.deployment_store.get(
                 sess, request.deployment_name, request.namespace)
             if deployment_pb:
                 return GetDeploymentResponse(status=Status.OK(),
                                              deployment=deployment_pb)
             else:
                 return GetDeploymentResponse(status=Status.NOT_FOUND(
                     'Deployment "{}" in namespace "{}" not found'.
                     format(request.deployment_name,
                            request.namespace)))
         except BentoMLException as e:
             logger.error("RPC ERROR GetDeployment: %s", e)
             return GetDeploymentResponse(status=e.status_proto)
         except Exception as e:  # pylint: disable=broad-except
             logger.error("RPC ERROR GetDeployment: %s", e)
             return GetDeploymentResponse(status=Status.INTERNAL())
Esempio n. 8
0
    def describe(self, deployment_pb):
        try:
            deployment_spec = deployment_pb.spec
            lambda_deployment_config = deployment_spec.aws_lambda_operator_config
            lambda_deployment_config.region = (lambda_deployment_config.region
                                               or get_default_aws_region())
            if not lambda_deployment_config.region:
                raise InvalidArgument('AWS region is missing')

            bento_pb = self.yatai_service.GetBento(
                GetBentoRequest(
                    bento_name=deployment_spec.bento_name,
                    bento_version=deployment_spec.bento_version,
                ))
            bento_service_metadata = bento_pb.bento.bento_service_metadata
            api_names = ([lambda_deployment_config.api_name]
                         if lambda_deployment_config.api_name else
                         [api.name for api in bento_service_metadata.apis])

            try:
                cf_client = boto3.client('cloudformation',
                                         lambda_deployment_config.region)
                stack_name = generate_aws_compatible_string(
                    '{ns}-{name}'.format(ns=deployment_pb.namespace,
                                         name=deployment_pb.name))
                cloud_formation_stack_result = cf_client.describe_stacks(
                    StackName=stack_name)
                stack_result = cloud_formation_stack_result.get('Stacks')[0]
                # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/\
                # using-cfn-describing-stacks.html
                success_status = ['CREATE_COMPLETE', 'UPDATE_COMPLETE']
                if stack_result['StackStatus'] in success_status:
                    if stack_result.get('Outputs'):
                        outputs = stack_result['Outputs']
                    else:
                        return DescribeDeploymentResponse(
                            status=Status.ABORTED(
                                '"Outputs" field is not present'),
                            state=DeploymentState(
                                state=DeploymentState.ERROR,
                                error_message='"Outputs" field is not present',
                            ),
                        )
                elif stack_result[
                        'StackStatus'] in FAILED_CLOUDFORMATION_STACK_STATUS:
                    state = DeploymentState(state=DeploymentState.FAILED)
                    state.timestamp.GetCurrentTime()
                    return DescribeDeploymentResponse(status=Status.OK(),
                                                      state=state)
                else:
                    state = DeploymentState(state=DeploymentState.PENDING)
                    state.timestamp.GetCurrentTime()
                    return DescribeDeploymentResponse(status=Status.OK(),
                                                      state=state)
            except Exception as error:  # pylint: disable=broad-except
                state = DeploymentState(state=DeploymentState.ERROR,
                                        error_message=str(error))
                state.timestamp.GetCurrentTime()
                return DescribeDeploymentResponse(status=Status.INTERNAL(
                    str(error)),
                                                  state=state)
            outputs = {o['OutputKey']: o['OutputValue'] for o in outputs}
            info_json = {}

            if 'EndpointUrl' in outputs:
                info_json['endpoints'] = [
                    outputs['EndpointUrl'] + '/' + api_name
                    for api_name in api_names
                ]
            if 'S3Bucket' in outputs:
                info_json['s3_bucket'] = outputs['S3Bucket']

            state = DeploymentState(state=DeploymentState.RUNNING,
                                    info_json=json.dumps(info_json))
            state.timestamp.GetCurrentTime()
            return DescribeDeploymentResponse(status=Status.OK(), state=state)
        except BentoMLException as error:
            return DescribeDeploymentResponse(status=error.status_proto)
Esempio n. 9
0
def create_deployment(
    deployment_name,
    namespace,
    bento_name,
    bento_version,
    platform,
    operator_spec,
    labels=None,
    annotations=None,
    yatai_service=None,
):
    if yatai_service is None:
        from bentoml.yatai import get_yatai_service

        yatai_service = get_yatai_service()

    try:
        # Make sure there is no active deployment with the same deployment name
        get_deployment_pb = yatai_service.GetDeployment(
            GetDeploymentRequest(deployment_name=deployment_name, namespace=namespace)
        )
        if get_deployment_pb.status.status_code == status_pb2.Status.OK:
            raise BentoMLDeploymentException(
                'Deployment "{name}" already existed, use Update or Apply for updating'
                'existing deployment, or create the deployment with a different name or'
                'under a different deployment namespace'.format(name=deployment_name)
            )
        if get_deployment_pb.status.status_code != status_pb2.Status.NOT_FOUND:
            raise BentoMLDeploymentException(
                'Failed accesing YataiService deployment store. {error_code}:'
                '{error_message}'.format(
                    error_code=Status.Name(get_deployment_pb.status.status_code),
                    error_message=get_deployment_pb.status.error_message,
                )
            )

        deployment_dict = {
            "name": deployment_name,
            "namespace": namespace or config().get('deployment', 'default_namespace'),
            "labels": labels,
            "annotations": annotations,
            "spec": {
                "bento_name": bento_name,
                "bento_version": bento_version,
                "operator": platform,
            },
        }

        operator = platform.replace('-', '_').upper()
        try:
            operator_value = DeploymentSpec.DeploymentOperator.Value(operator)
        except ValueError:
            return ApplyDeploymentResponse(
                status=Status.INVALID_ARGUMENT('Invalid platform "{}"'.format(platform))
            )
        if operator_value == DeploymentSpec.AWS_SAGEMAKER:
            deployment_dict['spec']['sagemaker_operator_config'] = {
                'region': operator_spec.get('region')
                or config().get('aws', 'default_region'),
                'instance_count': operator_spec.get('instance_count')
                or config().getint('sagemaker', 'default_instance_count'),
                'instance_type': operator_spec.get('instance_type')
                or config().get('sagemaker', 'default_instance_type'),
                'api_name': operator_spec.get('api_name', ''),
            }
        elif operator_value == DeploymentSpec.AWS_LAMBDA:
            deployment_dict['spec']['aws_lambda_operator_config'] = {
                'region': operator_spec.get('region')
                or config().get('aws', 'default_region')
            }
            if operator_spec.get('api_name'):
                deployment_dict['spec']['aws_lambda_operator_config'][
                    'api_name'
                ] = operator_spec['api_name']
        elif operator_value == DeploymentSpec.GCP_FCUNTION:
            deployment_dict['spec']['gcp_function_operatorConfig'] = {
                'region': operator_spec.get('region')
                or config().get('google-cloud', 'default_region')
            }
            if operator_spec.get('api_name'):
                deployment_dict['spec']['gcp_function_operator_config'][
                    'api_name'
                ] = operator_spec['api_name']
        elif operator_value == DeploymentSpec.KUBERNETES:
            deployment_dict['spec']['kubernetes_operator_config'] = {
                'kube_namespace': operator_spec.get('kube_namespace', ''),
                'replicas': operator_spec.get('replicas', 0),
                'service_name': operator_spec.get('service_name', ''),
                'service_type': operator_spec.get('service_type', ''),
            }
        else:
            raise BentoMLDeploymentException(
                'Platform "{}" is not supported in the current version of '
                'BentoML'.format(platform)
            )

        return apply_deployment(deployment_dict, yatai_service)
    except BentoMLException as error:
        return ApplyDeploymentResponse(status=Status.INTERNAL(str(error)))
Esempio n. 10
0
    def ApplyDeployment(self, request, context=None):
        try:
            # apply default namespace if not set
            request.deployment.namespace = (request.deployment.namespace
                                            or self.default_namespace)

            validation_errors = validate_deployment_pb(request.deployment)
            if validation_errors:
                raise InvalidArgument(
                    'Failed to validate deployment. {errors}'.format(
                        errors=validation_errors))

            previous_deployment = self.deployment_store.get(
                request.deployment.name, request.deployment.namespace)
            if not previous_deployment:
                request.deployment.created_at.GetCurrentTime()
            request.deployment.last_updated_at.GetCurrentTime()

            self.deployment_store.insert_or_update(request.deployment)
            # find deployment operator based on deployment spec
            operator = get_deployment_operator(self, request.deployment)

            # deploying to target platform
            if previous_deployment:
                response = operator.update(request.deployment,
                                           previous_deployment)
            else:
                response = operator.add(request.deployment)

            if response.status.status_code == status_pb2.Status.OK:
                # update deployment state
                if response and response.deployment:
                    self.deployment_store.insert_or_update(response.deployment)
                else:
                    raise BentoMLException(
                        "DeploymentOperator Internal Error: failed to add or update "
                        "deployment metadata to database")
                logger.info(
                    "ApplyDeployment (%s, namespace %s) succeeded",
                    request.deployment.name,
                    request.deployment.namespace,
                )
            else:
                if not previous_deployment:
                    # When failed to create the deployment, delete it from active
                    # deployments records
                    self.deployment_store.delete(request.deployment.name,
                                                 request.deployment.namespace)
                logger.debug(
                    "ApplyDeployment (%s, namespace %s) failed: %s",
                    request.deployment.name,
                    request.deployment.namespace,
                    response.status.error_message,
                )

            return response

        except BentoMLException as e:
            logger.error("RPC ERROR ApplyDeployment: %s", e)
            return ApplyDeploymentResponse(status=e.status_proto)
        except Exception as e:
            logger.error("URPC ERROR ApplyDeployment: %s", e)
            return ApplyDeploymentResponse(status=Status.INTERNAL(str(e)))
Esempio n. 11
0
        def DeleteDeployment(self, request, context=None):
            try:
                request.namespace = request.namespace or self.default_namespace
                deployment_pb = self.deployment_store.get(
                    request.deployment_name, request.namespace
                )

                if deployment_pb:
                    # find deployment operator based on deployment spec
                    operator = get_deployment_operator(self, deployment_pb)

                    # executing deployment deletion
                    response = operator.delete(deployment_pb)

                    # if delete successful, remove it from active deployment records
                    # table
                    if response.status.status_code == status_pb2.Status.OK:
                        track_deployment_delete(
                            deployment_pb.spec.operator, deployment_pb.created_at
                        )
                        self.deployment_store.delete(
                            request.deployment_name, request.namespace
                        )
                        return response

                    # If force delete flag is True, we will remove the record
                    # from yatai database.
                    if request.force_delete:
                        # Track deployment delete before it vanquishes from deployment
                        # store
                        track_deployment_delete(
                            deployment_pb.spec.operator, deployment_pb.created_at, True
                        )
                        self.deployment_store.delete(
                            request.deployment_name, request.namespace
                        )
                        return DeleteDeploymentResponse(status=Status.OK())

                    if response.status.status_code == status_pb2.Status.NOT_FOUND:
                        modified_message = (
                            'Cloud resources not found, error: {} - it may have been '
                            'deleted manually. Try delete deployment '
                            'with "--force" option to ignore this error '
                            'and force deleting the deployment record'.format(
                                response.status.error_message
                            )
                        )
                        response.status.error_message = modified_message

                    return response
                else:
                    return DeleteDeploymentResponse(
                        status=Status.NOT_FOUND(
                            'Deployment "{}" in namespace "{}" not found'.format(
                                request.deployment_name, request.namespace
                            )
                        )
                    )

            except BentoMLException as e:
                logger.error("RPC ERROR DeleteDeployment: %s", e)
                return DeleteDeploymentResponse(status=e.status_proto)
            except Exception as e:  # pylint: disable=broad-except
                logger.error("RPC ERROR DeleteDeployment: %s", e)
                return DeleteDeploymentResponse(status=Status.INTERNAL(str(e)))
Esempio n. 12
0
    def describe(self, deployment_pb):
        try:
            deployment_spec = deployment_pb.spec
            ec2_deployment_config = deployment_spec.aws_ec2_operator_config
            ec2_deployment_config.region = (ec2_deployment_config.region
                                            or get_default_aws_region())
            if not ec2_deployment_config.region:
                raise InvalidArgument("AWS region is missing")

            bento_pb = self.yatai_service.GetBento(
                GetBentoRequest(
                    bento_name=deployment_spec.bento_name,
                    bento_version=deployment_spec.bento_version,
                ))
            bento_service_metadata = bento_pb.bento.bento_service_metadata
            api_names = [api.name for api in bento_service_metadata.apis]

            deployment_stack_name = generate_aws_compatible_string(
                "btml-stack-{namespace}-{name}".format(
                    namespace=deployment_pb.namespace,
                    name=deployment_pb.name))
            try:
                cf_client = boto3.client("cloudformation",
                                         ec2_deployment_config.region)
                cloudformation_stack_result = cf_client.describe_stacks(
                    StackName=deployment_stack_name)
                stack_result = cloudformation_stack_result.get("Stacks")[0]

                if stack_result.get("Outputs"):
                    outputs = stack_result.get("Outputs")
                else:
                    return DescribeDeploymentResponse(
                        status=Status.ABORTED(
                            '"Outputs" field is not present'),
                        state=DeploymentState(
                            state=DeploymentState.ERROR,
                            error_message='"Outputs" field is not present',
                        ),
                    )

                if stack_result[
                        "StackStatus"] in FAILED_CLOUDFORMATION_STACK_STATUS:
                    state = DeploymentState(state=DeploymentState.FAILED)
                    return DescribeDeploymentResponse(status=Status.OK(),
                                                      state=state)

            except Exception as error:  # pylint: disable=broad-except
                state = DeploymentState(state=DeploymentState.ERROR,
                                        error_message=str(error))
                return DescribeDeploymentResponse(status=Status.INTERNAL(
                    str(error)),
                                                  state=state)

            info_json = {}
            outputs = {o["OutputKey"]: o["OutputValue"] for o in outputs}
            if "AutoScalingGroup" in outputs:
                info_json[
                    "InstanceDetails"] = get_instance_ip_from_scaling_group(
                        [outputs["AutoScalingGroup"]],
                        ec2_deployment_config.region)
                info_json["Endpoints"] = get_endpoints_from_instance_address(
                    info_json["InstanceDetails"], api_names)
            if "S3Bucket" in outputs:
                info_json["S3Bucket"] = outputs["S3Bucket"]
            if "TargetGroup" in outputs:
                info_json["TargetGroup"] = outputs["TargetGroup"]
            if "Url" in outputs:
                info_json["Url"] = outputs["Url"]

            healthy_target = get_healthy_target(outputs["TargetGroup"],
                                                ec2_deployment_config.region)
            if healthy_target:
                deployment_state = DeploymentState.RUNNING
            else:
                deployment_state = DeploymentState.PENDING
            state = DeploymentState(state=deployment_state,
                                    info_json=json.dumps(info_json))
            return DescribeDeploymentResponse(status=Status.OK(), state=state)

        except BentoMLException as error:
            return DescribeDeploymentResponse(status=error.status_proto)
Esempio n. 13
0
        def ContainerizeBento(self, request, context=None):
            bento_id = f"{request.bento_name}_{request.bento_version}"
            with lock(self.db, [(bento_id, LockType.READ)]) as (sess, _):
                try:
                    ensure_docker_available_or_raise()
                    tag = request.tag
                    if tag is None or len(tag) == 0:
                        name = to_valid_docker_image_name(request.bento_name)
                        version = to_valid_docker_image_version(
                            request.bento_version)
                        tag = f"{name}:{version}"
                    if ":" not in tag:
                        version = to_valid_docker_image_version(
                            request.bento_version)
                        tag = f"{tag}:{version}"
                    import docker

                    docker_client = docker.from_env()
                    bento_pb = self.db.metadata_store.get(
                        sess, request.bento_name, request.bento_version)
                    if not bento_pb:
                        raise YataiRepositoryException(
                            f'BentoService '
                            f'{request.bento_name}:{request.bento_version} '
                            f'does not exist')

                    with TempDirectory() as temp_dir:
                        temp_bundle_path = f'{temp_dir}/{bento_pb.name}'
                        bento_service_bundle_path = bento_pb.uri.uri
                        if bento_pb.uri.type == BentoUri.S3:
                            bento_service_bundle_path = self.repo.get(
                                bento_pb.name, bento_pb.version)
                        elif bento_pb.uri.type == BentoUri.GCS:
                            bento_service_bundle_path = self.repo.get(
                                bento_pb.name, bento_pb.version)
                        safe_retrieve(bento_service_bundle_path,
                                      temp_bundle_path)
                        try:
                            docker_client.images.build(
                                path=temp_bundle_path,
                                tag=tag,
                                buildargs=dict(request.build_args),
                            )
                        except (
                                docker.errors.APIError,
                                docker.errors.BuildError,
                        ) as error:
                            logger.error(
                                f'Encounter container building issue: {error}')
                            raise YataiRepositoryException(error)
                        if request.push is True:
                            try:
                                docker_client.images.push(
                                    repository=request.repository, tag=tag)
                            except docker.errors.APIError as error:
                                raise YataiRepositoryException(error)
                        return ContainerizeBentoResponse(status=Status.OK(),
                                                         tag=tag)
                except BentoMLException as e:
                    logger.error(f"RPC ERROR ContainerizeBento: {e}")
                    return ContainerizeBentoResponse(status=e.status_proto)
                except Exception as e:  # pylint: disable=broad-except
                    logger.error(f"RPC ERROR ContainerizeBento: {e}")
                    return ContainerizeBentoResponse(status=Status.INTERNAL(e))
Esempio n. 14
0
 def UploadBento(self, request_iterator, context=None):
     if not is_file_system_repo(self.repo):
         logger.error(
             "UploadBento RPC only works with File System based repository, "
             "for other types of repositories(s3, gcs, minio), "
             "use pre-signed URL for upload")
         return UploadBentoResponse(status=Status.INTERNAL(''))
     try:
         with self.db.create_session() as sess:
             lock_obj = None
             bento_pb = None
             with TempDirectory() as temp_dir:
                 temp_tar_path = os.path.join(
                     temp_dir, f'{uuid.uuid4().hex[:12]}.tar')
                 file = open(temp_tar_path, 'wb+')
                 for request in request_iterator:
                     # Initial request is without bundle
                     if not request.bento_bundle:
                         bento_name = request.bento_name
                         bento_version = request.bento_version
                         bento_pb = self.db.metadata_store.get(
                             sess, bento_name, bento_version)
                         if not bento_pb:
                             result_status = Status.NOT_FOUND(
                                 "BentoService `{}:{}` is not found".
                                 format(bento_name, bento_version))
                             return UploadBentoResponse(
                                 status=result_status)
                         if bento_pb.status:
                             if bento_pb.status.status == UploadStatus.DONE:
                                 return UploadBentoResponse(
                                     status=Status.CANCELLED(
                                         f"Bento bundle `{bento_name}:"
                                         f"{bento_version}` is uploaded"
                                     ))
                             if bento_pb.status.status == UploadStatus.UPLOADING:
                                 return UploadBentoResponse(
                                     status=Status.CANCELLED(
                                         f"Bento bundle `{bento_name}:"
                                         f"{bento_version}` is currently "
                                         f"uploading"))
                         if lock_obj is None:
                             lock_obj = LockStore.acquire(
                                 sess=sess,
                                 lock_type=LockType.WRITE,
                                 resource_id=
                                 f'{bento_name}_{bento_version}',
                                 ttl_min=DEFAULT_TTL_MIN,
                             )
                     else:
                         if (bento_name == request.bento_name
                                 and bento_version
                                 == request.bento_version):
                             file.write(request.bento_bundle)
                         else:
                             lock_obj.release(sess)
                             raise BadInput(
                                 f"Incoming stream request doesn't match "
                                 f"with initial request info "
                                 f"{bento_name}:{bento_version} - "
                                 f"{request.bento_name}:"
                                 f"{request.bento_version}")
                 file.seek(0)
                 with tarfile.open(fileobj=file, mode='r') as tar:
                     tar.extractall(path=bento_pb.uri.uri)
                 upload_status = UploadStatus(status=UploadStatus.DONE)
                 upload_status.updated_at.GetCurrentTime()
                 self.db.metadata_store.update_upload_status(
                     sess, bento_name, bento_version, upload_status)
                 lock_obj.release(sess)
                 return UploadBentoResponse(status=Status.OK())
     except BentoMLException as e:
         logger.error("RPC ERROR UploadBento: %s", e)
         return UploadBentoResponse(status=e.status_proto)
     except Exception as e:  # pylint: disable=broad-except
         logger.error("RPC ERROR UploadBento: %s", e)
         return UploadBentoResponse(status=Status.INTERNAL())
     finally:
         if file is not None:
             file.close()