def DescribeDeployment(self, request, context=None): try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.deployment_store.get( request.deployment_name, request.namespace ) if deployment_pb: operator = get_deployment_operator(deployment_pb) response = operator.describe(deployment_pb, self) if response.status.status_code == status_pb2.Status.OK: with self.deployment_store.update_deployment( request.deployment_name, request.namespace ) as deployment: deployment.state = ProtoMessageToDict(response.state) return response else: return DescribeDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace ) ) ) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return DescribeDeploymentResponse(Status.INTERNAL(str(e)))
def _parse_aws_client_exception_or_raise(e): """parse botocore.exceptions.ClientError into Bento StatusProto We handle two most common errors when deploying to Sagemaker. 1. Authenication issue/invalid access(InvalidSignatureException) 2. resources not found (ValidationException) It will return correlated StatusProto(NOT_FOUND, UNAUTHENTICATED) Args: e: ClientError from botocore.exceptions Returns: StatusProto """ error_response = e.response.get('Error', {}) error_code = error_response.get('Code') error_message = error_response.get('Message', 'Unknown') error_log_message = 'AWS ClientError for {operation}: {code} - {message}'.format( operation=e.operation_name, code=error_code, message=error_message ) if error_code == 'ValidationException': logger.error(error_log_message) return Status.NOT_FOUND(error_response.get('Message', 'Unknown')) elif error_code == 'InvalidSignatureException': logger.error(error_log_message) return Status.UNAUTHENTICATED(error_response.get('Message', 'Unknown')) else: logger.error(error_log_message) raise e
def DescribeDeployment(self, request, context=None): with self.db.create_session() as sess: try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.db.deployment_store.get( sess, request.deployment_name, request.namespace ) if deployment_pb: operator = get_deployment_operator(self, deployment_pb) response = operator.describe(deployment_pb) if response.status.status_code == status_pb2.Status.OK: with self.db.deployment_store.update_deployment( sess, request.deployment_name, request.namespace ) as deployment: deployment.state = ProtoMessageToDict(response.state) return response else: return DescribeDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace ) ) ) except BentoMLException as e: logger.error("RPC ERROR DescribeDeployment: %s", e) return DeleteDeploymentResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR DescribeDeployment: %s", e) return DeleteDeploymentResponse(status=Status.INTERNAL())
def GetBento(self, request, context=None): try: # TODO: validate request bento_pb = self.bento_metadata_store.get(request.bento_name, request.bento_version) if request.bento_version.lower() == 'latest': logger.info( 'Getting latest version %s:%s', request.bento_name, bento_pb.version, ) if bento_pb: if bento_pb.uri.type == BentoUri.S3: bento_pb.uri.s3_presigned_url = self.repo.get( bento_pb.name, bento_pb.version) return GetBentoResponse(status=Status.OK(), bento=bento_pb) else: return GetBentoResponse(status=Status.NOT_FOUND( "BentoService `{}:{}` is not found".format( request.bento_name, request.bento_version))) except BentoMLException as e: logger.error("RPC ERROR GetBento: %s", e) return GetBentoResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR GetBento: %s", e) return GetBentoResponse(status=Status.INTERNAL())
def GetBento(self, request, context=None): try: # TODO: validate request bento_metadata_pb = self.bento_metadata_store.get( request.bento_name, request.bento_version ) if request.bento_version.lower() == 'latest': logger.info( 'Getting latest version %s:%s', request.bento_name, bento_metadata_pb.version, ) if bento_metadata_pb: return GetBentoResponse(status=Status.OK(), bento=bento_metadata_pb) else: return GetBentoResponse( status=Status.NOT_FOUND( "BentoService `{}:{}` is not found".format( request.bento_name, request.bento_version ) ) ) except BentoMLException as e: logger.error("RPC ERROR GetBento: %s", e) return GetBentoResponse(status=e.status_proto)
def GetDeployment(self, request, context=None): with self.db.create_session() as sess: try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.db.deployment_store.get( sess, request.deployment_name, request.namespace ) if deployment_pb: return GetDeploymentResponse( status=Status.OK(), deployment=deployment_pb ) else: return GetDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace ) ) ) except BentoMLException as e: logger.error("RPC ERROR GetDeployment: %s", e) return GetDeploymentResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR GetDeployment: %s", e) return GetDeploymentResponse(status=Status.INTERNAL())
def DeleteDeployment(self, request, context=None): try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.deployment_store.get(request.deployment_name, request.namespace) if deployment_pb: # find deployment operator based on deployment spec operator = get_deployment_operator(deployment_pb) # executing deployment deletion response = operator.delete(deployment_pb, self.repo) # if delete successful, remove it from active deployment records table if response.status.status_code == status_pb2.Status.OK: self.deployment_store.delete(request.deployment_name, request.namespace) return response else: return DeleteDeploymentResponse(status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace))) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return DeleteDeploymentResponse(status=Status.INTERNAL(e))
def delete(self, deployment_pb, yatai_service=None): try: state = self.describe(deployment_pb, yatai_service).state if state.state != DeploymentState.RUNNING: message = ( 'Failed to delete, no active deployment {name}. ' 'The current state is {state}'.format( name=deployment_pb.name, state=DeploymentState.State.Name(state.state), ) ) return DeleteDeploymentResponse(status=Status.ABORTED(message)) deployment_spec = deployment_pb.spec aws_config = deployment_spec.aws_lambda_operator_config bento_pb = yatai_service.GetBento( GetBentoRequest( bento_name=deployment_spec.bento_name, bento_version=deployment_spec.bento_version, ) ) bento_service_metadata = bento_pb.bento.bento_service_metadata # We are not validating api_name, because for delete, you don't # need them. api_names = ( [aws_config.api_name] if aws_config.api_name else [api.name for api in bento_service_metadata.apis] ) with TempDirectory() as serverless_project_dir: generate_aws_lambda_serverless_config( bento_service_metadata.env.python_version, deployment_pb.name, api_names, serverless_project_dir, aws_config.region, # BentoML deployment namespace is mapping to serverless `stage` # concept stage=deployment_pb.namespace, ) response = call_serverless_command(['remove'], serverless_project_dir) stack_name = '{name}-{namespace}'.format( name=deployment_pb.name, namespace=deployment_pb.namespace ) if "Serverless: Stack removal finished..." in response: status = Status.OK() elif "Stack '{}' does not exist".format(stack_name) in response: status = Status.NOT_FOUND( 'Deployment {} not found'.format(stack_name) ) else: status = Status.ABORTED() return DeleteDeploymentResponse(status=status) except BentoMLException as error: return DeleteDeploymentResponse(status=exception_to_return_status(error))
def DeleteDeployment(self, request, context=None): try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.deployment_store.get( request.deployment_name, request.namespace ) if deployment_pb: # find deployment operator based on deployment spec operator = get_deployment_operator(self, deployment_pb) # executing deployment deletion response = operator.delete(deployment_pb) # if delete successful, remove it from active deployment records table if response.status.status_code == status_pb2.Status.OK: self.deployment_store.delete( request.deployment_name, request.namespace ) return response # If force delete flag is True, we will remove the record # from yatai database. if request.force_delete: self.deployment_store.delete( request.deployment_name, request.namespace ) return DeleteDeploymentResponse(status=Status.OK()) if response.status.status_code == status_pb2.Status.NOT_FOUND: modified_message = ( 'Cloud resources not found, error: {} - it may have been ' 'deleted manually. Try delete deployment ' 'with "--force" option to ignore this error ' 'and force deleting the deployment record'.format( response.status.error_message ) ) response.status.error_message = modified_message return response else: return DeleteDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace ) ) ) except BentoMLException as e: logger.error("RPC ERROR DeleteDeployment: %s", e) return DeleteDeploymentResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR DeleteDeployment: %s", e) return DeleteDeploymentResponse(status=Status.INTERNAL(str(e)))
def create_yatai_service_mock(): yatai_service_mock = Mock() yatai_service_mock.ApplyDeployment.return_value = ApplyDeploymentResponse() yatai_service_mock.DeleteDeployment.return_value = DeleteDeploymentResponse() yatai_service_mock.DescribeDeployment.return_value = DescribeDeploymentResponse() yatai_service_mock.GetDeployment.return_value = GetDeploymentResponse( status=Status.NOT_FOUND() ) yatai_service_mock.ListDeployments.return_value = ListDeploymentsResponse() return yatai_service_mock
def GetBento(self, request, context=None): try: # TODO: validate request bento_metadata_pb = self.bento_metadata_store.get( request.bento_name, request.bento_version) if bento_metadata_pb: return GetBentoResponse(status=Status.OK(), bento=bento_metadata_pb) else: return GetBentoResponse(status=Status.NOT_FOUND( "Bento `{}:{}` is not found".format( request.bento_name, request.bento_version))) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return GetBentoResponse(status=Status.INTERNAL(str(e)))
def GetDeployment(self, request, context=None): try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.deployment_store.get(request.deployment_name, request.namespace) if deployment_pb: return GetDeploymentResponse(status=Status.OK(), deployment=deployment_pb) else: return GetDeploymentResponse(status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace))) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return GetDeploymentResponse(status=Status.INTERNAL(str(e)))
def delete(self, deployment_pb, repo=None): state = self.describe(deployment_pb, repo).state if state.state != DeploymentState.RUNNING: message = ('Failed to delete, no active deployment {name}. ' 'The current state is {state}'.format( name=deployment_pb.name, state=DeploymentState.State.Name(state.state), )) return DeleteDeploymentResponse(status=Status.ABORTED(message)) deployment_spec = deployment_pb.spec aws_config = deployment_spec.aws_lambda_operator_config bento_path = repo.get(deployment_spec.bento_name, deployment_spec.bento_version) bento_config = load_bentoml_config(bento_path) with TemporaryServerlessConfig( archive_path=bento_path, deployment_name=deployment_pb.name, region=aws_config.region, stage=deployment_pb.namespace, provider_name='aws', functions=generate_aws_handler_functions_config( bento_config['apis']), ) as tempdir: response = call_serverless_command(['remove'], tempdir) stack_name = '{name}-{namespace}'.format( name=deployment_pb.name, namespace=deployment_pb.namespace) if "Serverless: Stack removal finished..." in response: status = Status.OK() elif "Stack '{}' does not exist".format(stack_name) in response: status = Status.NOT_FOUND('Resource not found') else: status = Status.ABORTED() return DeleteDeploymentResponse(status=status)
def DeleteDeployment(self, request, context=None): deployment_id = f"{request.deployment_name}_{request.namespace}" with lock(self.db, [(deployment_id, LockType.WRITE)]) as ( sess, _, ): try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.db.deployment_store.get( sess, request.deployment_name, request.namespace) if deployment_pb: # find deployment operator based on deployment spec operator = get_deployment_operator(self, deployment_pb) # executing deployment deletion response = operator.delete(deployment_pb) # if delete successful, remove it from active deployment records # table if response.status.status_code == status_pb2.Status.OK: track_deployment_delete( deployment_pb.spec.operator, deployment_pb.created_at) self.db.deployment_store.delete( sess, request.deployment_name, request.namespace) return response # If force delete flag is True, we will remove the record # from yatai database. if request.force_delete: # Track deployment delete before it # vanishes from deployment store track_deployment_delete( deployment_pb.spec.operator, deployment_pb.created_at, True, ) self.db.deployment_store.delete( sess, request.deployment_name, request.namespace) return DeleteDeploymentResponse(status=Status.OK()) if response.status.status_code == status_pb2.Status.NOT_FOUND: modified_message = ( 'Cloud resources not found, error: {} - it ' 'may have been deleted manually. Try delete ' 'deployment with "--force" option to ignore this ' 'error and force deleting the deployment record' .format(response.status.error_message)) response.status.error_message = modified_message return response else: return DeleteDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'. format(request.deployment_name, request.namespace))) except BentoMLException as e: logger.error("RPC ERROR DeleteDeployment: %s", e) return DeleteDeploymentResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR DeleteDeployment: %s", e) return DeleteDeploymentResponse( status=Status.INTERNAL(str(e)))
def UploadBento(self, request_iterator, context=None): if not is_file_system_repo(self.repo): logger.error( "UploadBento RPC only works with File System based repository, " "for other types of repositories(s3, gcs, minio), " "use pre-signed URL for upload") return UploadBentoResponse(status=Status.INTERNAL('')) try: with self.db.create_session() as sess: lock_obj = None bento_pb = None with TempDirectory() as temp_dir: temp_tar_path = os.path.join( temp_dir, f'{uuid.uuid4().hex[:12]}.tar') file = open(temp_tar_path, 'wb+') for request in request_iterator: # Initial request is without bundle if not request.bento_bundle: bento_name = request.bento_name bento_version = request.bento_version bento_pb = self.db.metadata_store.get( sess, bento_name, bento_version) if not bento_pb: result_status = Status.NOT_FOUND( "BentoService `{}:{}` is not found". format(bento_name, bento_version)) return UploadBentoResponse( status=result_status) if bento_pb.status: if bento_pb.status.status == UploadStatus.DONE: return UploadBentoResponse( status=Status.CANCELLED( f"Bento bundle `{bento_name}:" f"{bento_version}` is uploaded" )) if bento_pb.status.status == UploadStatus.UPLOADING: return UploadBentoResponse( status=Status.CANCELLED( f"Bento bundle `{bento_name}:" f"{bento_version}` is currently " f"uploading")) if lock_obj is None: lock_obj = LockStore.acquire( sess=sess, lock_type=LockType.WRITE, resource_id= f'{bento_name}_{bento_version}', ttl_min=DEFAULT_TTL_MIN, ) else: if (bento_name == request.bento_name and bento_version == request.bento_version): file.write(request.bento_bundle) else: lock_obj.release(sess) raise BadInput( f"Incoming stream request doesn't match " f"with initial request info " f"{bento_name}:{bento_version} - " f"{request.bento_name}:" f"{request.bento_version}") file.seek(0) with tarfile.open(fileobj=file, mode='r') as tar: tar.extractall(path=bento_pb.uri.uri) upload_status = UploadStatus(status=UploadStatus.DONE) upload_status.updated_at.GetCurrentTime() self.db.metadata_store.update_upload_status( sess, bento_name, bento_version, upload_status) lock_obj.release(sess) return UploadBentoResponse(status=Status.OK()) except BentoMLException as e: logger.error("RPC ERROR UploadBento: %s", e) return UploadBentoResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR UploadBento: %s", e) return UploadBentoResponse(status=Status.INTERNAL()) finally: if file is not None: file.close()