def update_bento_service_metadata( self, bento_name, bento_version, bento_service_metadata_pb ): with create_session(self.sess_maker) as sess: try: bento_obj = ( sess.query(Bento) .filter_by(name=bento_name, version=bento_version, deleted=False) .one() ) service_metadata = ProtoMessageToDict(bento_service_metadata_pb) bento_obj.bento_service_metadata = service_metadata if service_metadata.get('labels', None) is not None: bento = ( sess.query(Bento) .filter_by(name=bento_name, version=bento_version) .one() ) add_or_update_labels( sess, RESOURCE_TYPE.bento, bento.id, service_metadata['labels'] ) except NoResultFound: raise YataiRepositoryException( "Bento %s:%s is not found in repository" % bento_name, bento_version )
def _deployment_pb_to_orm_obj(deployment_pb, deployment_obj=Deployment()): deployment_obj.name = deployment_pb.name deployment_obj.namespace = deployment_pb.namespace deployment_obj.spec = ProtoMessageToDict(deployment_pb.spec) deployment_obj.state = ProtoMessageToDict(deployment_pb.state) deployment_obj.labels = dict(deployment_pb.labels) deployment_obj.annotations = dict(deployment_pb.annotations) return deployment_obj
def _deployment_pb_to_orm_obj(deployment_pb, deployment_obj=Deployment()): deployment_obj.name = deployment_pb.name deployment_obj.namespace = deployment_pb.namespace deployment_obj.spec = ProtoMessageToDict(deployment_pb.spec) deployment_obj.state = ProtoMessageToDict(deployment_pb.state) deployment_obj.annotations = dict(deployment_pb.annotations) deployment_obj.created_at = deployment_pb.created_at.ToDatetime() deployment_obj.last_updated_at = deployment_pb.last_updated_at.ToDatetime() return deployment_obj
def validate_pb_schema(pb, schema): pb_dict = ProtoMessageToDict(pb) v = Validator(schema) if v.validate(pb_dict): return None else: return v.errors
def _get_bento_service_event_properties(bento_service, properties=None): bento_service_metadata = bento_service.get_bento_service_metadata_pb() if properties is None: properties = {} artifact_types = set() input_types = set() output_types = set() for artifact in bento_service_metadata.artifacts: artifact_types.add(artifact.artifact_type) for api in bento_service_metadata.apis: input_types.add(api.input_type) output_types.add(api.output_type) if input_types: properties["input_types"] = list(input_types) if output_types: properties["output_types"] = list(output_types) if artifact_types: properties["artifact_types"] = list(artifact_types) else: properties["artifact_types"] = ["NO_ARTIFACT"] env_dict = ProtoMessageToDict(bento_service_metadata.env) if 'conda_env' in env_dict: env_dict['conda_env'] = YAML().load(env_dict['conda_env']) properties['env'] = env_dict return properties
def validate_deployment_pb(pb): pb_dict = ProtoMessageToDict(pb) v = YataiDeploymentValidator(deployment_schema) if v.validate(pb_dict): return None else: return v.errors
def DescribeDeployment(self, request, context=None): try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.deployment_store.get( request.deployment_name, request.namespace ) if deployment_pb: operator = get_deployment_operator(deployment_pb) response = operator.describe(deployment_pb, self) if response.status.status_code == status_pb2.Status.OK: with self.deployment_store.update_deployment( request.deployment_name, request.namespace ) as deployment: deployment.state = ProtoMessageToDict(response.state) return response else: return DescribeDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace ) ) ) except BentoMLException as e: logger.error("INTERNAL ERROR: %s", e) return DescribeDeploymentResponse(Status.INTERNAL(str(e)))
def _bento_service_metadata_to_event_properties(bento_service_metadata, properties=None): if properties is None: properties = {} artifact_types = set() handler_types = set() for artifact in bento_service_metadata.artifacts: artifact_types.add(artifact.artifact_type) for api in bento_service_metadata.apis: handler_types.add(api.handler_type) if handler_types: properties["handler_types"] = list(handler_types) if artifact_types: properties["artifact_types"] = list(artifact_types) else: properties["artifact_types"] = ["NO_ARTIFACT"] env_dict = ProtoMessageToDict(bento_service_metadata.env) if 'conda_env' in env_dict: env_dict['conda_env'] = YAML().load(env_dict['conda_env']) properties['env'] = env_dict return properties
class Bento(Base): __tablename__ = 'bentos' __table_args__ = tuple( UniqueConstraint('name', 'version', name='_name_version_uc')) id = Column(Integer, primary_key=True) name = Column(String, nullable=False) version = Column(String, nullable=False) # Storage URI for this Bento uri = Column(String, nullable=False) # Name is is required for PostgreSQL and any future supported database which # requires an explicitly named type, or an explicitly named constraint in order to # generate the type and/or a table that uses it. uri_type = Column(Enum(*BentoUri.StorageType.keys(), name='uri_type'), default=BentoUri.UNSET) # JSON filed mapping directly to BentoServiceMetadata proto message bento_service_metadata = Column(JSON, nullable=False, default={}) # Time of AddBento call, the time of Bento creation can be found in metadata field created_at = Column(DateTime, default=datetime.datetime.utcnow) # latest upload status, JSON message also includes last update timestamp upload_status = Column(JSON, nullable=False, default=ProtoMessageToDict(DEFAULT_UPLOAD_STATUS)) # mark as deleted deleted = Column(Boolean, default=False)
class Bento(Base): __tablename__ = 'bentos' __table_args__ = tuple(UniqueConstraint('name', 'version', name='_name_version_uc')) id = Column(Integer, primary_key=True) name = Column(String, nullable=False) version = Column(String, nullable=False) # Storage URI for this Bento uri = Column(String, nullable=False) uri_type = Column(Enum(*BentoUri.StorageType.keys()), default=BentoUri.UNSET) # JSON filed mapping directly to BentoServiceMetadata proto message bento_service_metadata = Column(JSON, nullable=False, default={}) # Time of AddBento call, the time of Bento creation can be found in metadata field created_at = Column(DateTime, default=datetime.datetime.utcnow) # latest upload status, JSON message also includes last update timestamp upload_status = Column( JSON, nullable=False, default=ProtoMessageToDict(DEFAULT_UPLOAD_STATUS) ) # mark as deleted deleted = Column(Boolean, default=False)
def DescribeDeployment(self, request, context=None): with self.db.create_session() as sess: try: request.namespace = request.namespace or self.default_namespace deployment_pb = self.db.deployment_store.get( sess, request.deployment_name, request.namespace ) if deployment_pb: operator = get_deployment_operator(self, deployment_pb) response = operator.describe(deployment_pb) if response.status.status_code == status_pb2.Status.OK: with self.db.deployment_store.update_deployment( sess, request.deployment_name, request.namespace ) as deployment: deployment.state = ProtoMessageToDict(response.state) return response else: return DescribeDeploymentResponse( status=Status.NOT_FOUND( 'Deployment "{}" in namespace "{}" not found'.format( request.deployment_name, request.namespace ) ) ) except BentoMLException as e: logger.error("RPC ERROR DescribeDeployment: %s", e) return DeleteDeploymentResponse(status=e.status_proto) except Exception as e: # pylint: disable=broad-except logger.error("RPC ERROR DescribeDeployment: %s", e) return DeleteDeploymentResponse(status=Status.INTERNAL())
def info(bundle_path=bundle_path): """ List all APIs defined in the BentoService loaded from saved bundle """ track_cli('info') bento_service_metadata_pb = load_bento_service_metadata(bundle_path) output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2) _echo(output)
def info(bento=None): """ List all APIs defined in the BentoService loaded from saved bundle """ saved_bundle_path = resolve_bundle_path(bento, pip_installed_bundle_path) bento_service_metadata_pb = load_bento_service_metadata(saved_bundle_path) output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2) _echo(output)
def update_bento_service_metadata(self, bento_name, bento_version, bento_service_metadata_pb): with create_session(self.sess_maker) as sess: try: bento_obj = (sess.query(Bento).filter_by(name=bento_name, version=bento_version, deleted=False).one()) bento_obj.bento_service_metadata = ProtoMessageToDict( bento_service_metadata_pb) except NoResultFound: raise YataiRepositoryException( "Bento %s:%s is not found in repository" % bento_name, bento_version)
def update_upload_status(sess, bento_name, bento_version, upload_status_pb): try: bento_obj = (sess.query(Bento).filter_by(name=bento_name, version=bento_version, deleted=False).one()) # TODO: # if bento_obj.upload_status and bento_obj.upload_status.updated_at > # upload_status_pb.updated_at, update should be ignored bento_obj.upload_status = ProtoMessageToDict(upload_status_pb) except NoResultFound: raise YataiRepositoryException( "Bento %s:%s is not found in repository" % bento_name, bento_version)