def update_bento_service_metadata(
self, bento_name, bento_version, bento_service_metadata_pb
):
with create_session(self.sess_maker) as sess:
try:
bento_obj = (
sess.query(Bento)
.filter_by(name=bento_name, version=bento_version, deleted=False)
.one()
)
service_metadata = ProtoMessageToDict(bento_service_metadata_pb)
bento_obj.bento_service_metadata = service_metadata
if service_metadata.get('labels', None) is not None:
bento = (
sess.query(Bento)
.filter_by(name=bento_name, version=bento_version)
.one()
)
add_or_update_labels(
sess, RESOURCE_TYPE.bento, bento.id, service_metadata['labels']
)
except NoResultFound:
raise YataiRepositoryException(
"Bento %s:%s is not found in repository" % bento_name, bento_version
)
def _deployment_pb_to_orm_obj(deployment_pb, deployment_obj=Deployment()):
deployment_obj.name = deployment_pb.name
deployment_obj.namespace = deployment_pb.namespace
deployment_obj.spec = ProtoMessageToDict(deployment_pb.spec)
deployment_obj.state = ProtoMessageToDict(deployment_pb.state)
deployment_obj.labels = dict(deployment_pb.labels)
deployment_obj.annotations = dict(deployment_pb.annotations)
return deployment_obj
def _deployment_pb_to_orm_obj(deployment_pb, deployment_obj=Deployment()):
deployment_obj.name = deployment_pb.name
deployment_obj.namespace = deployment_pb.namespace
deployment_obj.spec = ProtoMessageToDict(deployment_pb.spec)
deployment_obj.state = ProtoMessageToDict(deployment_pb.state)
deployment_obj.annotations = dict(deployment_pb.annotations)
deployment_obj.created_at = deployment_pb.created_at.ToDatetime()
deployment_obj.last_updated_at = deployment_pb.last_updated_at.ToDatetime()
return deployment_obj
def validate_pb_schema(pb, schema):
pb_dict = ProtoMessageToDict(pb)
v = Validator(schema)
if v.validate(pb_dict):
return None
else:
return v.errors
def _get_bento_service_event_properties(bento_service, properties=None):
bento_service_metadata = bento_service.get_bento_service_metadata_pb()
if properties is None:
properties = {}
artifact_types = set()
input_types = set()
output_types = set()
for artifact in bento_service_metadata.artifacts:
artifact_types.add(artifact.artifact_type)
for api in bento_service_metadata.apis:
input_types.add(api.input_type)
output_types.add(api.output_type)
if input_types:
properties["input_types"] = list(input_types)
if output_types:
properties["output_types"] = list(output_types)
if artifact_types:
properties["artifact_types"] = list(artifact_types)
else:
properties["artifact_types"] = ["NO_ARTIFACT"]
env_dict = ProtoMessageToDict(bento_service_metadata.env)
if 'conda_env' in env_dict:
env_dict['conda_env'] = YAML().load(env_dict['conda_env'])
properties['env'] = env_dict
return properties
def validate_deployment_pb(pb):
pb_dict = ProtoMessageToDict(pb)
v = YataiDeploymentValidator(deployment_schema)
if v.validate(pb_dict):
return None
else:
return v.errors
def DescribeDeployment(self, request, context=None):
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.deployment_store.get(
request.deployment_name, request.namespace
)
if deployment_pb:
operator = get_deployment_operator(deployment_pb)
response = operator.describe(deployment_pb, self)
if response.status.status_code == status_pb2.Status.OK:
with self.deployment_store.update_deployment(
request.deployment_name, request.namespace
) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("INTERNAL ERROR: %s", e)
return DescribeDeploymentResponse(Status.INTERNAL(str(e)))
def _bento_service_metadata_to_event_properties(bento_service_metadata,
properties=None):
if properties is None:
properties = {}
artifact_types = set()
handler_types = set()
for artifact in bento_service_metadata.artifacts:
artifact_types.add(artifact.artifact_type)
for api in bento_service_metadata.apis:
handler_types.add(api.handler_type)
if handler_types:
properties["handler_types"] = list(handler_types)
if artifact_types:
properties["artifact_types"] = list(artifact_types)
else:
properties["artifact_types"] = ["NO_ARTIFACT"]
env_dict = ProtoMessageToDict(bento_service_metadata.env)
if 'conda_env' in env_dict:
env_dict['conda_env'] = YAML().load(env_dict['conda_env'])
properties['env'] = env_dict
return properties
class Bento(Base):
__tablename__ = 'bentos'
__table_args__ = tuple(
UniqueConstraint('name', 'version', name='_name_version_uc'))
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
version = Column(String, nullable=False)
# Storage URI for this Bento
uri = Column(String, nullable=False)
# Name is is required for PostgreSQL and any future supported database which
# requires an explicitly named type, or an explicitly named constraint in order to
# generate the type and/or a table that uses it.
uri_type = Column(Enum(*BentoUri.StorageType.keys(), name='uri_type'),
default=BentoUri.UNSET)
# JSON filed mapping directly to BentoServiceMetadata proto message
bento_service_metadata = Column(JSON, nullable=False, default={})
# Time of AddBento call, the time of Bento creation can be found in metadata field
created_at = Column(DateTime, default=datetime.datetime.utcnow)
# latest upload status, JSON message also includes last update timestamp
upload_status = Column(JSON,
nullable=False,
default=ProtoMessageToDict(DEFAULT_UPLOAD_STATUS))
# mark as deleted
deleted = Column(Boolean, default=False)
class Bento(Base):
__tablename__ = 'bentos'
__table_args__ = tuple(UniqueConstraint('name', 'version', name='_name_version_uc'))
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
version = Column(String, nullable=False)
# Storage URI for this Bento
uri = Column(String, nullable=False)
uri_type = Column(Enum(*BentoUri.StorageType.keys()), default=BentoUri.UNSET)
# JSON filed mapping directly to BentoServiceMetadata proto message
bento_service_metadata = Column(JSON, nullable=False, default={})
# Time of AddBento call, the time of Bento creation can be found in metadata field
created_at = Column(DateTime, default=datetime.datetime.utcnow)
# latest upload status, JSON message also includes last update timestamp
upload_status = Column(
JSON, nullable=False, default=ProtoMessageToDict(DEFAULT_UPLOAD_STATUS)
)
# mark as deleted
deleted = Column(Boolean, default=False)
def DescribeDeployment(self, request, context=None):
with self.db.create_session() as sess:
try:
request.namespace = request.namespace or self.default_namespace
deployment_pb = self.db.deployment_store.get(
sess, request.deployment_name, request.namespace
)
if deployment_pb:
operator = get_deployment_operator(self, deployment_pb)
response = operator.describe(deployment_pb)
if response.status.status_code == status_pb2.Status.OK:
with self.db.deployment_store.update_deployment(
sess, request.deployment_name, request.namespace
) as deployment:
deployment.state = ProtoMessageToDict(response.state)
return response
else:
return DescribeDeploymentResponse(
status=Status.NOT_FOUND(
'Deployment "{}" in namespace "{}" not found'.format(
request.deployment_name, request.namespace
)
)
)
except BentoMLException as e:
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=e.status_proto)
except Exception as e: # pylint: disable=broad-except
logger.error("RPC ERROR DescribeDeployment: %s", e)
return DeleteDeploymentResponse(status=Status.INTERNAL())
def info(bundle_path=bundle_path):
"""
List all APIs defined in the BentoService loaded from saved bundle
"""
track_cli('info')
bento_service_metadata_pb = load_bento_service_metadata(bundle_path)
output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2)
_echo(output)
def info(bento=None):
"""
List all APIs defined in the BentoService loaded from saved bundle
"""
saved_bundle_path = resolve_bundle_path(bento, pip_installed_bundle_path)
bento_service_metadata_pb = load_bento_service_metadata(saved_bundle_path)
output = json.dumps(ProtoMessageToDict(bento_service_metadata_pb), indent=2)
_echo(output)
def update_bento_service_metadata(self, bento_name, bento_version,
bento_service_metadata_pb):
with create_session(self.sess_maker) as sess:
try:
bento_obj = (sess.query(Bento).filter_by(name=bento_name,
version=bento_version,
deleted=False).one())
bento_obj.bento_service_metadata = ProtoMessageToDict(
bento_service_metadata_pb)
except NoResultFound:
raise YataiRepositoryException(
"Bento %s:%s is not found in repository" % bento_name,
bento_version)
def update_upload_status(sess, bento_name, bento_version,
upload_status_pb):
try:
bento_obj = (sess.query(Bento).filter_by(name=bento_name,
version=bento_version,
deleted=False).one())
# TODO:
# if bento_obj.upload_status and bento_obj.upload_status.updated_at >
# upload_status_pb.updated_at, update should be ignored
bento_obj.upload_status = ProtoMessageToDict(upload_status_pb)
except NoResultFound:
raise YataiRepositoryException(
"Bento %s:%s is not found in repository" % bento_name,
bento_version)