def dangerously_delete(self, bento_name, bento_version):
with create_session(self.sess_maker) as sess:
try:
bento_obj = (sess.query(Bento).filter_by(
name=bento_name, version=bento_version).one())
if not bento_obj.deleted:
raise BentoMLRepositoryException(
"Bento %s:%s has already been deleted" % bento_name,
bento_version,
)
bento_obj.deleted = True
except NoResultFound:
raise BentoMLRepositoryException(
"Bento %s:%s is not found in repository" % bento_name,
bento_version)
def get(self, bento_name, bento_version):
saved_path = os.path.join(self.base_path, bento_name, bento_version)
if not os.path.exists(saved_path):
raise BentoMLRepositoryException(
"Bento {}:{} not found in target repository".format(
bento_name, bento_version))
return saved_path
def __init__(self, base_url):
if not os.path.exists(base_url):
raise BentoMLRepositoryException(
"Local path '{}' not found when "
"initializing local Bento repository")
self.base_path = base_url
self.uri_type = BentoUri.LOCAL
def get(self, bento_name, bento_version=None):
if bento_version is None or bento_version == 'latest':
bento_version = self.get_latest_version(bento_name)
saved_path = os.path.join(self.base_path, bento_name, bento_version)
if not os.path.exists(saved_path):
raise BentoMLRepositoryException(
"Bento {}:{} not found in target repository".format(
bento_name, bento_version))
return saved_path
def update_bento_service_metadata(self, bento_name, bento_version,
bento_service_metadata_pb):
with create_session(self.sess_maker) as sess:
try:
bento_obj = (sess.query(Bento).filter_by(name=bento_name,
version=bento_version,
deleted=False).one())
bento_obj.bento_service_metadata = ProtoMessageToDict(
bento_service_metadata_pb)
except NoResultFound:
raise BentoMLRepositoryException(
"Bento %s:%s is not found in repository" % bento_name,
bento_version)
def dangerously_delete(self, bento_name, bento_version):
# Remove s3 path containing related Bento files
object_name = self._get_object_name(bento_name, bento_version)
try:
response = self.s3_client.delete_object(Bucket=self.bucket, Key=object_name)
DELETE_MARKER = 'DeleteMarker' # whether object is successfully deleted.
except Exception as e:
raise BentoMLRepositoryException(
"Not able to delete object on S3. Error: {}".format(e)
)
return response[DELETE_MARKER]
def update_upload_status(self, bento_name, bento_version,
upload_status_pb):
with create_session(self.sess_maker) as sess:
try:
bento_obj = (sess.query(Bento).filter_by(name=bento_name,
version=bento_version,
deleted=False).one())
# TODO:
# if bento_obj.upload_status and bento_obj.upload_status.updated_at >
# upload_status_pb.updated_at, update should be ignored
bento_obj.upload_status = ProtoMessageToDict(upload_status_pb)
except NoResultFound:
raise BentoMLRepositoryException(
"Bento %s:%s is not found in repository" % bento_name,
bento_version)
def get(self, bento_name, bento_version):
# Return s3 path containing uploaded Bento files
object_name = self._get_object_name(bento_name, bento_version)
try:
response = self.s3_client.generate_presigned_url(
'get_object',
Params={'Bucket': self.bucket, 'Key': object_name},
ExpiresIn=self._expiration,
)
except Exception as e:
raise BentoMLRepositoryException(
"Not able to get pre-signed URL on S3. Error: {}".format(e)
)
return response
def add(self, bento_name, bento_version):
# Generate pre-signed s3 path for upload
object_name = self._get_object_name(bento_name, bento_version)
try:
response = self.s3_client.generate_presigned_post(
self.bucket,
object_name,
Fields=None,
Conditions=None,
ExpiresIn=self._expiration,
)
except Exception as e:
raise BentoMLRepositoryException(
"Not able to get pre-signed URL on S3. Error: {}".format(e)
)
return response
def add(self, bento_name, bento_version):
# Generate pre-signed s3 path for upload
expiration = config('yatai').getint('bento_uri_default_expiration')
object_name = "/".join([self.base_path, bento_name, bento_version])
try:
response = self.s3_client.generate_presigned_post(
self.bucket,
object_name,
Fields=None,
Conditions=None,
ExpiresIn=expiration,
)
except Exception as e:
raise BentoMLRepositoryException(
"Not able to get pre-signed URL on S3. Error: {}".format(e))
return response
def add(self, bento_name, bento_version):
# Full path containing saved BentoArchive, it the base path with service name
# and service version as prefix. e.g.:
# with base_path = '/tmp/my_bento_archive/', the saved bento will resolve in
# the directory: '/tmp/my_bento_archive/service_name/version/'
target_dir = os.path.join(self.base_path, bento_name, bento_version)
# Ensure parent directory exist
Path(os.path.join(self.base_path), bento_name).mkdir(parents=True,
exist_ok=True)
# Raise if target bento version already exist in storage
if os.path.exists(target_dir):
raise BentoMLRepositoryException(
"Existing Bento {name}:{version} found in archive: {target_dir}"
.format(name=bento_name,
version=bento_version,
target_dir=target_dir))
# Create target directory for upload
os.mkdir(target_dir)
return BentoUri(type=self.uri_type, uri=target_dir)
def add(self, bento_name, bento_version):
# Generate pre-signed s3 path for upload
object_name = self._get_object_name(bento_name,
bento_version) + '.tar.gz'
try:
response = self.s3_client.generate_presigned_post(
self.bucket,
object_name,
Fields=None,
Conditions=None,
ExpiresIn=self._expiration,
)
except Exception as e:
raise BentoMLRepositoryException(
"Not able to get pre-signed URL on S3. Error: {}".format(e))
return BentoUri(
type=self.uri_type,
uri=response['url'],
additional_fields=json.dumps(response['fields']),
)
