def _check_bucket(self):
"""
Check bucket name is exist. If not exist, create new bucket
If bucket and metadata sub folder exist, get metadata(attributes, compressor) from there.
Returns:
:obj: `None`:
"""
_client = (
Minio(
self.endpoint,
access_key=self.access_key,
secret_key=self.secret_key,
secure=self.secure,
)
if not check_nas(self.endpoint)
else NAS(self.endpoint)
)
if _client.bucket_exists(self.bucket_name):
try:
_metadata = _client.get_object(self.bucket_name, "metadata.json")
except:
_client.remove_bucket(self.bucket_name)
raise FileNotFoundError(
"metadata.json is not in bucket name {}"
", So this bucket will be removed".format(self.bucket_name)
)
metadata_dict = json.loads(_metadata.read().decode("utf-8"))
if self.endpoint != metadata_dict["endpoint"]:
raise ValueError(
"Already created endpoint({}) doesn't current endpoint str({})"
" It may occurs permission denied error".format(
metadata_dict["endpoint"], self.endpoint
)
)
self.compressor = metadata_dict["compressor"]
self.metadata = metadata_dict
else:
logger.warn(
"{} {} is not exist!".format(self.optimizer_name, str(self.additional))
)
def _check_bucket(self):
"""
Check bucket name is exist. If not exist, create new bucket
If bucket and metadata sub folder exist, get metadata(attributes, compressor) from there.
Returns:
:obj: `None`:
"""
_client = (Minio(
self.endpoint,
access_key=self.access_key,
secret_key=self.secret_key,
secure=self.secure,
) if not check_nas(self.endpoint) else NAS(self.endpoint))
if _client.bucket_exists(self.bucket_name):
objects = _client.list_objects(self.bucket_name,
prefix="metadata/")
_metadata = None
for obj in objects:
_metadata = _client.get_object(self.bucket_name,
obj.object_name)
break
if not _metadata:
return
metadata_dict = json.loads(_metadata.read().decode("utf-8"))
if self.endpoint != metadata_dict["endpoint"]:
raise ValueError(
"Already created endpoint({}) doesn't current endpoint str({})"
" It may occurs permission denied error".format(
metadata_dict["endpoint"], self.endpoint))
self.compressor = metadata_dict["compressor"]
self.attributes = [
DataAttribute(**item) for item in metadata_dict["attributes"]
]
else:
logger.warn("{} {} is not exist!".format(self.dataset_name,
str(self.additional)))
def save(self, model, **kwargs):
if not self._client.bucket_exists(self.config.bucket_name):
self._client.make_bucket(self.config.bucket_name)
if not isinstance(kwargs, dict):
metadata = 0
else:
metadata = kwargs
model_folder = self._hashmap_transfer(metadata)
if model_folder in self.config.metadata["model"]:
logger.warn(
"{} {} is already exist, so model will be overwrited.".format(
self.config.model_name, str(self.config.additional)
)
)
self._save_with_clear(model_folder, model, overwrite=True)
else:
self.config.metadata["model"].update({model_folder: metadata})
self._save_with_clear(model_folder, model)
logger.info("model with {} is saved".format(str(metadata)))
def save(self, optimizer):
if not self._client.bucket_exists(self.config.bucket_name):
self._client.make_bucket(self.config.bucket_name)
step = self._get_step(optimizer)
if not step:
logger.error("{} {} step({})is not exist".format(
self.config.optimizer_name, self.config.additional, str(step)))
return
if step in self.config.metadata["optimizer"]:
logger.warn(
"{} {} is already exist, so optimizer will be overwrited.".
format(self.config.optimizer_name,
str(self.config.additional)))
self._save_with_clear(step, optimizer, overwrite=True)
else:
self._set_metadata(metadata=self.config.metadata,
optimizer=optimizer,
step=step)
self._save_with_clear(step, optimizer)
logger.info("optimizer with {} is saved".format(str(step)))