예제 #1
0
    def tearDown(self):
        if self.data_saver is not None:
            # delete bucket
            client = (Minio(**self.storage_config)
                      if not self.check_nas(self.data_config.endpoint) else
                      NAS(self.data_config.endpoint))
            objects = client.list_objects(self.data_config.bucket_name,
                                          recursive=True)
            for obj in objects:
                client.remove_object(self.data_config.bucket_name,
                                     obj.object_name)
            client.remove_bucket(self.data_config.bucket_name)

            # remove on local file
            for _file in self.data_saver.get_filelist:
                if os.path.exists(_file):
                    os.remove(_file)

        if self.dataset is not None:
            if os.path.exists(self.dataset.cache_path):
                os.remove(self.dataset.cache_path)

        if self.data_config_file is not None:
            if os.path.exists(self.data_config_file):
                os.remove(self.data_config_file)
예제 #2
0
    def _check_bucket(self):
        """
        Check bucket name is exist. If not exist, create new bucket
        If bucket and metadata sub folder exist, get metadata(attributes, compressor) from there.

        """
        _client = (Minio(self.endpoint,
                         access_key=self.access_key,
                         secret_key=self.secret_key,
                         secure=self.secure,
                         region=self.region)
                   if not check_nas(self.endpoint) else NAS(self.endpoint))
        if _client.bucket_exists(self.bucket_name):
            try:
                _metadata = _client.get_object(self.bucket_name,
                                               "metadata.json")
            except:
                _client.remove_bucket(self.bucket_name)
                raise FileNotFoundError(
                    "metadata.json is not in bucket name {}"
                    ", So this bucket will be removed".format(
                        self.bucket_name))

            metadata_dict = json.loads(_metadata.read().decode("utf-8"))
            if self.endpoint != metadata_dict["endpoint"]:
                raise ValueError(
                    "Already created endpoint({}) doesn't current endpoint str({})"
                    " It may occurs permission denied error".format(
                        metadata_dict["endpoint"], self.endpoint))

            self.compressor = metadata_dict["compressor"]
            self.metadata = metadata_dict
        else:
            logger.info("{} {} is not exist!".format(self.optimizer_name,
                                                     str(self.additional)))
예제 #3
0
파일: data.py 프로젝트: lkykor77/matorage
 def _create_client(self):
     return (Minio(
         endpoint=self.config.endpoint,
         access_key=self.config.access_key,
         secret_key=self.config.secret_key,
         secure=self.config.secure,
     ) if not check_nas(self.config.endpoint) else NAS(
         self.config.endpoint))
예제 #4
0
 def _create_client(self):
     return (
         Minio(
             endpoint=self.endpoint,
             access_key=self.access_key,
             secret_key=self.secret_key,
             secure=self.secure,
             region=self.region,
         )
         if not check_nas(self.endpoint)
         else NAS(self.endpoint)
     )
예제 #5
0
    def tearDown(self):
        if self.model_manager is not None:
            # delete bucket
            client = (Minio(**self.storage_config)
                      if not self.check_nas(self.model_config.endpoint) else
                      NAS(self.model_config.endpoint))
            objects = client.list_objects(self.model_config.bucket_name,
                                          recursive=True)
            for obj in objects:
                client.remove_object(self.model_config.bucket_name,
                                     obj.object_name)
            client.remove_bucket(self.model_config.bucket_name)

        if self.model_config_file is not None:
            if os.path.exists(self.model_config_file):
                os.remove(self.model_config_file)
예제 #6
0
파일: saver.py 프로젝트: jinserk/matorage
    def __init__(
        self,
        config,
        multipart_upload_size=5 * _MB,
        num_worker_threads=4,
        inmemory=False,
        refresh=False,
    ):

        self.config = config

        # Storage configuration
        self.multipart_upload_size = multipart_upload_size
        self.num_worker_threads = num_worker_threads

        # HDF5 configuration
        self.inmemory = inmemory

        self.filter = tb.Filters(**config.compressor)

        self._filelist = []
        self._file, self._earray = self._get_newfile()

        self._disconnected = False

        self._client = (Minio(
            endpoint=self.config.endpoint,
            access_key=self.config.access_key,
            secret_key=self.config.secret_key,
            secure=self.config.secure,
            region=self.config.region,
        ) if not check_nas(self.config.endpoint) else NAS(
            self.config.endpoint))
        self._check_and_create_bucket(refresh=refresh)

        self._uploader = Uploader(
            client=self._client,
            bucket=self.config.bucket_name,
            num_worker_threads=self.num_worker_threads,
            multipart_upload_size=self.multipart_upload_size,
            inmemory=self.inmemory,
        )

        atexit.register(self._exit)
예제 #7
0
    def _check_bucket(self):
        """
        Check bucket name is exist. If not exist, create new bucket
        If bucket and metadata sub folder exist, get metadata(attributes, compressor) from there.

        Returns:
            :obj: `None`:
        """
        _client = (Minio(
            self.endpoint,
            access_key=self.access_key,
            secret_key=self.secret_key,
            secure=self.secure,
        ) if not check_nas(self.endpoint) else NAS(self.endpoint))
        if _client.bucket_exists(self.bucket_name):
            objects = _client.list_objects(self.bucket_name,
                                           prefix="metadata/")
            _metadata = None
            for obj in objects:
                _metadata = _client.get_object(self.bucket_name,
                                               obj.object_name)
                break
            if not _metadata:
                return

            metadata_dict = json.loads(_metadata.read().decode("utf-8"))
            if self.endpoint != metadata_dict["endpoint"]:
                raise ValueError(
                    "Already created endpoint({}) doesn't current endpoint str({})"
                    " It may occurs permission denied error".format(
                        metadata_dict["endpoint"], self.endpoint))

            self.compressor = metadata_dict["compressor"]
            self.attributes = [
                DataAttribute(**item) for item in metadata_dict["attributes"]
            ]
        else:
            logger.warn("{} {} is not exist!".format(self.dataset_name,
                                                     str(self.additional)))
예제 #8
0
    def __init__(self, config, num_worker_threads=4, multipart_upload_size=5 * _MB):
        self.config = config
        self.num_worker_threads = num_worker_threads
        self.multipart_upload_size = multipart_upload_size

        self._client = (
            Minio(
                endpoint=self.config.endpoint,
                access_key=self.config.access_key,
                secret_key=self.config.secret_key,
                secure=self.config.secure,
            )
            if not check_nas(self.config.endpoint)
            else NAS(self.config.endpoint)
        )

        self._uploader = Uploader(
            client=self._client,
            bucket=self.config.bucket_name,
            num_worker_threads=self.num_worker_threads,
            multipart_upload_size=self.multipart_upload_size,
            inmemory=True,
        )
예제 #9
0
class DataSaverTest(StorageTest, unittest.TestCase):
    minio_config = Minio(
        endpoint="127.0.0.1:9001",
        access_key="minio",
        secret_key="miniosecretkey",
        secure=False,
    )
    nas_config = NAS(path="/tmp/unittest")

    def _create_bucket(self):
        self.minio_config.make_bucket('testminio')
        self.nas_config.make_bucket('testnas')

    def _get_obj_names(self, generator):
        res = sorted([g.object_name for g in generator])
        return res

    def _check_assertEqual(self, prefix=''):
        for r in [True, False]:
            self.assertEqual(
                self._get_obj_names(
                    self.minio_config.list_objects('testminio',
                                                   prefix=prefix,
                                                   recursive=r)),
                self._get_obj_names(
                    self.nas_config.list_objects('testnas',
                                                 prefix=prefix,
                                                 recursive=r)))

    def test_bucket_exists(self):
        self._create_bucket()
        self.assertEqual(self.minio_config.bucket_exists('testminio'),
                         self.nas_config.bucket_exists('testnas'))

    def test_fput_object_with_list_objects(self):
        self._create_bucket()

        with open('test', 'w') as f:
            f.write('this is test')

        for _object_name in ['test', 'metadata/test']:
            self.minio_config.fput_object(bucket_name='testminio',
                                          object_name=_object_name,
                                          file_path='test')
        for _object_name in ['test', 'metadata/test']:
            self.nas_config.fput_object(bucket_name='testnas',
                                        object_name=_object_name,
                                        file_path='test')

        self._check_assertEqual(prefix='')
        self._check_assertEqual(prefix='metadata/')
        self._check_assertEqual(prefix='metadata')
        self._check_assertEqual(prefix='meta')

        self._check_assertEqual(prefix='metadata/t')
        self._check_assertEqual(prefix='metadata/test')

        self.assertEqual(
            self._get_obj_names(
                self.minio_config.list_objects('testminio',
                                               prefix='metadata/test/')),
            self._get_obj_names(
                self.nas_config.list_objects('testnas',
                                             prefix='metadata/test/')))

        self.assertNotEqual(
            self._get_obj_names(
                self.minio_config.list_objects('testminio',
                                               prefix='metadata/test/',
                                               recursive=True)),
            self._get_obj_names(
                self.nas_config.list_objects('testnas',
                                             prefix='metadata/test/',
                                             recursive=True)))