def test_length_is_string(self): client = Minio('localhost:9000') client.copy_object('..hello', '1', '/testbucket/object')
def ner_upload(): os.chdir('/home/' + user + '/Tap/dataset/tap/transcription') transript_file = '' for file in glob.glob("*.txt"): transript_file = file transript_fileWOExt = os.path.splitext(transript_file)[0] with open(transript_file, 'r') as file: transript = file.read().replace('\n', '') print(transript) # logging.getLogger("deeppavlov").propagate = False try: ner_model = build_model(configs.ner.ner_ontonotes_bert, download=False) except: print("An exception occurred") ner = ner_model([transript]) fileNameWOExt = os.path.splitext(transript_file)[0] ACCESS_KEY = "admin" SECRET_KEY = "password" #Initialise MinioClient minioClient = Minio('127.0.0.1:9000', access_key=ACCESS_KEY, secret_key=SECRET_KEY, secure=False) # Insert output from Deepspeech into this variable metadata = {"Transcript": transript, "Entities": json.dumps(ner)} print(metadata) print('metadata size: ' + str(sys.getsizeof(metadata))) # Append metadata onto original video and upload to new Minio Bucket found = False try: print('before minio stat object') print( minioClient.stat_object("video", transript_fileWOExt + ".mp4")) found = True print('after minio stat object') except: print('Error in found') if (found): print('found') try: copy_result = minioClient.copy_object( "postprocess", transript_fileWOExt + ".mp4", "video/" + transript_fileWOExt + ".mp4", metadata=metadata) os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' + transript_file) except: print('error') os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' + transript_file) else: print('not found') try: copy_result = minioClient.copy_object( "postprocess", transript_fileWOExt + ".WAV", "preprocess/" + transript_fileWOExt + ".WAV", metadata=metadata) os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' + transript_file) except: print('error') os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' + transript_file)
def test_object_is_string(self): client = Minio('localhost:9000') client.copy_object('hello', 12, 12)
def test_object_is_not_empty_string(self): client = Minio('localhost:9000') client.copy_object('hello', ' \t \n ', '')
try: client.fput_object('posts-bucket', 'posts8.json', './posts.json', content_type="application/json") except ResponseError as err: print(err) # List buckets buckets = client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date) # List all files in bucket list_objects(client, bucket.name) for obj in client.list_objects(bucket.name, recursive=True): try: print( client.fget_object(bucket.name, obj.object_name, './results/' + obj.object_name)) except ResponseError as err: print(err) #Copy object with new name print("------ After Copy: ------------") try: client.copy_object('posts-bucket', 'posts9.json', '/mybucket/posts.json') except ResponseError as err: print(err) list_objects(client, 'posts-bucket')
# 第一次运行报错 minio.error.PreconditionFailed: PreconditionFailed: message: At least one of the preconditions you specified did not hold. # 不在copy_object函数加上最后一个参数就不报错了: copy_conditions,metadata=metadata copy_conditions = CopyConditions() # Set modified condition, copy object modified since 2014 April. t = (2014, 4, 0, 0, 0, 0, 0, 0, 0) mod_since = datetime.utcfromtimestamp(time.mktime(t)) copy_conditions.set_modified_since(mod_since) # Set unmodified condition, copy object unmodified since 2014 April. copy_conditions.set_unmodified_since(mod_since) # Set matching ETag condition, copy object which matches the following ETag. copy_conditions.set_match_etag("31624deb84149d2f8ef9c385918b653a") # Set matching ETag except condition, copy object which does not match the following ETag. copy_conditions.set_match_etag_except("31624deb84149d2f8ef9c385918b653a") # Set metadata metadata = {"test-key": "test-data"} minioClient = Minio('ggtaiwanmini.117503445.top:9000', access_key='admin', secret_key='admin123', secure=False) try: copy_result = minioClient.copy_object( "maylogs", "newcopy.txt", "/liqiqi/min-io.txt") #这里加不加最后一个参数有待讨论 print(copy_result) except ResponseError as err: print(err)
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and # my-objectname are dummy values, please replace them with original values. from datetime import datetime, timezone from minio import CopyConditions, Minio client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEY', secret_key='YOUR-SECRETKEY') # copy an object from a bucket to another. result = client.copy_object( "my-bucket", "my-object", "/my-sourcebucket/my-sourceobject", ) print(result.object_name, result.version_id) # copy an object with condition. copy_conditions = CopyConditions() # Set modified condition, copy object modified since 1st April 2014. mod_since = datetime(2014, 4, 1, tzinfo=timezone.utc) copy_conditions.set_modified_since(mod_since) # Set unmodified condition, copy object unmodified since 1st April 2014. # copy_conditions.set_unmodified_since(mod_since) # Set matching ETag condition, copy object which matches the following ETag. # copy_conditions.set_match_etag("31624deb84149d2f8ef9c385918b653a")
class MinioClientWrapper: """Wrapper around minio""" def __init__( self, endpoint: str, access_key: str = None, secret_key: str = None, secure: bool = False, ): self.__metadata_prefix = "x-amz-meta-" self.endpoint = endpoint self.access_key = access_key self.secret_key = secret_key self.secure = secure self.endpoint_url = ("https://" if secure else "http://") + endpoint try: self._minio = Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=secure ) except MinioException: logging.exception("Could not create minio client") raise def __remove_objects_recursively(self, bucket_name): to_del = [ obj.object_name for obj in self.list_objects(bucket_name, recursive=True) ] self.remove_objects(bucket_name, to_del) def create_bucket(self, bucket_name, delete_contents_if_exists=False): try: if not self.exists_bucket(bucket_name): self._minio.make_bucket(bucket_name) elif delete_contents_if_exists: return self.__remove_objects_recursively(bucket_name) except MinioException: logging.exception("Could not create bucket") return False # it probably already exists and is return True def remove_bucket(self, bucket_name, delete_contents=False): try: if self.exists_bucket(bucket_name): if delete_contents: self.__remove_objects_recursively(bucket_name) self._minio.remove_bucket(bucket_name) except MinioException: logging.exception("Could not remove bucket") return False return True def exists_bucket(self, bucket_name): try: return self._minio.bucket_exists(bucket_name) except MinioException: logging.exception("Could not check bucket for existence") return False def upload_file(self, bucket_name, object_name, filepath, metadata=None): """Note metadata are special, you need to use the 'X-Amz-Meta' standard, i.e: - key and value must be strings - and the keys are case insensitive: key1 -- > Key1 key_one --> Key_one key-one --> Key-One """ try: _metadata = {} if metadata is not None: for key in metadata.keys(): _metadata[self.__metadata_prefix + key] = metadata[key] self._minio.fput_object( bucket_name, object_name, filepath, metadata=_metadata ) except MinioException: logging.exception("Could not upload file") return False return True def download_file(self, bucket_name, object_name, filepath): try: self._minio.fget_object(bucket_name, object_name, filepath) except MinioException: logging.exception("Could not download file") return False return True def get_metadata(self, bucket_name, object_name): try: obj = self._minio.stat_object(bucket_name, object_name) assert obj.metadata # nosec return dict(obj.metadata) except MinioException: logging.exception("Could not get metadata") return {} def list_objects( self, bucket_name: str, prefix: Optional[str] = None, recursive: bool = False ) -> Iterator[Object]: try: return self._minio.list_objects( bucket_name=bucket_name, prefix=prefix, recursive=recursive ) except MinioException: logging.exception("Could not list objects") return [] def remove_objects(self, bucket_name: str, objects: List[str]): try: delete = [DeleteObject(name, version_id=None) for name in objects] iter_errors: Iterator[DeleteError] = self._minio.remove_objects( bucket_name, delete ) for err in iter_errors: log.error( "Failed to delete '%s' [version=%s]: %s (code: %s)", err.name, err.version_id, err.message, err.code, ) except MinioException: logging.exception("Could remove objects") return False return True def exists_object(self, bucket_name, object_name, recursive=False): """This seems to be pretty heavy, should be used with care""" try: for obj in self.list_objects(bucket_name, recursive=recursive): if obj.object_name == object_name: return True except MinioException: logging.exception("Could check object for existence") return False return False def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3)): try: return self._minio.presigned_put_object( bucket_name, object_name, expires=dt ) except MinioException: logging.exception("Could create presigned put url") return "" def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3)): try: return self._minio.presigned_get_object( bucket_name, object_name, expires=dt ) except MinioException: logging.exception("Could create presigned get url") return "" def copy_object( self, to_bucket_name: str, to_object_name: str, from_bucket: str, from_object: str, ): try: # ValueError for arguments result: ObjectWriteResult = self._minio.copy_object( bucket_name=to_bucket_name, object_name=to_object_name, source=CopySource(from_bucket, from_object), ) return result.bucket_name == to_bucket_name except MinioException: logging.exception("Could not copy") return False
from datetime import datetime, timezone from minio import Minio from minio.commonconfig import REPLACE, CopySource client = Minio( "play.min.io", access_key="Q3AM3UQ867SPQQA43P2F", secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ) # copy an object from a bucket to another. result = client.copy_object( "my-bucket", "my-object", CopySource("my-sourcebucket", "my-sourceobject"), ) print(result.object_name, result.version_id) # copy an object with condition. result = client.copy_object( "my-bucket", "my-object", CopySource( "my-sourcebucket", "my-sourceobject", modified_since=datetime(2014, 4, 1, tzinfo=timezone.utc), ), ) print(result.object_name, result.version_id)
def test_valid_copy_source(self): client = Minio('localhost:9000') client.copy_object('hello', '1', '/testbucket/object')
'posts2.json', './posts.json', content_type='application/json') except MinioException as err: print("Minio exception:", err) except InvalidResponseError as err: print("Invalid response error:", err) for bucket in client.list_buckets(): print(bucket.name, bucket.creation_date) list_objects(client, bucket.name) # Copy object with new name print("------ After Copy: ------------") try: client.copy_object('posts-bucket', 'postFromMyblogs.json', CopySource('myblogs', 'posts.json')) except MinioException as err: print("Minio exception:", err) except InvalidResponseError as err: print("Invalid response error:", err) list_objects(client, 'posts-bucket') #Get all objects from 'posts-bucket' for bucket in client.list_buckets(): for obj in client.list_objects(bucket.name, recursive=True): try: print( client.fget_object(bucket.name, obj.object_name, './results/' + obj.object_name)) except MinioException as err: print("Minio exception:", err)
class UpdateTableFileCommand: def __init__(self, table, file_url, **options): self.table = table self.file_url = file_url self.file_url_info = urlparse(file_url) self.hasher = hashlib.sha512() self.file_size = 0 minio_endpoint = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc self.should_upload = minio_endpoint != self.file_url_info.netloc self.minio = Minio( minio_endpoint, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY ) self._output_file = None self.delete_source = options["delete_source"] @property def output_file(self): if not self._output_file: self._output_file = NamedTemporaryFile(delete=False) return self._output_file def read_file_chunks(self, chunk_size): response = requests.get(self.file_url, stream=True) num_chunks = ( math.ceil(int(response.headers["Content-Length"]) / chunk_size) if response.headers.get("Content-Length") else None ) chunks = response.iter_content(chunk_size=chunk_size) for chunk in tqdm(chunks, desc=f"Downloading {self.file_url} chunks...", total=num_chunks): self.file_size += len(chunk) self.hasher.update(chunk) yield chunk def process_file_chunk(self, chunk, chunk_size): if self.should_upload: self.output_file.write(chunk) def finish_process(self): source = self.file_url_info.path # /BUCKET_NAME/OBJ_PATH suffix = "".join(Path(source).suffixes) dest_name = f"{self.table.dataset.slug}/{self.table.name}{suffix}" bucket = settings.MINIO_STORAGE_DATASETS_BUCKET_NAME is_same_file = source == f"/{bucket}/{dest_name}" if self.should_upload: self.output_file.close() progress = MinioProgress() self.log(f"Uploading file to bucket: {bucket}") content_type, encoding = mimetypes.guess_type(dest_name) if encoding == "gzip": # quando é '.csv.gz' o retorno de guess_type é ('text/csv', 'gzip') content_type = "application/gzip" elif encoding is None: content_type = "text/plain" self.minio.fput_object( bucket, dest_name, self.output_file.name, progress=progress, content_type=content_type ) elif not is_same_file: self.log(f"Copying {source} to bucket {bucket}") self.minio.copy_object(bucket, dest_name, source) if self.delete_source: self.log(f"Deleting {source}") split_source = source.split("/") source_bucket, source_obj = split_source[1], "/".join(split_source[2:]) self.minio.remove_object(source_bucket, source_obj) else: self.log(f"Using {source} as the dataset file.", end="") os.remove(self.output_file.name) return f"{settings.AWS_S3_ENDPOINT_URL}{bucket}/{dest_name}" @classmethod def execute(cls, dataset_slug, tablename, file_url, **options): table = Table.with_hidden.for_dataset(dataset_slug).named(tablename) self = cls(table, file_url, **options) chunk_size = settings.MINIO_DATASET_DOWNLOAD_CHUNK_SIZE for chunk in self.read_file_chunks(chunk_size): self.process_file_chunk(chunk, chunk_size) new_file_url = self.finish_process() table_file, created = self.create_table_file(new_file_url) table_file_url = f"https://{settings.APP_HOST}{table_file.admin_url}" if created: self.log(f"\nNew TableFile entry: {table_file_url}") else: self.log(f"\nUsing existing TableFile entry: {table_file_url}") self.log(f"File hash: {table_file.sha512sum}") self.log(f"File size: {table_file.readable_size}") def create_table_file(self, file_url): filename = Path(urlparse(file_url).path).name table_file, created = TableFile.objects.get_or_create( table=self.table, file_url=file_url, sha512sum=self.hasher.hexdigest(), size=str(self.file_size), filename=filename, ) return table_file, created def log(self, msg, *args, **kwargs): print(msg, *args, **kwargs)
class S3Client: """ Wrapper around minio """ def __init__(self, endpoint, access_key=None, secret_key=None, secure=False): self.__metadata_prefix = "x-amz-meta-" self.client = None try: self.client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure) except ResponseError as _err: logging.exception("Could not create minio client") def __remove_objects_recursively(self, bucket_name): objs = self.list_objects(bucket_name, recursive=True) to_del = [] for obj in objs: to_del.append(obj.object_name) self.remove_objects(bucket_name, to_del) def create_bucket(self, bucket_name, delete_contents_if_exists=False): try: if not self.exists_bucket(bucket_name): self.client.make_bucket(bucket_name) elif delete_contents_if_exists: return self.__remove_objects_recursively(bucket_name) except ResponseError as _err: logging.exception("Could not create bucket") return False # it probably already exists and is return True def remove_bucket(self, bucket_name, delete_contents=False): try: if self.exists_bucket(bucket_name): if delete_contents: self.__remove_objects_recursively(bucket_name) self.client.remove_bucket(bucket_name) except ResponseError as _err: logging.exception("Could not remove bucket") return False return True def exists_bucket(self, bucket_name): try: return self.client.bucket_exists(bucket_name) except ResponseError as _err: logging.exception("Could not check bucket for existence") return False def list_buckets(self): try: return self.client.list_buckets() except ResponseError as _err: logging.exception("Could not list bucket") return [] def upload_file(self, bucket_name, object_name, filepath, metadata=None): """ Note metadata are special, you need to use the 'X-Amz-Meta' standard, i.e: - key and value must be strings - and the keys are case insensitive: key1 -- > Key1 key_one --> Key_one key-one --> Key-One """ try: _metadata = {} if metadata is not None: for key in metadata.keys(): _metadata[self.__metadata_prefix + key] = metadata[key] self.client.fput_object(bucket_name, object_name, filepath, metadata=_metadata) except ResponseError as _err: logging.exception("Could not upload file") return False return True def download_file(self, bucket_name, object_name, filepath): try: self.client.fget_object(bucket_name, object_name, filepath) except ResponseError as _err: logging.exception("Could not download file") return False return True def get_metadata(self, bucket_name, object_name): try: obj = self.client.stat_object(bucket_name, object_name) _metadata = obj.metadata metadata = {} for key in _metadata.keys(): _key = key[len(self.__metadata_prefix):] metadata[_key] = _metadata[key] return metadata except ResponseError as _err: logging.exception("Could not get metadata") return {} def list_objects(self, bucket_name, prefix=None, recursive=False): try: return self.client.list_objects(bucket_name, prefix=prefix, recursive=recursive) except ResponseError as _err: logging.exception("Could not list objects") return [] def list_objects_v2(self, bucket_name, recursive=False): try: return self.client.list_objects_v2(bucket_name, recursive=recursive) except ResponseError as _err: logging.exception("Could not list objects") return [] def remove_objects(self, bucket_name, objects): try: for del_err in self.client.remove_objects(bucket_name, objects): msg = "Deletion Error: {}".format(del_err) log.debug(msg) except ResponseError as _err: logging.exception("Could remove objects") return False return True def exists_object(self, bucket_name, object_name, recursive=False): ''' This seems to be pretty heavy, should be used with care ''' try: objects = self.list_objects(bucket_name, recursive=recursive) for obj in objects: if obj.object_name == object_name: return True except ResponseError as _err: logging.exception("Could check object for existence") return False return False def search(self, bucket_name, query, recursive=True, include_metadata=False): results = [] objs = self.list_objects(bucket_name, recursive=recursive) _query = re.compile(query, re.IGNORECASE) for obj in objs: if _query.search(obj.object_name): results.append(obj) if include_metadata: metadata = self.get_metadata(bucket_name, obj.object_name) for key in metadata.keys(): if _query.search(key) or _query.search(metadata[key]): results.append(obj) for r in results: msg = "Object {} in bucket {} matches query {}".format( r.object_name, r.bucket_name, query) log.debug(msg) return results def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3)): try: return self.client.presigned_put_object(bucket_name, object_name, expires=dt) except ResponseError as _err: logging.exception("Could create presigned put url") return "" def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3)): try: return self.client.presigned_get_object(bucket_name, object_name, expires=dt) except ResponseError as _err: logging.exception("Could create presigned get url") return "" def copy_object(self, to_bucket_name, to_object_name, from_bucket_object_name): try: ret = self.client.copy_object(to_bucket_name, to_object_name, from_bucket_object_name, CopyConditions()) print(ret) return True except ResponseError as _err: logging.exception("Could not copy") return False
from minio.error import ResponseError client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEY', secret_key='YOUR-SECRETKEY') # client.trace_on(sys.stderr) copy_conditions = CopyConditions() # Set modified condition, copy object modified since 2014 April. t = (2014, 4, 0, 0, 0, 0, 0, 0, 0) mod_since = datetime.utcfromtimestamp(time.mktime(t)) copy_conditions.set_modified_since(mod_since) # Set unmodified condition, copy object unmodified since 2014 April. # copy_conditions.set_unmodified_since(mod_since) # Set matching ETag condition, copy object which matches the following ETag. # copy_conditions.set_match_etag("31624deb84149d2f8ef9c385918b653a") # Set matching ETag except condition, copy object which does not match the # following ETag. # copy_conditions.set_match_etag_except("31624deb84149d2f8ef9c385918b653a") try: copy_result = client.copy_object("my-bucket", "my-object", "/my-sourcebucket/my-sourceobject", copy_conditions) print(copy_result) except ResponseError as err: print(err)
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and # my-objectname are dummy values, please replace them with original values. from minio import Minio from minio.error import ResponseError client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEY', secret_key='YOUR-SECRETKEY') # Set the metadata metadata = {"test_meta_key": "test_meta_value"} try: copy_result = client.copy_object("my-bucket", "my-object", "/my-sourcebucket/my-sourceobject", metadata=metadata) print(copy_result) except ResponseError as err: print(err)
import uuid ACCESS_KEY = "xx" SECRET_KEY = "xx" #Initialise MinioClient minioClient = Minio('127.0.0.1:9000', access_key=ACCESS_KEY, secret_key=SECRET_KEY, secure=False) # Insert output from Deepspeech into this variable metadata = {"Transcript": "xx", "Entities": "[[['xx','xx']],[['xx','xx']]]"} # Append metadata onto original video and upload to new Minio Bucket try: copy_result = minioClient.copy_object("output_Bucket", "updatedFileName", "original_Bucket/fileToBeUpdated", metadata=metadata) except ResponseError as err: print(err) # Error-Checking to validate metadata has been appended to the corresponding video try: print( minioClient.fget_object(bucket_name="output_Bucket", object_name="updatedFileName")) except ResponseError as err: print(err)
def main(): """ Functional testing of minio python library. """ fake = Factory.create() client = Minio('play.minio.io:9000', 'Q3AM3UQ867SPQQA43P2F', 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG') _http = urllib3.PoolManager( cert_reqs='CERT_REQUIRED', ca_certs=certifi.where() ) # Get unique bucket_name, object_name. bucket_name = uuid.uuid4().__str__() object_name = uuid.uuid4().__str__() # Enable trace # client.trace_on(sys.stderr) # Make a new bucket. bucket_name = 'minio-pytest' client.make_bucket(bucket_name) is_s3 = client._endpoint_url.startswith("s3.amazonaws") if is_s3: client.make_bucket(bucket_name+'.unique', location='us-west-1') ## Check if return codes a valid from server. if is_s3: try: client.make_bucket(bucket_name+'.unique', location='us-west-1') except BucketAlreadyOwnedByYou as err: pass except BucketAlreadyExists as err: pass except ResponseError as err: raise # Check if bucket was created properly. client.bucket_exists(bucket_name) if is_s3: client.bucket_exists(bucket_name+'.unique') # List all buckets. buckets = client.list_buckets() for bucket in buckets: _, _ = bucket.name, bucket.creation_date with open('testfile', 'wb') as file_data: file_data.write(fake.text().encode('utf-8')) file_data.close() # Put a file file_stat = os.stat('testfile') with open('testfile', 'rb') as file_data: client.put_object(bucket_name, object_name, file_data, file_stat.st_size) file_data.close() with open('largefile', 'wb') as file_data: for i in range(0, 104857): file_data.write(fake.text().encode('utf-8')) file_data.close() # Fput a file client.fput_object(bucket_name, object_name+'-f', 'testfile') if is_s3: client.fput_object(bucket_name, object_name+'-f', 'testfile', metadata={'x-amz-storage-class': 'STANDARD_IA'}) # Fput a large file. client.fput_object(bucket_name, object_name+'-large', 'largefile') if is_s3: client.fput_object(bucket_name, object_name+'-large', 'largefile', metadata={'x-amz-storage-class': 'STANDARD_IA'}) # Copy a file client.copy_object(bucket_name, object_name+'-copy', '/'+bucket_name+'/'+object_name+'-f') try: copy_conditions = CopyConditions() copy_conditions.set_match_etag('test-etag') client.copy_object(bucket_name, object_name+'-copy', '/'+bucket_name+'/'+object_name+'-f', copy_conditions) except PreconditionFailed as err: if err.message != 'At least one of the preconditions you specified did not hold.': raise # Fetch stats on your object. client.stat_object(bucket_name, object_name) # Fetch stats on your object. client.stat_object(bucket_name, object_name+'-f') # Fetch stats on your large object. client.stat_object(bucket_name, object_name+'-large') # Fetch stats on your object. client.stat_object(bucket_name, object_name+'-copy') # Get a full object object_data = client.get_object(bucket_name, object_name) with open('newfile', 'wb') as file_data: for data in object_data: file_data.write(data) file_data.close() # Get a full object locally. client.fget_object(bucket_name, object_name, 'newfile-f') client.fput_object(bucket_name, object_name+'-f', 'testfile', metadata={'x-amz-meta-testing': 'value'}) stat = client.fget_object(bucket_name, object_name+'-f', 'newfile-f-custom') if not stat.metadata.has_key('X-Amz-Meta-Testing'): raise ValueError('Metadata key \'x-amz-meta-testing\' not found') value = stat.metadata['X-Amz-Meta-Testing'] if value != 'value': raise ValueError('Metadata key has unexpected' ' value {0}'.format(value)) # List all object paths in bucket. print("Listing using ListObjects") objects = client.list_objects(bucket_name, recursive=True) for obj in objects: _, _, _, _, _, _ = obj.bucket_name, obj.object_name, \ obj.last_modified, \ obj.etag, obj.size, \ obj.content_type # List all object paths in bucket using V2 API. print("Listing using ListObjectsV2") objects = client.list_objects_v2(bucket_name, recursive=True) for obj in objects: _, _, _, _, _, _ = obj.bucket_name, obj.object_name, \ obj.last_modified, \ obj.etag, obj.size, \ obj.content_type presigned_get_object_url = client.presigned_get_object(bucket_name, object_name) response = _http.urlopen('GET', presigned_get_object_url) if response.status != 200: raise ResponseError(response, 'GET', bucket_name, object_name).get_exception() presigned_put_object_url = client.presigned_put_object(bucket_name, object_name) value = fake.text().encode('utf-8') data = io.BytesIO(value).getvalue() response = _http.urlopen('PUT', presigned_put_object_url, body=data) if response.status != 200: raise ResponseError(response, 'PUT', bucket_name, object_name).get_exception() object_data = client.get_object(bucket_name, object_name) if object_data.read() != value: raise ValueError('Bytes not equal') # Post policy. policy = PostPolicy() policy.set_bucket_name(bucket_name) policy.set_key_startswith('objectPrefix/') expires_date = datetime.utcnow()+timedelta(days=10) policy.set_expires(expires_date) client.presigned_post_policy(policy) # Remove all objects. client.remove_object(bucket_name, object_name) client.remove_object(bucket_name, object_name+'-f') client.remove_object(bucket_name, object_name+'-large') client.remove_object(bucket_name, object_name+'-copy') policy_name = client.get_bucket_policy(bucket_name) if policy_name != Policy.NONE: raise ValueError('Policy name is invalid ' + policy_name) # Set read-only policy successfully. client.set_bucket_policy(bucket_name, '1/', Policy.READ_ONLY) # Set read-write policy successfully. client.set_bucket_policy(bucket_name, '1/', Policy.READ_WRITE) # Reset policy to NONE. client.set_bucket_policy(bucket_name, '', Policy.NONE) # Validate if the policy is reverted back to NONE. policy_name = client.get_bucket_policy(bucket_name) if policy_name != Policy.NONE: raise ValueError('Policy name is invalid ' + policy_name) # Upload some new objects to prepare for multi-object delete test. print("Prepare for remove_objects() test.") object_names = [] for i in range(10): curr_object_name = object_name+"-{}".format(i) # print("object-name: {}".format(curr_object_name)) client.fput_object(bucket_name, curr_object_name, "testfile") object_names.append(curr_object_name) # delete the objects in a single library call. print("Performing remove_objects() test.") del_errs = client.remove_objects(bucket_name, object_names) had_errs = False for del_err in del_errs: had_errs = True print("Remove objects err is {}".format(del_err)) if had_errs: print("Removing objects FAILED - it had unexpected errors.") raise else: print("Removing objects worked as expected.") # Remove a bucket. This operation will only work if your bucket is empty. print("Deleting buckets and finishing tests.") client.remove_bucket(bucket_name) if client._endpoint_url.startswith("s3.amazonaws"): client.remove_bucket(bucket_name+'.unique') # Remove temporary files. os.remove('testfile') os.remove('newfile') os.remove('newfile-f') os.remove('largefile') os.remove('newfile-f-custom')
def main(): """ Functional testing of minio python library. """ fake = Factory.create() client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'), os.getenv('SECRET_KEY')) _http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) # Get unique bucket_name, object_name. bucket_name = uuid.uuid4().__str__() object_name = uuid.uuid4().__str__() # Enable trace # client.trace_on(sys.stderr) # Make a new bucket. bucket_name = 'minio-pytest' print(client.make_bucket(bucket_name)) print(client.make_bucket(bucket_name + '.unique', location='us-west-1')) ## Check if return codes a valid from server. try: client.make_bucket(bucket_name + '.unique', location='us-west-1') except ResponseError as err: if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']: pass else: raise # Check if bucket was created properly. print(client.bucket_exists(bucket_name)) print(client.bucket_exists(bucket_name + '.unique')) # List all buckets. buckets = client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date) with open('testfile', 'wb') as file_data: file_data.write(fake.text().encode('utf-8')) file_data.close() # Put a file file_stat = os.stat('testfile') with open('testfile', 'rb') as file_data: client.put_object(bucket_name, object_name, file_data, file_stat.st_size) file_data.close() # Fput a file print(client.fput_object(bucket_name, object_name + '-f', 'testfile')) # Copy a file print( client.copy_object(bucket_name, object_name + '-copy', '/' + bucket_name + '/' + object_name + '-f')) try: copy_conditions = CopyConditions() copy_conditions.set_match_etag('test-etag') print( client.copy_object(bucket_name, object_name + '-copy', '/' + bucket_name + '/' + object_name + '-f', copy_conditions)) except ResponseError as err: if err.code != 'PreconditionFailed': raise if err.message != 'At least one of the pre-conditions you specified did not hold': raise # Fetch stats on your object. print(client.stat_object(bucket_name, object_name)) # Fetch stats on your object. print(client.stat_object(bucket_name, object_name + '-f')) # Fetch stats on your object. print(client.stat_object(bucket_name, object_name + '-copy')) # Get a full object object_data = client.get_object(bucket_name, object_name) with open('newfile', 'wb') as file_data: for data in object_data: file_data.write(data) file_data.close() # Get a full object locally. print(client.fget_object(bucket_name, object_name, 'newfile-f')) # List all object paths in bucket. print("Listing using ListObjects API") objects = client.list_objects(bucket_name, recursive=True) for obj in objects: print(obj.bucket_name, obj.object_name, obj.last_modified, \ obj.etag, obj.size, obj.content_type) # List all object paths in bucket using V2 API. print("Listing using ListObjectsV2 API") objects = client.list_objects_v2(bucket_name, recursive=True) for obj in objects: print(obj.bucket_name, obj.object_name, obj.last_modified, \ obj.etag, obj.size, obj.content_type) presigned_get_object_url = client.presigned_get_object( bucket_name, object_name) response = _http.urlopen('GET', presigned_get_object_url) if response.status != 200: response_error = ResponseError(response) raise response_error.get(bucket_name, object_name) presigned_put_object_url = client.presigned_put_object( bucket_name, object_name) value = fake.text().encode('utf-8') data = io.BytesIO(value).getvalue() response = _http.urlopen('PUT', presigned_put_object_url, body=data) if response.status != 200: response_error = ResponseError(response) raise response_error.put(bucket_name, object_name) object_data = client.get_object(bucket_name, object_name) if object_data.read() != value: raise ValueError('Bytes not equal') # Post policy. policy = PostPolicy() policy.set_bucket_name(bucket_name) policy.set_key_startswith('objectPrefix/') expires_date = datetime.utcnow() + timedelta(days=10) policy.set_expires(expires_date) print(client.presigned_post_policy(policy)) # Remove an object. print(client.remove_object(bucket_name, object_name)) print(client.remove_object(bucket_name, object_name + '-f')) print(client.remove_object(bucket_name, object_name + '-copy')) policy_name = client.get_bucket_policy(bucket_name) if policy_name != Policy.NONE: raise ValueError('Policy name is invalid ' + policy_name) # Set read-write policy successfully. client.set_bucket_policy(bucket_name, '', Policy.READ_WRITE) # Reset policy to NONE. client.set_bucket_policy(bucket_name, '', Policy.NONE) # Validate if the policy is reverted back to NONE. policy_name = client.get_bucket_policy(bucket_name) if policy_name != Policy.NONE: raise ValueError('Policy name is invalid ' + policy_name) # Upload some new objects to prepare for multi-object delete test. print("Prepare for remove_objects() test.") object_names = [] for i in range(10): curr_object_name = object_name + "-{}".format(i) print("object-name: {}".format(curr_object_name)) print(client.fput_object(bucket_name, curr_object_name, "testfile")) object_names.append(curr_object_name) # delete the objects in a single library call. print("Performing remove_objects() test.") del_errs = client.remove_objects(bucket_name, object_names) had_errs = False for del_err in del_errs: had_errs = True print("Err is {}".format(del_err)) if had_errs: print("remove_objects() FAILED - it had unexpected errors.") else: print("remove_objects() worked as expected.") # Remove a bucket. This operation will only work if your bucket is empty. print(client.remove_bucket(bucket_name)) print(client.remove_bucket(bucket_name + '.unique')) # Remove temporary files. os.remove('testfile') os.remove('newfile') os.remove('newfile-f')