def test_public_read(self, mock_connection): content = ''' <AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> \ <Owner> \ <ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID> \ <DisplayName>[email protected]</DisplayName> \ </Owner> \ <AccessControlList> \ <Grant> \ <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser"> \ <ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID> \ <DisplayName>[email protected]</DisplayName> \ <URI>http://acs.amazonaws.com/groups/global/AllUsers</URI> \ </Grantee> \ <Permission>READ</Permission> \ </Grant> \ <Grant> \ <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser"> \ <ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID> \ <DisplayName>[email protected]</DisplayName> \ </Grantee> \ <Permission>FULL_CONTROL</Permission> \ </Grant> \ </AccessControlList> \ </AccessControlPolicy> ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/hello/?acl', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=content)) client = Minio('localhost:9000') acl = client.get_bucket_acl('hello') eq_(Acl.public_read(), acl)
def get_presigned_get_url(filename, config, secrets): """Generate a presigned get URL using the Minio S3 client. Args: filename: Name of file to use in S3 config: Pyglidein cluster config dictionary secrets: Pyglidein cluster secrets dictionary Returns: string: Presigned Get URL """ from minio import Minio from minio.error import ResponseError config_startd_logging = config['StartdLogging'] secrets_startd_logging = secrets['StartdLogging'] client = Minio(config_startd_logging['url'], access_key=secrets_startd_logging['access_key'], secret_key=secrets_startd_logging['secret_key'], secure=True ) try: return client.presigned_get_object(config_startd_logging['bucket'], filename) except ResponseError as err: print(err)
def test_empty_list_objects_works(self, mock_connection): mock_data = """<?xml version="1.0"?> <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Name>bucket</Name> <Prefix/> <Marker/> <IsTruncated>false</IsTruncated> <MaxKeys>1000</MaxKeys> <Delimiter/> </ListBucketResult> """ mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse( "GET", "https://localhost:9000/bucket/?max-keys=1000", {"User-Agent": _DEFAULT_USER_AGENT}, 200, content=mock_data, ) ) client = Minio("localhost:9000") bucket_iter = client.list_objects("bucket", recursive=True) buckets = [] for bucket in bucket_iter: buckets.append(bucket) eq_(0, len(buckets))
def test_remove_bucket_works(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('DELETE', 'http://localhost:9000/hello/', {'User-Agent': _DEFAULT_USER_AGENT}, 204)) client = Minio('http://localhost:9000') client.remove_bucket('hello')
def test_empty_list_uploads_test(self, mock_connection): mock_data = '''<?xml version="1.0"?> <ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Bucket>golang</Bucket> <KeyMarker/> <UploadIdMarker/> <NextKeyMarker/> <NextUploadIdMarker/> <EncodingType/> <MaxUploads>1000</MaxUploads> <IsTruncated>false</IsTruncated> <Prefix/> <Delimiter/> </ListMultipartUploadsResult> ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('GET', 'https://localhost:9000/bucket/?max-uploads=1000&uploads', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') upload_iter = client._list_incomplete_uploads('bucket', '', True, False) uploads = [] for upload in upload_iter: uploads.append(upload) eq_(0, len(uploads))
def test_empty_list_parts_works(self, mock_connection): mock_data = '''<?xml version="1.0"?> <ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Bucket>bucket</Bucket> <Key>go1.4.2</Key> <UploadId>ntWSjzBytPT2xKLaMRonzXncsO10EH4Fc-Iq2-4hG-ulRYB</UploadId> <Initiator> <ID>minio</ID> <DisplayName>minio</DisplayName> </Initiator> <Owner> <ID>minio</ID> <DisplayName>minio</DisplayName> </Owner> <StorageClass>STANDARD</StorageClass> <PartNumberMarker>0</PartNumberMarker> <NextPartNumberMarker>0</NextPartNumberMarker> <MaxParts>1000</MaxParts> <IsTruncated>false</IsTruncated> </ListPartsResult> ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('GET', 'https://localhost:9000/bucket/key?max-parts=1000&uploadId=upload_id', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') part_iter = client._list_object_parts('bucket', 'key', 'upload_id') parts = [] for part in part_iter: parts.append(part) eq_(0, len(parts))
def test_notification_config_id_key_is_optional(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse( 'PUT', 'https://localhost:9000/my-test-bucket/?notification=', { 'Content-Md5': 'f+TfVp/A4pNnI7S4S+MkFg==', 'Content-Length': '196', 'User-Agent': _DEFAULT_USER_AGENT, }, 200, content="" ) ) client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Arn': 'arn1', 'Events': ['s3:ObjectCreated:*'], } ] } )
def test_set_bucket_acl_works(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('PUT', 'http://localhost:9000/hello/?acl', {'x-amz-acl': 'private', 'User-Agent': _DEFAULT_USER_AGENT}, 200)) client = Minio('http://localhost:9000') client.set_bucket_acl('hello', Acl.private())
def test_list_uploads_works(self, mock_connection): mock_data = '''<?xml version="1.0"?> <ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Bucket>golang</Bucket> <KeyMarker/> <UploadIdMarker/> <NextKeyMarker>keymarker</NextKeyMarker> <NextUploadIdMarker>uploadidmarker</NextUploadIdMarker> <EncodingType/> <MaxUploads>1000</MaxUploads> <IsTruncated>false</IsTruncated> <Upload> <Key>go1.4.2</Key> <UploadId>uploadid</UploadId> <Initiator> <ID/> <DisplayName/> </Initiator> <Owner> <ID/> <DisplayName/> </Owner> <StorageClass/> <Initiated>2015-05-30T14:43:35.349Z</Initiated> </Upload> <Upload> <Key>go1.5.0</Key> <UploadId>uploadid2</UploadId> <Initiator> <ID/> <DisplayName/> </Initiator> <Owner> <ID/> <DisplayName/> </Owner> <StorageClass/> <Initiated>2015-05-30T15:00:07.759Z</Initiated> </Upload> <Prefix/> <Delimiter/> </ListMultipartUploadsResult> ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('GET', 'https://localhost:9000/bucket/?delimiter=%2F&max-uploads=1000&uploads', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') upload_iter = client._list_incomplete_uploads('bucket', '', False, False) uploads = [] for upload in upload_iter: uploads.append(upload) eq_(2, len(uploads))
def test_bucket_exists_works(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('HEAD', 'https://localhost:9000/hello/', {'User-Agent': _DEFAULT_USER_AGENT}, 400)) client = Minio('localhost:9000') result = client.bucket_exists('hello')
def test_make_bucket_throws_fail(self, mock_connection): error_xml = generate_error('code', 'message', 'request_id', 'host_id', 'resource') mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('PUT', 'http://localhost:9000/hello/', {'User-Agent': _DEFAULT_USER_AGENT}, 409, content=error_xml)) client = Minio('http://localhost:9000') client.make_bucket('hello')
def test_get_object_throws_fail(self, mock_connection): error_xml = generate_error('code', 'message', 'request_id', 'host_id', 'resource', 'bucket', 'object') mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/hello/key', {'User-Agent': _DEFAULT_USER_AGENT}, 404, content=error_xml)) client = Minio('localhost:9000') client.get_object('hello', 'key')
def test_notification_config_events_key_is_present(self): client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Id': '1', 'Arn': 'arn1', } ] } )
def test_can_include_response_headers(self): client = Minio('localhost:9000', 'my_access_key', 'my_secret_key', secure=True) client._get_bucket_region = mock.Mock(return_value='us-east-1') r = client.presigned_get_object( 'mybucket', 'myfile.pdf', response_headers={ 'Response-Content-Type': 'application/pdf', 'Response-Content-Disposition': 'inline; filename="test.pdf"' }) self.assertIn('inline', r) self.assertIn('test.pdf', r) self.assertIn('application%2Fpdf', r)
def test_notification_config_arn_key_is_present(self): client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Id': '1', 'Events': ['s3:ObjectCreated:*'], } ] } )
def test_notification_config_id_key_is_string(self): client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Id': 1, 'Arn': 'abc', 'Events': ['s3:ObjectCreated:*'], } ] } )
def test_empty_list_buckets_works(self, mock_connection): mock_data = '<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Buckets>' \ '</Buckets><Owner><ID>minio</ID><DisplayName>minio</DisplayName></Owner></ListAllMyBucketsResult>' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') buckets = client.list_buckets() count = 0 for bucket in buckets: count += 1 eq_(0, count)
def test_object_is_tuple(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('POST', 'https://localhost:9000/hello/?delete', {'Content-Length': 95, 'User-Agent': _DEFAULT_USER_AGENT, 'Content-Md5': u'5Tg5SmU9Or43L4+iIyfPrQ=='}, 200, content='<Delete/>') ) client = Minio('localhost:9000') for err in client.remove_objects('hello', ('Ab', 'c')): print(err)
def test_stat_object_works(self, mock_connection): mock_headers = { 'content-type': 'application/octet-stream', 'last-modified': 'Fri, 26 Jun 2015 19:05:37 GMT', 'content-length': 11, 'etag': '5eb63bbbe01eeed093cb22bb8f5acdc3' } mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('HEAD', 'http://localhost:9000/hello/world', {'User-Agent': _DEFAULT_USER_AGENT}, 200, response_headers=mock_headers)) client = Minio('http://localhost:9000') client.stat_object('hello', 'world')
def test_notification_config_has_valid_keys(self): client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfiguration': [ { 'Id': '1', 'Arn': 'arn1', 'Events': ['s3:ObjectCreated:*'], } ] } )
def test_notification_config_has_valid_event_names(self): client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Id': '1', 'Arn': 'arn1', 'Events': ['object_created'], } ] } )
def test_list_objects_works(self, mock_connection): mock_data = '''<?xml version="1.0"?> <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Name>bucket</Name> <Prefix/> <Marker/> <MaxKeys>1000</MaxKeys> <Delimiter/> <IsTruncated>false</IsTruncated> <Contents> <Key>key1</Key> <LastModified>2015-05-05T02:21:15.716Z</LastModified> <ETag>5eb63bbbe01eeed093cb22bb8f5acdc3</ETag> <Size>11</Size> <StorageClass>STANDARD</StorageClass> <Owner> <ID>minio</ID> <DisplayName>minio</DisplayName> </Owner> </Contents> <Contents> <Key>key2</Key> <LastModified>2015-05-05T20:36:17.498Z</LastModified> <ETag>2a60eaffa7a82804bdc682ce1df6c2d4</ETag> <Size>1661</Size> <StorageClass>STANDARD</StorageClass> <Owner> <ID>minio</ID> <DisplayName>minio</DisplayName> </Owner> </Contents> </ListBucketResult> ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/bucket/?delimiter=%2F&max-keys=1000&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') bucket_iter = client.list_objects('bucket') buckets = [] for bucket in bucket_iter: # cause an xml exception and fail if we try retrieving again mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/bucket/?delimiter=%2F&max-keys=1000&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content='')) buckets.append(bucket) eq_(2, len(buckets))
def test_notification_config_filterspec_is_valid_1(self): client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Id': '1', 'Arn': 'arn1', 'Events': ['s3:ObjectCreated:*'], 'Filter': [] } ] } )
def test_empty_list_objects_works(self, mock_connection): mock_data = '''<?xml version="1.0"?> <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Name>bucket</Name> <Prefix/> <Marker/> <MaxKeys>1000</MaxKeys> <Delimiter/> <IsTruncated>true</IsTruncated> </ListBucketResult> ''' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', 'http://localhost:9000/bucket/?max-keys=1000', {}, 200, content=mock_data)) client = Minio('http://localhost:9000') bucket_iter = client.list_objects('bucket', recursive=True) buckets = [] for bucket in bucket_iter: buckets.append(bucket) eq_(0, len(buckets))
def get(cls, raw_msg_id): """ Get raw message from db or ObjectStorage service :param raw_msg_id: :return: a RawMessage or NotFound exception """ try: raw_msg = super(RawMessage, cls).get(raw_msg_id) except Exception as exc: log.warn(exc) raise NotFound if raw_msg.raw_data == '' and raw_msg.uri != '': # means raw message data have been stored in object store # need to retrieve raw_data from it url = urlparse.urlsplit(raw_msg.uri) path = url.path.strip("/") if url.scheme == 's3': minioConf = Configuration("global").get("object_store") minioClient = Minio(minioConf["endpoint"], access_key=minioConf["access_key"], secret_key=minioConf["secret_key"], secure=False, region=minioConf["location"]) try: resp = minioClient.get_object(url.netloc, path) except Exception as exc: log.warn(exc) raise NotFound # resp is a urllib3.response.HTTPResponse class try: raw_msg.raw_data = resp.data except Exception as exc: log.warn(exc) raise NotFound else: log.warn("raw message uri scheme not implemented") raise NotFound return raw_msg
def test_list_buckets_works(self, mock_connection): mock_data = '<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Buckets>' \ '<Bucket><Name>hello</Name><CreationDate>2015-06-22T23:07:43.240Z</CreationDate></Bucket><Bucket>' \ '<Name>world</Name><CreationDate>2015-06-22T23:07:56.766Z</CreationDate></Bucket>' \ '</Buckets><Owner><ID>minio</ID><DisplayName>minio</DisplayName></Owner></ListAllMyBucketsResult>' mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') buckets = client.list_buckets() buckets_list = [] count = 0 for bucket in buckets: count += 1 buckets_list.append(bucket) eq_(2, count) eq_('hello', buckets_list[0].name) eq_(datetime(2015, 6, 22, 23, 7, 43, 240000, pytz.utc), buckets_list[0].creation_date) eq_('world', buckets_list[1].name) eq_(datetime(2015, 6, 22, 23, 7, 56, 766000, pytz.utc), buckets_list[1].creation_date)
def test_connection(): # Initialize httpClient with relevant timeout. httpClient = urllib3.PoolManager( timeout=30, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), retries=urllib3.Retry( total=3, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] ) ) # Initialize minioClient with an endpoint and access/secret keys. minioClient = Minio(S3_CONN_INFO['endpoint'], access_key=S3_CONN_INFO['access_key'], secret_key=S3_CONN_INFO['secret_key'], secure=False, http_client=httpClient) buckets = minioClient.list_buckets()
def test_notification_config_filterspec_is_valid_8(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse( 'PUT', 'https://localhost:9000/my-test-bucket/?notification=', { 'Content-Length': '206', 'Content-Md5': 'AGCNfbD5OuiyIJFd+r67MA==', 'User-Agent': _DEFAULT_USER_AGENT, }, 200, content="" ) ) client = Minio('localhost:9000') client.set_bucket_notification( 'my-test-bucket', { 'QueueConfigurations': [ { 'Id': '1', 'Arn': 'arn1', 'Events': ['s3:ObjectCreated:*'], 'Filter': { 'Key': { 'FilterRules': [ { 'Name': 'suffix', 'Value': 'abc' } ] } } } ] } )
# # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys from minio import Minio minioClient = Minio('localhost:9000', access_key='miniouser', secret_key='leftfoot1', secure=False) if len(sys.argv) == 0: buckets = minioClient.list_buckets() print([bucket.name for bucket in buckets]) else: objects = minioClient.list_objects(sys.argv[1]) sizes =[object.size for object in objects] print(sum(sizes) / 1e6)
def __init__(self, _bucket="default"): self.client = Minio(self.server, access_key=self.secret_id, secret_key=self.secret_key, secure=self.isSSL); self.bucket = _bucket
class S3DataStore(DataStore): """ An implementation of the data store using S3 for storing policy checkpoints when using Coach in distributed mode. The policy checkpoints are written by the trainer and read by the rollout worker. """ def __init__(self, params: S3DataStoreParameters): """ :param params: The parameters required to use the S3 data store. """ super(S3DataStore, self).__init__(params) self.params = params access_key = None secret_key = None if params.creds_file: config = ConfigParser() config.read(params.creds_file) try: access_key = config.get('default', 'aws_access_key_id') secret_key = config.get('default', 'aws_secret_access_key') except Error as e: print("Error when reading S3 credentials file: %s", e) else: access_key = os.environ.get('ACCESS_KEY_ID') secret_key = os.environ.get('SECRET_ACCESS_KEY') self.mc = Minio(self.params.end_point, access_key=access_key, secret_key=secret_key) def deploy(self) -> bool: return True def get_info(self): return "s3://{}/{}".format(self.params.bucket_name) def undeploy(self) -> bool: return True def save_to_store(self): self._save_to_store(self.params.checkpoint_dir) def _save_to_store(self, checkpoint_dir): """ save_to_store() uploads the policy checkpoint, gifs and videos to the S3 data store. It reads the checkpoint state files and uploads only the latest checkpoint files to S3. It is used by the trainer in Coach when used in the distributed mode. """ try: # remove lock file if it exists self.mc.remove_object(self.params.bucket_name, SyncFiles.LOCKFILE.value) # Acquire lock self.mc.put_object(self.params.bucket_name, SyncFiles.LOCKFILE.value, io.BytesIO(b''), 0) state_file = CheckpointStateFile(os.path.abspath(checkpoint_dir)) if state_file.exists(): ckpt_state = state_file.read() checkpoint_file = None for root, dirs, files in os.walk(checkpoint_dir): for filename in files: if filename == CheckpointStateFile.checkpoint_state_filename: checkpoint_file = (root, filename) continue if filename.startswith(ckpt_state.name): abs_name = os.path.abspath( os.path.join(root, filename)) rel_name = os.path.relpath(abs_name, checkpoint_dir) self.mc.fput_object(self.params.bucket_name, rel_name, abs_name) abs_name = os.path.abspath( os.path.join(checkpoint_file[0], checkpoint_file[1])) rel_name = os.path.relpath(abs_name, checkpoint_dir) self.mc.fput_object(self.params.bucket_name, rel_name, abs_name) # upload Finished if present if os.path.exists( os.path.join(checkpoint_dir, SyncFiles.FINISHED.value)): self.mc.put_object(self.params.bucket_name, SyncFiles.FINISHED.value, io.BytesIO(b''), 0) # upload Ready if present if os.path.exists( os.path.join(checkpoint_dir, SyncFiles.TRAINER_READY.value)): self.mc.put_object(self.params.bucket_name, SyncFiles.TRAINER_READY.value, io.BytesIO(b''), 0) # release lock self.mc.remove_object(self.params.bucket_name, SyncFiles.LOCKFILE.value) if self.params.expt_dir and os.path.exists(self.params.expt_dir): for filename in os.listdir(self.params.expt_dir): if filename.endswith((".csv", ".json")): self.mc.fput_object( self.params.bucket_name, filename, os.path.join(self.params.expt_dir, filename)) if self.params.expt_dir and os.path.exists( os.path.join(self.params.expt_dir, 'videos')): for filename in os.listdir( os.path.join(self.params.expt_dir, 'videos')): self.mc.fput_object( self.params.bucket_name, filename, os.path.join(self.params.expt_dir, 'videos', filename)) if self.params.expt_dir and os.path.exists( os.path.join(self.params.expt_dir, 'gifs')): for filename in os.listdir( os.path.join(self.params.expt_dir, 'gifs')): self.mc.fput_object( self.params.bucket_name, filename, os.path.join(self.params.expt_dir, 'gifs', filename)) except ResponseError as e: print("Got exception: %s\n while saving to S3", e) def load_from_store(self): """ load_from_store() downloads a new checkpoint from the S3 data store when it is not available locally. It is used by the rollout workers when using Coach in distributed mode. """ try: state_file = CheckpointStateFile( os.path.abspath(self.params.checkpoint_dir)) # wait until lock is removed while True: objects = self.mc.list_objects_v2(self.params.bucket_name, SyncFiles.LOCKFILE.value) if next(objects, None) is None: try: # fetch checkpoint state file from S3 self.mc.fget_object(self.params.bucket_name, state_file.filename, state_file.path) except Exception as e: continue break time.sleep(10) # Check if there's a finished file objects = self.mc.list_objects_v2(self.params.bucket_name, SyncFiles.FINISHED.value) if next(objects, None) is not None: try: self.mc.fget_object( self.params.bucket_name, SyncFiles.FINISHED.value, os.path.abspath( os.path.join(self.params.checkpoint_dir, SyncFiles.FINISHED.value))) except Exception as e: pass # Check if there's a ready file objects = self.mc.list_objects_v2(self.params.bucket_name, SyncFiles.TRAINER_READY.value) if next(objects, None) is not None: try: self.mc.fget_object( self.params.bucket_name, SyncFiles.TRAINER_READY.value, os.path.abspath( os.path.join(self.params.checkpoint_dir, SyncFiles.TRAINER_READY.value))) except Exception as e: pass checkpoint_state = state_file.read() if checkpoint_state is not None: objects = self.mc.list_objects_v2(self.params.bucket_name, prefix=checkpoint_state.name, recursive=True) for obj in objects: filename = os.path.abspath( os.path.join(self.params.checkpoint_dir, obj.object_name)) if not os.path.exists(filename): self.mc.fget_object(obj.bucket_name, obj.object_name, filename) except ResponseError as e: print("Got exception: %s\n while loading from S3", e) def setup_checkpoint_dir(self, crd=None): if crd: self._save_to_store(crd)
# -*- coding: utf-8 -*- # MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname # are dummy values, please replace them with original values. from minio import Minio client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') # List all object paths in bucket that begin with my-prefixname. uploads = client.list_incomplete_uploads('my-bucketname', prefix='my-prefixname', recursive=True) for obj in uploads: print(obj.bucket_name, obj.object_name, obj.upload_id, obj.size)
cos_endpoint = get_secret_creds("/app/secrets/cos_endpoint") cos_access_key = get_secret_creds("/app/secrets/cos_access_key") cos_secret_key = get_secret_creds("/app/secrets/cos_secret_key") ''' Remove possible http scheme for Minio ''' url = re.compile(r"https?://") cos_endpoint = url.sub('', cos_endpoint) WML_CREDENTIALS = { "url": wml_url, "instance_id": wml_instance_id, "apikey": wml_apikey } ''' Load Spark model ''' cos = Minio(cos_endpoint, access_key=cos_access_key, secret_key=cos_secret_key, secure=True) cos.fget_object(cos_bucket_name, model_filepath, model_filepath) cos.fget_object(cos_bucket_name, train_data_filepath, train_data_filepath) cos.fget_object(cos_bucket_name, 'evaluation.json', 'evaluation.json') if aios_manifest_path: cos.fget_object(cos_bucket_name, aios_manifest_path, aios_manifest_path) os.system('unzip %s' % model_filepath) print('model ' + model_filepath + ' is downloaded') os.system('unzip %s' % train_data_filepath) print('train_data ' + train_data_filepath + ' is downloaded') sc = SparkContext() model = PipelineModel.load(model_filepath.split('.')[0])
def test_bucket_is_not_empty_string(self): client = Minio('localhost:9000') client.remove_bucket(' \t \n ')
def test_object_is_string(self): client = Minio('localhost:9000') client.presigned_put_object('hello', 1234)
transcription = db.Column(db.Text, nullable=False) original = db.Column(db.Boolean) upload_ip = db.Column(db.String(128), nullable=False) # returns a direct link to the image def get_url(self): return '{}/memes/{}'.format(app.config['MINIO_URL'], self.id) # returns a link to the meme page for this meme def get_page(self): return '/meme/{}'.format(self.id) # Minio configuration minioClient = Minio(app.config['MINIO_URL'].split('/')[2], access_key=app.config['MINIO_ACCESS_KEY'], secret_key=app.config['MINIO_SECRET_KEY'], secure='https://' in app.config['MINIO_URL']) # Allowed image file extensions ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'} # does a file have the correct extension def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS # TODO: this is really basic CSRF that uses sessions, when adding accounts this should be replaced @app.before_request def csrf_protect():
from minio.error import ResponseError post_policy = PostPolicy() # set bucket name location for uploads. post_policy.set_bucket_name('my-bucketname') # set key prefix for all incoming uploads. post_policy.set_key_startswith('my-objectname') # set content length for incoming uploads. post_policy.set_content_length_range(10, 1024) # set expiry 10 days into future. expires_date = datetime.utcnow()+timedelta(days=10) post_policy.set_expires(expires_date) client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') try: curl_str = 'curl -X POST my-bucketname.s3.amazonaws.com/' curl_cmd = [curl_str] signed_form_data = client.presigned_post_policy(post_policy) for field in signed_form_data: curl_cmd.append('-F {0}={1}'.format(field, signed_form_data[field])) # print curl command to upload files. curl_cmd.append('-F file=@<FILE>') print(' '.join(curl_cmd)) except ResponseError as err: print(err)
def test_default_user_agent(self): client = Minio('localhost') self.assertEqual(client._user_agent, _DEFAULT_USER_AGENT)
# Use Pillow for thumbnail from PIL import Image config.load_incluster_config() v1 = client.CoreV1Api() for secrets in v1.list_secret_for_all_namespaces().items: if secrets.metadata.name == 'minio': access_key = base64.b64decode(secrets.data['access_key']) secret_key = base64.b64decode(secrets.data['secret_key']) # Replace the DNS below with the minio service name (helm release name -svc) client = Minio('minio-minio-svc:9000', access_key=access_key, secret_key=secret_key, secure=False) def thumbnail(context): bucket = os.path.dirname(context['Key']) _, file_extension = os.path.splitext(context['Key']) filename = os.path.basename(context['Key']) print file_extension.upper() if file_extension.upper() != ".JPEG": return "Not a picture" if context['EventType'] == "s3:ObjectCreated:Put" and bucket == 'foobar':
from minio import Minio from minio.error import ResponseError minioClient = Minio('ggtaiwanmini.117503445.top:9000', access_key='admin', secret_key='admin123', secure=False) #通过文件上传到对象中。 # Put an object 'myobject' with contents from '/tmp/otherobject', upon success prints the etag identifier computed by server. # fanhui ('0200eb05a5d0e2a77478699a9244a941', None) # 这里上传了一张图片 try: print( minioClient.fput_object('liqiqi', 'myobjectsss.jpg', 'C://User//minioPythonDemo//test.jpg')) except ResponseError as err: print(err) # Put on object 'myobject.csv' with contents from # '/tmp/otherobject.csv' as 'application/csv'. # try: # print(minioClient.fput_object('mybucket', 'myobject.csv', # '/tmp/otherobject.csv', # content_type='application/csv')) # except ResponseError as err: # print(err)
def __init__(self, host, bucket, access_key, secret_key, region): self._bucket = bucket self._conn = Minio(host, access_key=access_key, secret_key=secret_key, region=region) found = self._conn.bucket_exists(bucket) if not found: quit('Minio: Unable to connect to ' + host + '/' + bucket)
def test_bucket_is_string(self): client = Minio('localhost:9000') client.remove_bucket(1234)
def check_obj_exists(s3: minio.Minio, bucket_name, object_name): try: return s3.stat_object(bucket_name, object_name) except minio.error.S3Error: return False
role_arn = "ROLE-ARN" # Role session name if available. role_session_name = "ROLE-SESSION-NAME" # External ID if available. external_id = "EXTERNAL-ID" # Policy if available. policy = "POLICY" # Region if available. region = "REGION" provider = AssumeRoleProvider( sts_endpoint, access_key, secret_key, policy=policy, region=region, role_arn=role_arn, role_session_name=role_session_name, external_id=external_id, ) client = Minio("MINIO-HOST:MINIO-PORT", credentials=provider) # Get information of an object. stat = client.stat_object("my-bucket", "my-object") print(stat)
class CosClient(LoggingConfigurable): client = None def __init__(self, config=None, endpoint=None, access_key=None, secret_key=None, bucket=None): super().__init__() if config: self.endpoint = urlparse(config.metadata['cos_endpoint']) self.access_key = config.metadata['cos_username'] self.secret_key = config.metadata['cos_password'] self.bucket = config.metadata['cos_bucket'] else: self.endpoint = urlparse(endpoint) self.access_key = access_key self.secret_key = secret_key self.bucket = bucket # Infer secure from the endpoint's scheme. self.secure = self.endpoint.scheme == 'https' self.client = self.__initialize_object_store() def __initialize_object_store(self): # Initialize minioClient with an endpoint and access/secret keys. self.client = Minio(endpoint=self.endpoint.netloc, access_key=self.access_key, secret_key=self.secret_key, secure=self.secure) # Make a bucket with the make_bucket API call. try: if not self.client.bucket_exists(self.bucket): self.client.make_bucket(self.bucket) except BucketAlreadyOwnedByYou as ex: self.log.warning("Object Storage bucket already owned by you", exc_info=True) raise ex from ex except BucketAlreadyExists as ex: self.log.warning("Object Storage bucket already exists", exc_info=True) raise ex from ex except ResponseError as ex: self.log.error("Object Storage error", exc_info=True) raise ex from ex except SignatureDoesNotMatch as ex: self.log.error("Incorrect Object Storage credentials supplied") raise ex from ex return self.client def upload_file(self, file_name, file_path): """ Uploads contents from a file, located on the local filesystem at `file_path`, as `file_name` in object storage. :param file_name: Name of the file object in object storage :param file_path: Path on the local filesystem from which object data will be read. :return: """ try: self.client.fput_object(bucket_name=self.bucket, object_name=file_name, file_path=file_path) except BaseException as ex: self.log.error('Error uploading file {} to bucket {}'.format( file_path, self.bucket), exc_info=True) raise ex from ex def upload_file_to_dir(self, dir, file_name, file_path): """ Uploads contents from a file, located on the local filesystem at `file_path`, as `file_name` in object storage. :param dir: the directory where the file should be uploaded to :param file_name: Name of the file object in object storage :param file_path: Path on the local filesystem from which object data will be read. :return: """ self.upload_file(os.path.join(dir, file_name), file_path) def download_file(self, file_name, file_path): """ Downloads and saves the object as a file in the local filesystem. :param file_name: Name of the file object in object storage :param file_path: Path on the local filesystem to which the object data will be written. :return: """ try: self.client.fget_object(bucket_name=self.bucket, object_name=file_name, file_path=file_path) except BaseException as ex: self.log.error('Error reading file {} from bucket {}'.format( file_name, self.bucket), exc_info=True) raise ex from ex def download_file_from_dir(self, dir, file_name, file_path): """ Downloads and saves the object as a file in the local filesystem. :param dir: the directory where the file is located :param file_name: Name of the file object in object storage :param file_path: Path on the local filesystem to which the object data will be written. :return: """ self.download_file(os.path.join(dir, file_name), file_path)
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # A Chain credentials provider, provides a way of chaining multiple providers # together and will pick the first available using priority order of the # 'providers' list from minio import Minio from minio.credentials import (AWSConfigProvider, ChainedProvider, EnvAWSProvider, IamAwsProvider) client = Minio( 's3.amazonaws.com', credentials=ChainedProvider( [ IamAwsProvider(), AWSConfigProvider(), EnvAWSProvider(), ] ) ) # Get information of an object. stat = client.stat_object("my-bucketname", "my-objectname") print(stat)
from minio import Minio MINIO_URL = os.environ.get('MINIO_URL') @contextlib.contextmanager def nostdout(): save_stdout = sys.stdout sys.stdout = io.BytesIO() yield sys.stdout = save_stdout minioClient = Minio(MINIO_URL, access_key=os.environ.get('MINIO_ACCESS_KEY'), secret_key=os.environ.get('MINIO_SECRET_KEY'), secure=False) caffe.set_mode_cpu() # Select desired model net = caffe.Net('./models/colorization_deploy_v2.prototxt', './models/colorization_release_v2.caffemodel', caffe.TEST) (H_in, W_in) = net.blobs['data_l'].data.shape[2:] # get input shape (H_out, W_out) = net.blobs['class8_ab'].data.shape[2:] # get output shape pts_in_hull = np.load('./resources/pts_in_hull.npy') # load cluster centers net.params['class8_ab'][0].data[:, :, 0, 0] = pts_in_hull.transpose( (1, 0)) # populate cluster centers as 1x1 convolution kernel
from minio.error import ResponseError post_policy = PostPolicy() # set bucket name location for uploads. post_policy.set_bucket_name('my-bucketname') # set key prefix for all incoming uploads. post_policy.set_key_startswith('my-objectname') # set content length for incoming uploads. post_policy.set_content_length_range(10, 1024) # set expiry 10 days into future. expires_date = datetime.utcnow()+timedelta(days=10) post_policy.set_expires(expires_date) client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') try: url_str, signed_form_data = client.presigned_post_policy(post_policy) curl_str = 'curl -X POST {0}'.format(url_str) curl_cmd = [curl_str] for field in signed_form_data: curl_cmd.append('-F {0}={1}'.format(field, signed_form_data[field])) # print curl command to upload files. curl_cmd.append('-F file=@<FILE>') print(' '.join(curl_cmd)) except ResponseError as err: print(err)
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are # dummy values, please replace them with original values. from minio import Minio from minio.commonconfig import ENABLED, Filter from minio.lifecycleconfig import Expiration, LifecycleConfig, Rule client = Minio( "play.min.io", access_key="Q3AM3UQ867SPQQA43P2F", secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ) config = LifecycleConfig([ Rule( ENABLED, rule_filter=Filter(prefix="logs/"), rule_id="rule2", expiration=Expiration(days=365), ), ], ) client.set_bucket_lifecycle("my-bucketname", config)
filename = data_url response = requests.get(data_url, allow_redirects=True) if data_url.find('/'): filename = data_url.rsplit('/', 1)[1] open(filename, 'wb').write(response.content) ''' Read data with Spark SQL ''' spark = SparkSession.builder.getOrCreate() df_data = spark.read.csv(path=filename, sep=",", header=True, inferSchema=True) df_data.head() ''' Upload data to Cloud object storage ''' cos = Minio(cos_endpoint, access_key=cos_access_key, secret_key=cos_secret_key, secure=True) if not cos.bucket_exists(cos_bucket_name): try: cos.make_bucket(cos_bucket_name) except ResponseError as err: print(err) cos.fput_object(cos_bucket_name, filename, filename) print('Data ' + filename + ' is uploaded to bucket at ' + cos_bucket_name) with open("/tmp/filename", "w") as report: report.write(filename) df_data.printSchema()
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os, time import sys import requests from minio import Minio from minio.error import ResponseError import json MINIO_URL = os.environ.get('MINIO_URL', 'minionas.uvadcos.io/') MINIO_ACCESS_KEY = os.environ.get('MINIO_ACCESS_KEY') MINIO_SECRET = os.environ.get('MINIO_SECRET') ORS_URL = os.environ.get("ORS_URL", "http://mds.ors/") minioClient = Minio(MINIO_URL, access_key=MINIO_ACCESS_KEY, secret_key=MINIO_SECRET, secure=False) def get_distribution(id): """Validates that given identifier exists in Mongo. Returns location in minio. """ if isinstance(id, list): locations = [] names = [] for i in id: location, name = get_distribution(i) if location == '': continue locations.append(location) names.append(name)
def main(): """ Functional testing of minio python library. """ fake = Factory.create() client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'), os.getenv('SECRET_KEY')) _http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) # Get unique bucket_name, object_name. bucket_name = uuid.uuid4().__str__() object_name = uuid.uuid4().__str__() # Enable trace # client.trace_on(sys.stderr) # Make a new bucket. bucket_name = 'minio-pytest' print(client.make_bucket(bucket_name)) print(client.make_bucket(bucket_name + '.unique', location='us-west-1')) ## Check if return codes a valid from server. try: client.make_bucket(bucket_name + '.unique', location='us-west-1') except ResponseError as err: if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']: pass else: raise # Check if bucket was created properly. print(client.bucket_exists(bucket_name)) print(client.bucket_exists(bucket_name + '.unique')) # List all buckets. buckets = client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date) with open('testfile', 'wb') as file_data: file_data.write(fake.text().encode('utf-8')) file_data.close() # Put a file file_stat = os.stat('testfile') with open('testfile', 'rb') as file_data: client.put_object(bucket_name, object_name, file_data, file_stat.st_size) file_data.close() # Fput a file print(client.fput_object(bucket_name, object_name + '-f', 'testfile')) # Copy a file print( client.copy_object(bucket_name, object_name + '-copy', '/' + bucket_name + '/' + object_name + '-f')) try: copy_conditions = CopyConditions() copy_conditions.set_match_etag('test-etag') print( client.copy_object(bucket_name, object_name + '-copy', '/' + bucket_name + '/' + object_name + '-f', copy_conditions)) except ResponseError as err: if err.code != 'PreconditionFailed': raise if err.message != 'At least one of the pre-conditions you specified did not hold': raise # Fetch stats on your object. print(client.stat_object(bucket_name, object_name)) # Fetch stats on your object. print(client.stat_object(bucket_name, object_name + '-f')) # Fetch stats on your object. print(client.stat_object(bucket_name, object_name + '-copy')) # Get a full object object_data = client.get_object(bucket_name, object_name) with open('newfile', 'wb') as file_data: for data in object_data: file_data.write(data) file_data.close() # Get a full object locally. print(client.fget_object(bucket_name, object_name, 'newfile-f')) # List all object paths in bucket. print("Listing using ListObjects API") objects = client.list_objects(bucket_name, recursive=True) for obj in objects: print(obj.bucket_name, obj.object_name, obj.last_modified, \ obj.etag, obj.size, obj.content_type) # List all object paths in bucket using V2 API. print("Listing using ListObjectsV2 API") objects = client.list_objects_v2(bucket_name, recursive=True) for obj in objects: print(obj.bucket_name, obj.object_name, obj.last_modified, \ obj.etag, obj.size, obj.content_type) presigned_get_object_url = client.presigned_get_object( bucket_name, object_name) response = _http.urlopen('GET', presigned_get_object_url) if response.status != 200: response_error = ResponseError(response) raise response_error.get(bucket_name, object_name) presigned_put_object_url = client.presigned_put_object( bucket_name, object_name) value = fake.text().encode('utf-8') data = io.BytesIO(value).getvalue() response = _http.urlopen('PUT', presigned_put_object_url, body=data) if response.status != 200: response_error = ResponseError(response) raise response_error.put(bucket_name, object_name) object_data = client.get_object(bucket_name, object_name) if object_data.read() != value: raise ValueError('Bytes not equal') # Post policy. policy = PostPolicy() policy.set_bucket_name(bucket_name) policy.set_key_startswith('objectPrefix/') expires_date = datetime.utcnow() + timedelta(days=10) policy.set_expires(expires_date) print(client.presigned_post_policy(policy)) # Remove an object. print(client.remove_object(bucket_name, object_name)) print(client.remove_object(bucket_name, object_name + '-f')) print(client.remove_object(bucket_name, object_name + '-copy')) policy_name = client.get_bucket_policy(bucket_name) if policy_name != Policy.NONE: raise ValueError('Policy name is invalid ' + policy_name) # Set read-write policy successfully. client.set_bucket_policy(bucket_name, '', Policy.READ_WRITE) # Reset policy to NONE. client.set_bucket_policy(bucket_name, '', Policy.NONE) # Validate if the policy is reverted back to NONE. policy_name = client.get_bucket_policy(bucket_name) if policy_name != Policy.NONE: raise ValueError('Policy name is invalid ' + policy_name) # Upload some new objects to prepare for multi-object delete test. print("Prepare for remove_objects() test.") object_names = [] for i in range(10): curr_object_name = object_name + "-{}".format(i) print("object-name: {}".format(curr_object_name)) print(client.fput_object(bucket_name, curr_object_name, "testfile")) object_names.append(curr_object_name) # delete the objects in a single library call. print("Performing remove_objects() test.") del_errs = client.remove_objects(bucket_name, object_names) had_errs = False for del_err in del_errs: had_errs = True print("Err is {}".format(del_err)) if had_errs: print("remove_objects() FAILED - it had unexpected errors.") else: print("remove_objects() worked as expected.") # Remove a bucket. This operation will only work if your bucket is empty. print(client.remove_bucket(bucket_name)) print(client.remove_bucket(bucket_name + '.unique')) # Remove temporary files. os.remove('testfile') os.remove('newfile') os.remove('newfile-f')
def test_create_minio_bucket(): """Test if buckets can be put into the Minio storage.""" # Create Necessary Config Parameters config.AWS_PKG_BUCKET = "test-bayesian-core-package-data" config.AWS_EPV_BUCKET = "test-bayesian-core-data" minioClient = Minio(config.LOCAL_MINIO_ENDPOINT, access_key=config.MINIO_ACCESS_KEY, secret_key=config.MINIO_SECRET_KEY, secure=False) try: minioClient.make_bucket(config.AWS_EPV_BUCKET, location="us-east-1") minioClient.make_bucket(config.AWS_PKG_BUCKET, location="us-east-1") except (BucketAlreadyOwnedByYou, BucketAlreadyExists): pass except ResponseError as err: logger.error(err) try: minioClient.fput_object( config.AWS_PKG_BUCKET, 'pypi/access_points/github_details.json', 'test/data/S3-data/pypi/access_points/github_details.json') minioClient.fput_object( config.AWS_PKG_BUCKET, 'pypi/access_points/libraries_io.json', 'test/data/S3-data/pypi/access_points/libraries_io.json') minioClient.fput_object( config.AWS_EPV_BUCKET, 'pypi/access_points/0.4.59/code_metrics.json', 'test/data/S3-data/pypi/access_points/0.4.59/code_metrics.json') minioClient.fput_object( config.AWS_EPV_BUCKET, 'pypi/access_points/0.4.59/security_issues.json', 'test/data/S3-data/pypi/access_points/0.4.59/security_issues.json') minioClient.fput_object( config.AWS_EPV_BUCKET, 'pypi/access_points/0.4.59/source_licenses.json', 'test/data/S3-data/pypi/access_points/0.4.59/source_licenses.json') minioClient.fput_object( config.AWS_EPV_BUCKET, 'pypi/access_points/0.4.59/metadata.json', 'test/data/S3-data/pypi/access_points/0.4.59/metadata.json') minioClient.fput_object( config.AWS_EPV_BUCKET, 'pypi/access_points/0.4.59.json', 'test/data/S3-data/pypi/access_points/0.4.59.json') except ResponseError as err: logger.error(err) assert minioClient.bucket_exists(config.AWS_PKG_BUCKET) assert minioClient.bucket_exists(config.AWS_EPV_BUCKET)
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are # dummy values, please replace them with original values. from minio import Minio from minio.error import ResponseError from minio.policy import Policy client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') # Make a new bucket try: # Set policy Policy.READ_ONLY to bucket 'my-bucketname' which # enables 'my-bucketname' readable by everyone. client.set_bucket_policy('my-bucketname', '', Policy.READ_ONLY) # Set policy Policy.READ_WRITE to bucket 'my-bucketname' and # prefix 'public-folder/' which enables # 'my-bucketname/public-folder/' read/writeable by everyone. client.set_bucket_policy('my-bucketname', 'public-folder/', Policy.READ_WRITE) # Set policy Policy.WRITE_ONLY to bucket 'my-bucketname' and
# -*- coding: utf-8 -*- # Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are # dummy values, please replace them with original values. from minio import Minio client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') buckets = client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date)
import os import time import openpyxl from minio import Minio from minio.error import ResponseError minio_client = Minio("192.168.8.106:9000", access_key="AKIAIOSFODNN7EXAMPLE", secret_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", secure=False) excel_client = openpyxl.Workbook() excel_sheet1 = excel_client.create_sheet(index=0, title="上传文件测试结果") excel_sheet2 = excel_client.create_sheet(index=1, title="下载文件测试结果") excel_row = ['A', 'B', 'C', 'D'] excel_sheet1['A1'] = "对象服务" excel_sheet1['B1'] = "0.7 MB" excel_sheet1['C1'] = "5.88 MB" excel_sheet1['D1'] = "88.9 MB" excel_sheet2['A1'] = "对象服务" excel_sheet2['B1'] = "0.7 MB" excel_sheet2['C1'] = "5.88 MB" excel_sheet2['D1'] = "88.9 MB" def upload(object_name, file_path): try: minio_client.fput_object("test1", object_name, file_path,
def test_set_app_info(self): client = Minio('localhost') expected_user_agent = _DEFAULT_USER_AGENT + ' hello/' + minio_version client.set_app_info('hello', minio_version) self.assertEqual(client._user_agent, expected_user_agent)
def test_remove_bucket_invalid_name(self): client = Minio('localhost:9000') client.remove_bucket('ABCD')
# -*- coding: utf-8 -*- # MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are # dummy values, please replace them with original values. from minio import Minio from minio.error import ResponseError client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') # Make a new bucket try: client.make_bucket('my-bucketname') except ResponseError as err: print(err)
def test_set_app_info_requires_non_empty_version(self): client = Minio('localhost:9000') self.assertRaises(ValueError, client.set_app_info, 'hello', '')