def test_get_policy_for_non_existent_bucket(self, mock_connection):
mock_server = MockConnection()
mock_connection.return_value = mock_server
bucket_name = 'non-existent-bucket'
mock_server.mock_add_request(
MockResponse(
'GET',
'https://localhost:9000/' + bucket_name + '/?policy=',
{'User-Agent': _DEFAULT_USER_AGENT},
404,
))
client = Minio('localhost:9000')
client.get_bucket_policy(bucket_name)
def test_get_policy_for_existent_bucket_with_prefix(
self, mock_connection
):
mock_data = '{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::test-bucket"],"Sid":""},{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":"*"},"Resource":["arn:aws:s3:::test-bucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["test-prefix-readonly"]}},"Effect":"Allow","Principal":{"AWS":"*"},"Resource":["arn:aws:s3:::test-bucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":"*"},"Resource":["arn:aws:s3:::test-bucket/test-prefix-readonly*"],"Sid":""}]}' # NOQA
mock_server = MockConnection()
mock_connection.return_value = mock_server
bucket_name = 'test-bucket'
prefix_name = 'test-prefix-readonly'
mock_server.mock_add_request(
MockResponse(
'GET',
'https://localhost:9000/' + bucket_name + '/?policy=',
{'User-Agent': _DEFAULT_USER_AGENT},
200,
content=mock_data
)
)
client = Minio('localhost:9000')
response = client.get_bucket_policy(bucket_name, prefix_name)
eq_(response, Policy.READ_ONLY)
def test_get_policy_for_non_existent_bucket(self, mock_connection):
mock_server = MockConnection()
mock_connection.return_value = mock_server
bucket_name = 'non-existent-bucket'
error = ("<ErrorResponse>"
"<Code>NoSuchBucket</Code>"
"<Message>No such bucket</Message><RequestId>1234</RequestId>"
"<Resource>/non-existent-bucket</Resource>"
"<HostId>abcd</HostId>"
"<BucketName>non-existent-bucket</BucketName>"
"</ErrorResponse>")
mock_server.mock_add_request(
MockResponse('GET',
'https://localhost:9000/' + bucket_name + '?policy=',
{'User-Agent': _DEFAULT_USER_AGENT},
404,
response_headers={"Content-Type": "application/xml"},
content=error.encode()))
client = Minio('localhost:9000')
client.get_bucket_policy(bucket_name)
def test_get_policy_for_existent_bucket(self, mock_connection):
mock_data = {
"Version":
"2012-10-17",
"Statement": [{
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "s3:GetBucketLocation",
"Resource": "arn:aws:s3:::test-bucket"
}, {
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "s3:ListBucket",
"Resource": "arn:aws:s3:::test-bucket"
}, {
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::test-bucket/*"
}]
}
mock_server = MockConnection()
mock_connection.return_value = mock_server
bucket_name = 'test-bucket'
mock_server.mock_add_request(
MockResponse('GET',
'https://localhost:9000/' + bucket_name + '/?policy=',
{'User-Agent': _DEFAULT_USER_AGENT},
200,
content=mock_data))
client = Minio('localhost:9000')
response = client.get_bucket_policy(bucket_name)
eq_(response, mock_data)
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('play.minio.io:9000',
'Q3AM3UQ867SPQQA43P2F',
'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG')
_http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()
)
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
client.make_bucket(bucket_name)
is_s3 = client._endpoint_url.startswith("s3.amazonaws")
if is_s3:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
## Check if return codes a valid from server.
if is_s3:
try:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
except BucketAlreadyOwnedByYou as err:
pass
except BucketAlreadyExists as err:
pass
except ResponseError as err:
raise
# Check if bucket was created properly.
client.bucket_exists(bucket_name)
if is_s3:
client.bucket_exists(bucket_name+'.unique')
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
_, _ = bucket.name, bucket.creation_date
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data,
file_stat.st_size)
file_data.close()
with open('largefile', 'wb') as file_data:
for i in range(0, 104857):
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Fput a file
client.fput_object(bucket_name, object_name+'-f', 'testfile')
if is_s3:
client.fput_object(bucket_name, object_name+'-f', 'testfile',
metadata={'x-amz-storage-class': 'STANDARD_IA'})
# Fput a large file.
client.fput_object(bucket_name, object_name+'-large', 'largefile')
if is_s3:
client.fput_object(bucket_name, object_name+'-large', 'largefile',
metadata={'x-amz-storage-class': 'STANDARD_IA'})
# Copy a file
client.copy_object(bucket_name, object_name+'-copy',
'/'+bucket_name+'/'+object_name+'-f')
try:
copy_conditions = CopyConditions()
copy_conditions.set_match_etag('test-etag')
client.copy_object(bucket_name, object_name+'-copy',
'/'+bucket_name+'/'+object_name+'-f',
copy_conditions)
except PreconditionFailed as err:
if err.message != 'At least one of the preconditions you specified did not hold.':
raise
# Fetch stats on your object.
client.stat_object(bucket_name, object_name)
# Fetch stats on your object.
client.stat_object(bucket_name, object_name+'-f')
# Fetch stats on your large object.
client.stat_object(bucket_name, object_name+'-large')
# Fetch stats on your object.
client.stat_object(bucket_name, object_name+'-copy')
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
client.fget_object(bucket_name, object_name, 'newfile-f')
client.fput_object(bucket_name, object_name+'-f', 'testfile',
metadata={'x-amz-meta-testing': 'value'})
stat = client.fget_object(bucket_name, object_name+'-f', 'newfile-f-custom')
if not stat.metadata.has_key('X-Amz-Meta-Testing'):
raise ValueError('Metadata key \'x-amz-meta-testing\' not found')
value = stat.metadata['X-Amz-Meta-Testing']
if value != 'value':
raise ValueError('Metadata key has unexpected'
' value {0}'.format(value))
# List all object paths in bucket.
print("Listing using ListObjects")
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
_, _, _, _, _, _ = obj.bucket_name, obj.object_name, \
obj.last_modified, \
obj.etag, obj.size, \
obj.content_type
# List all object paths in bucket using V2 API.
print("Listing using ListObjectsV2")
objects = client.list_objects_v2(bucket_name, recursive=True)
for obj in objects:
_, _, _, _, _, _ = obj.bucket_name, obj.object_name, \
obj.last_modified, \
obj.etag, obj.size, \
obj.content_type
presigned_get_object_url = client.presigned_get_object(bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
raise ResponseError(response,
'GET',
bucket_name,
object_name).get_exception()
presigned_put_object_url = client.presigned_put_object(bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
raise ResponseError(response,
'PUT',
bucket_name,
object_name).get_exception()
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow()+timedelta(days=10)
policy.set_expires(expires_date)
client.presigned_post_policy(policy)
# Remove all objects.
client.remove_object(bucket_name, object_name)
client.remove_object(bucket_name, object_name+'-f')
client.remove_object(bucket_name, object_name+'-large')
client.remove_object(bucket_name, object_name+'-copy')
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Set read-only policy successfully.
client.set_bucket_policy(bucket_name, '1/', Policy.READ_ONLY)
# Set read-write policy successfully.
client.set_bucket_policy(bucket_name, '1/', Policy.READ_WRITE)
# Reset policy to NONE.
client.set_bucket_policy(bucket_name, '', Policy.NONE)
# Validate if the policy is reverted back to NONE.
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Upload some new objects to prepare for multi-object delete test.
print("Prepare for remove_objects() test.")
object_names = []
for i in range(10):
curr_object_name = object_name+"-{}".format(i)
# print("object-name: {}".format(curr_object_name))
client.fput_object(bucket_name, curr_object_name, "testfile")
object_names.append(curr_object_name)
# delete the objects in a single library call.
print("Performing remove_objects() test.")
del_errs = client.remove_objects(bucket_name, object_names)
had_errs = False
for del_err in del_errs:
had_errs = True
print("Remove objects err is {}".format(del_err))
if had_errs:
print("Removing objects FAILED - it had unexpected errors.")
raise
else:
print("Removing objects worked as expected.")
# Remove a bucket. This operation will only work if your bucket is empty.
print("Deleting buckets and finishing tests.")
client.remove_bucket(bucket_name)
if client._endpoint_url.startswith("s3.amazonaws"):
client.remove_bucket(bucket_name+'.unique')
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
os.remove('largefile')
os.remove('newfile-f-custom')
# Copyright (C) 2016 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
# dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com', secure=True,
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Make a new bucket
try:
# Get current policy of bucket 'my-bucketname'.
print(client.get_bucket_policy('my-bucketname'))
except ResponseError as err:
print(err)
class Operation:
client = None
def __init__(self):
minio_conf = conf.Conf()
self.client = Minio(minio_conf.get_endpoint(),
access_key=minio_conf.get_access_key(),
secret_key=minio_conf.get_secret_key(),
secure=minio_conf.get_secure())
def make_bucket(self, bucket_name):
try:
self.client.make_bucket(bucket_name=bucket_name)
except ResponseError as err:
print err
exit(0)
print "cminio: create [%s] bucket success" % bucket_name
def list_buckets(self):
buckets = self.client.list_buckets()
print '[bucket name]\t', '[create time]\t'
for bucket in buckets:
print '%s\t%s\t' % (bucket.name, bucket.creation_date)
def bucket_exists(self, bucket_name):
try:
if self.client.bucket_exists(bucket_name=bucket_name):
print "cminio: the [%s] bucket is exist" % bucket_name
else:
print "cminio: the [%s] bucket is not exist" % bucket_name
except ResponseError as err:
print err
def list_objects(self, prefix, bucket_name, recursive):
objects = self.client.list_objects(bucket_name=bucket_name,
prefix=prefix,
recursive=recursive)
print "[bucket name]\t[object name]\t[object last modified]\t[etag]\t[size]\t[content type]\t"
i = 0
for object in objects:
print "%s\t%s\t%s\t%s\t%s\t%s\t" % (
object.bucket_name, object.object_name.encode('utf-8'),
object.last_modified, object.etag, object.size,
object.content_type)
i = i + 1
print "\n Total: %s object[s]" % i
def get_bucket_policy(self, bucket_name):
policy = self.client.get_bucket_policy(bucket_name=bucket_name)
print "[%s] policy: %s" % (bucket_name, policy)
def set_bucket_policy(self, bucket_name, path):
bucket_policy = policy.Policy()
self.client.set_bucket_policy(bucket_name,
bucket_policy.policy_from_disk(path))
def get_bucket_notification(self, bucket_name):
notification = self.client.get_bucket_notification(
bucket_name=bucket_name)
print "[%s] bucket: %s" % (bucket_name, notification)
def set_bucket_notification(self, bucket_name, notification):
try:
self.client.set_bucket_notification(bucket_name, notification)
except ResponseError as err:
print err
def remove_all_bucket_notification(self, bucket_name):
self.client.remove_all_bucket_notification(bucket_name)
def get_object(self,
bucket_name,
object_name,
request_headers=None,
output=None):
try:
data = self.client.get_object(bucket_name,
object_name,
request_headers=request_headers)
with open(output, "wb") as file_data:
for d in data.stream(32 * 1024):
file_data.write(d)
except ResponseError as err:
print err
class BaseTask():
name = 'BaseTask'
body_id = None
services = []
def __init__(self):
self.config = get_config(os.getenv('APPLICATION_MODE',
'DEVELOPMENT'))()
self.init_logging()
self.init_db()
self.default_config = {
"id": "",
"rgs": "",
"url": "",
"force_full_sync": 0,
"wait_time": 0.2,
"geofabrik_package": None,
"osm_relation": None,
"name": None
}
def init_logging(self):
# prepare datalog
self.datalog = logging.getLogger('datalog')
self.datalog.setLevel(logging.DEBUG)
datalog_stream_handler = logging.StreamHandler()
datalog_stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s '))
datalog_stream_handler.setLevel(logging.DEBUG)
datalog_file_handler = logging.FileHandler(
"%s/%s-%s-%s.log" %
(self.config.LOG_DIR,
(time.strftime("%Y-%m-%d--%H-%M-%S")), self.name, self.body_id))
datalog_file_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s '))
datalog_file_handler.setLevel(logging.DEBUG)
self.datalog.addHandler(datalog_file_handler)
datalog_stream_handler = logging.StreamHandler()
datalog_stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s '))
datalog_stream_handler.setLevel(logging.DEBUG)
self.datalog.addHandler(datalog_stream_handler)
def init_db(self):
if 'mongodb' in self.services:
mongoengine.connect(db=self.config.MONGO_DB_NAME,
host=self.config.MONGO_DB_HOST,
port=self.config.MONGO_DB_PORT,
connect=False)
try:
self.db_raw_client = pymongo.MongoClient(
host=self.config.MONGO_DB_HOST,
port=self.config.MONGO_DB_PORT,
connect=False)
self.db_raw_client.server_info()
except pymongo.errors.ServerSelectionTimeoutError as err:
sys.exit('fatal: connection to MongoDB can\'t be established.')
self.db_raw = self.db_raw_client[self.config.MONGO_DB_NAME]
self.s3 = None
if 's3' in self.services:
self.s3 = Minio(self.config.S3_ENDPOINT,
access_key=self.config.S3_ACCESS_KEY,
secret_key=self.config.S3_SECRET_KEY,
secure=self.config.S3_SECURE)
try:
if not self.s3.bucket_exists(self.config.S3_BUCKET):
self.s3.make_bucket(self.config.S3_BUCKET,
location=self.config.S3_LOCATION)
except (MaxRetryError, ResponseError) as err:
sys.exit('fatal: connection to Minio can\'t be established.')
# Policies
needs_policy_update = False
try:
policies = json.loads(
self.s3.get_bucket_policy(
self.config.S3_BUCKET).decode("utf-8"))
if self.config.ENABLE_PROCESSING and len(
policies['Statement']) == 3:
needs_policy_update = True
except NoSuchBucketPolicy:
needs_policy_update = True
if needs_policy_update:
policies = [{
'Effect':
'Allow',
'Principal': {
'AWS': ['*']
},
'Action': ['s3:GetBucketLocation'],
'Resource': ['arn:aws:s3:::%s' % self.config.S3_BUCKET]
}, {
'Effect':
'Allow',
'Principal': {
'AWS': ['*']
},
'Action': ['s3:ListBucket'],
'Resource': ['arn:aws:s3:::%s' % self.config.S3_BUCKET],
'Condition': {
'StringEquals': {
's3:prefix': ['files']
}
}
}, {
'Effect':
'Allow',
'Principal': {
'AWS': ['*']
},
'Action': ['s3:GetObject'],
'Resource':
['arn:aws:s3:::%s/files*' % self.config.S3_BUCKET]
}]
if self.config.ENABLE_PROCESSING:
policies.append({
'Effect':
'Allow',
'Principal': {
'AWS': ['*']
},
'Action': ['s3:ListBucket'],
'Resource':
['arn:aws:s3:::%s' % self.config.S3_BUCKET],
'Condition': {
'StringEquals': {
's3:prefix': ['file-thumbnails']
}
}
})
policies.append({
'Effect':
'Allow',
'Principal': {
'AWS': ['*']
},
'Action': ['s3:GetObject'],
'Resource': [
'arn:aws:s3:::%s/file-thumbnails*' %
self.config.S3_BUCKET
]
})
self.s3.set_bucket_policy(
self.config.S3_BUCKET,
json.dumps({
'Version': '2012-10-17',
'Statement': policies
}))
self.es = None
if 'elasticsearch' in self.services:
self.es = Elasticsearch(self.config.ES_HOSTS)
def close(self):
self.close_connections()
self.close_logging()
def close_connections(self):
self.db_raw_client.close()
self.db_raw_client = None
mongoengine_disconnect()
if self.config.ES_ENABLED and self.es:
for conn in self.es.transport.connection_pool.connections:
conn.pool.close()
self.es = None
def close_logging(self):
for handler in self.datalog.handlers:
self.datalog.removeHandler(handler)
def get_file(self, file, save_to):
if file.storedAtMirror:
if not file.mirrorAccessUrl:
return False
r = requests.get(file.mirrorAccessUrl, stream=True)
if r.status_code != 200:
return False
with open(save_to, 'wb') as file_data:
for chunk in r.iter_content(chunk_size=32 * 1024):
if chunk:
file_data.write(chunk)
return True
else:
try:
data = self.s3.get_object(
self.config.S3_BUCKET,
"files/%s/%s" % (file.body.id, file.id))
except NoSuchKey:
return False
with open(save_to, 'wb') as file_data:
for chunk in data.stream(32 * 1024):
file_data.write(chunk)
return True
def execute(self, cmd, body_id):
new_env = os.environ.copy()
new_env['XDG_RUNTIME_DIR'] = '/tmp/'
try:
output, error = subprocess.Popen(
cmd.split(' '),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=new_env).communicate(
timeout=self.config.SUBPROCESS_TIMEOUT)
except subprocess.TimeoutExpired:
self.send_mail(self.config.ADMINS,
'critical error at oparl-mirror',
'command %s at %s takes forever' % (cmd, body_id))
return None
try:
if error is not None and error.decode().strip(
) != '' and 'WARNING **: clutter failed 0, get a life.' not in error.decode(
):
self.datalog.debug("pdf output at command %s; output: %s" %
(cmd, error.decode()))
except UnicodeDecodeError:
self.datalog.debug("pdf output at command %s; output: %s" %
(cmd, error))
return output
def get_body_config(self, body_id=False, filename=False):
if not filename:
filename = self.get_body_config_file(body_id)
try:
with open('%s/%s' %
(self.config.BODY_DIR, filename)) as body_config_file:
if not body_config_file:
return None
try:
body_config = deepcopy(self.default_config)
body_config.update(
yaml.load(body_config_file, Loader=yaml.SafeLoader))
if not body_config.get('active'):
return None
if self.config.BODY_LIST_MODE == 'blacklist':
if body_config.get('id') in self.config.BODY_LIST:
return None
else:
if body_config.get('id') not in self.config.BODY_LIST:
return None
return body_config
except ValueError:
return None
except FileNotFoundError:
return None
def get_region_config(self, region_id):
filename = self.get_region_config_file(region_id)
try:
with open(
'%s/%s' %
(self.config.REGION_DIR, filename)) as region_config_file:
if not region_config_file:
return None
try:
region_config = yaml.load(region_config_file,
Loader=yaml.SafeLoader)
if not region_config.get('active'):
return None
if self.config.REGION_LIST_MODE == 'blacklist':
if region_config.get('id') in self.config.REGION_LIST:
return None
else:
if region_config.get(
'id') not in self.config.REGION_LIST:
return None
return region_config
except ValueError:
return None
except FileNotFoundError:
return None
def send_mail(self, receivers=None, subject='', body=''):
smtp = smtplib.SMTP()
smtp.connect(self.config.MAIL_SERVER, self.config.MAIL_PORT)
smtp.ehlo_or_helo_if_needed()
if self.config.MAIL_USE_TLS:
smtp.starttls()
smtp.ehlo_or_helo_if_needed()
smtp.login(user=self.config.MAIL_USERNAME,
password=self.config.MAIL_PASSWORD)
body_full = '\r\n'.join([
'To: %s' % ', '.join(receivers),
'From: %s' % self.config.MAIL_FROM,
'Subject: %s' % subject, '', body
])
smtp.sendmail(self.config.MAIL_FROM, receivers, body_full)
def get_body_config_file(self, body_id):
for filename in os.listdir(self.config.BODY_DIR):
if filename.startswith('%s' % body_id):
return filename
def get_region_config_file(self, regio_id):
for filename in os.listdir(self.config.REGION_DIR):
if filename.startswith('%s' % regio_id):
return filename
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
# dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com', secure=True,
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Make a new bucket
try:
# Get current policy of bucket 'my-bucketname'.
print(client.get_bucket_policy('my-bucketname'))
# Get current policy of bucket 'my-bucketname' and prefix 'my-prefix'.
print(client.get_bucket_policy('my-bucketname', 'my-prefix'))
except ResponseError as err:
print(err)
class MinioAdapter(BaseStorageAdapter):
def __init__(self, endpoint=None, access_key=None, secret_key=None, secure=False, *args, **kwargs):
# Initialize minioClient with an endpoint and access/secret keys.
super().__init__(*args, **kwargs)
try:
self._client = Minio(endpoint=endpoint,
access_key=access_key,
secret_key=secret_key,
secure=secure)
except KeyError as err:
logger.error(err)
raise Exception("Please enter proper HOSTNAME,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY in .env ")
def bucket_exists(self, bucket_name):
if self._client.bucket_exists(bucket_name):
return True
return False
def create_bucket(self, bucket_name):
try:
if not self._client.bucket_exists(bucket_name):
self._client.make_bucket(bucket_name)
except ResponseError as err:
raise ResponseError
def remove_bucket(self, bucket_name):
try:
if self._client.bucket_exists(bucket_name):
self._client.remove_bucket(bucket_name)
except Exception as e:
raise e
def remove_file(self, bucket_name, file_name):
try:
if self._client.bucket_exists(self._client):
if (self._client.get_object(self._client, file_name)):
self._client.remove_object(bucket_name, file_name)
except Exception as e:
raise e
def get_bucket_list(self):
bucket_list = self._client.list_buckets()
return bucket_list
def get_all_files(self, bucket_name):
pass
def upload_file(self, bucket_name, file_name, file_path):
if (not self._client.bucket_exists(bucket_name)):
self._client.make_bucket(bucket_name=bucket_name)
try:
self._client.fput_object(bucket_name=bucket_name, object_name=file_name, file_path=file_path)
self.logger.info(f"Uploaded file {file_name}")
except ResponseError as e:
self.logger.error(e)
raise Exception(e)
def upload_data_stream(self, bucket_name, file_name, data_stream, length):
if (not self._client.bucket_exists(bucket_name)):
self._client.make_bucket(bucket_name=bucket_name)
try:
self._client.put_object(bucket_name=bucket_name, object_name=file_name, data=data_stream, length=length)
except ResponseError as err:
self.logger.error(err)
raise err
def download_all_files(self, bucket_name, download_path):
try:
if (self._client.bucket_exists(bucket_name)):
obj_list = self._client.list_objects(bucket_name)
for obj in obj_list:
self._client.fget_object(bucket_name=bucket_name, object_name=obj, file_path=download_path)
except Exception as e:
self.logger.error(e)
raise e
def download_n_files(self, bucket_name, download_path, num_of_files):
try:
count = 0
for file in self._client.list_objects(bucket_name):
self._client.fget_object(bucket_name=bucket_name, object_name=file, file_path=download_path)
count = count + 1
if count == num_of_files:
break
except ResponseError as e:
self.logger.error(e)
raise e
def count_files(self, bucket_name):
list = self._client.list_objects(bucket_name)
return len(list)
def get_policy(self, bucket_name):
policy = self._client.get_bucket_policy(bucket_name)
return policy
def set_policy(self, bucket_name, policy):
self._client.set_bucket_policy(bucket_name, policy)
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage.
# Copyright (C) 2016 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from minio import Minio
client = Minio(
"play.min.io",
access_key="Q3AM3UQ867SPQQA43P2F",
secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
)
policy = client.get_bucket_policy("my-bucket")
"Resource": "arn:aws:s3:::mybucket"
}, {
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::mybucket/*"
}]
}
minioClient.set_bucket_policy('mybucket', json.dumps(policy_read_only))
# Get current policy of all object paths in bucket "mybucket".
policy = minioClient.get_bucket_policy('mybucket')
print(policy)
notification = {
'QueueConfigurations': [{
'Id': '1',
'Arn': 'arn1',
'Events': ['s3:ObjectCreated:*'],
'Filter': {
'Key': {
'FilterRules': [{
'Name': 'prefix',
'Value': 'abc'
}]
}
}
import os
from minio import Minio
from minio.error import ResponseError
bucket_name = 'test101'
client = Minio('your-s3-endpoint.com',
access_key='xxxxxxxxxxxxxx',
secret_key='xxxxxxxxxxxxxx',
secure=True)
print("\n=== check if bucket '%s' exists ===" % bucket_name)
try:
print(client.bucket_exists(bucket_name))
print('\n=== check bucket info ===')
print(client.get_bucket_policy(bucket_name))
except ResponseError as err:
raise
print(err)
sys.exit(1)
print("\n=== list all buckets '%s' ===" % bucket_name)
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
print("\n=== put object to buckets '%s' ===" % bucket_name)
try:
client.fput_object(bucket_name,
'test1.csv',
'test1.csv',
from minio import Minio
client = Minio("s3.embl.de", access_key="ysun-user", secret_key="PZx9Djtl7yeV7Kp5k8Gsm7y2SFJ2Gw6W", secure=True)
buckets = client.list_buckets()
for bucket in buckets:
print (bucket).name
print (client.bucket_exists('platybrowser'))
print (client.get_bucket_policy('platybrowser'))
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
# dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
secure=True,
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Make a new bucket
try:
# Get current policy of bucket 'my-bucketname'.
print(client.get_bucket_policy('my-bucketname'))
except ResponseError as err:
print(err)
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name + '.unique', location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name + '.unique', location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name + '.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data,
file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name + '-f', 'testfile'))
# Copy a file
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f'))
try:
copy_conditions = CopyConditions()
copy_conditions.set_match_etag('test-etag')
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f',
copy_conditions))
except ResponseError as err:
if err.code != 'PreconditionFailed':
raise
if err.message != 'At least one of the pre-conditions you specified did not hold':
raise
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-f'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-copy'))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
print("Listing using ListObjects API")
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
# List all object paths in bucket using V2 API.
print("Listing using ListObjectsV2 API")
objects = client.list_objects_v2(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(
bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(
bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow() + timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name + '-f'))
print(client.remove_object(bucket_name, object_name + '-copy'))
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Set read-write policy successfully.
client.set_bucket_policy(bucket_name, '', Policy.READ_WRITE)
# Reset policy to NONE.
client.set_bucket_policy(bucket_name, '', Policy.NONE)
# Validate if the policy is reverted back to NONE.
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Upload some new objects to prepare for multi-object delete test.
print("Prepare for remove_objects() test.")
object_names = []
for i in range(10):
curr_object_name = object_name + "-{}".format(i)
print("object-name: {}".format(curr_object_name))
print(client.fput_object(bucket_name, curr_object_name, "testfile"))
object_names.append(curr_object_name)
# delete the objects in a single library call.
print("Performing remove_objects() test.")
del_errs = client.remove_objects(bucket_name, object_names)
had_errs = False
for del_err in del_errs:
had_errs = True
print("Err is {}".format(del_err))
if had_errs:
print("remove_objects() FAILED - it had unexpected errors.")
else:
print("remove_objects() worked as expected.")
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name + '.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
