def test_length_is_string(self):
client = Minio('localhost:9000')
client.copy_object('..hello', '1', '/testbucket/object')
def ner_upload():
os.chdir('/home/' + user + '/Tap/dataset/tap/transcription')
transript_file = ''
for file in glob.glob("*.txt"):
transript_file = file
transript_fileWOExt = os.path.splitext(transript_file)[0]
with open(transript_file, 'r') as file:
transript = file.read().replace('\n', '')
print(transript)
# logging.getLogger("deeppavlov").propagate = False
try:
ner_model = build_model(configs.ner.ner_ontonotes_bert,
download=False)
except:
print("An exception occurred")
ner = ner_model([transript])
fileNameWOExt = os.path.splitext(transript_file)[0]
ACCESS_KEY = "admin"
SECRET_KEY = "password"
#Initialise MinioClient
minioClient = Minio('127.0.0.1:9000',
access_key=ACCESS_KEY,
secret_key=SECRET_KEY,
secure=False)
# Insert output from Deepspeech into this variable
metadata = {"Transcript": transript, "Entities": json.dumps(ner)}
print(metadata)
print('metadata size: ' + str(sys.getsizeof(metadata)))
# Append metadata onto original video and upload to new Minio Bucket
found = False
try:
print('before minio stat object')
print(
minioClient.stat_object("video", transript_fileWOExt + ".mp4"))
found = True
print('after minio stat object')
except:
print('Error in found')
if (found):
print('found')
try:
copy_result = minioClient.copy_object(
"postprocess",
transript_fileWOExt + ".mp4",
"video/" + transript_fileWOExt + ".mp4",
metadata=metadata)
os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' +
transript_file)
except:
print('error')
os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' +
transript_file)
else:
print('not found')
try:
copy_result = minioClient.copy_object(
"postprocess",
transript_fileWOExt + ".WAV",
"preprocess/" + transript_fileWOExt + ".WAV",
metadata=metadata)
os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' +
transript_file)
except:
print('error')
os.remove('/home/' + user + '/Tap/dataset/tap/transcription/' +
transript_file)
def test_object_is_string(self):
client = Minio('localhost:9000')
client.copy_object('hello', 12, 12)
def test_object_is_not_empty_string(self):
client = Minio('localhost:9000')
client.copy_object('hello', ' \t \n ', '')
try:
client.fput_object('posts-bucket',
'posts8.json',
'./posts.json',
content_type="application/json")
except ResponseError as err:
print(err)
# List buckets
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
# List all files in bucket
list_objects(client, bucket.name)
for obj in client.list_objects(bucket.name, recursive=True):
try:
print(
client.fget_object(bucket.name, obj.object_name,
'./results/' + obj.object_name))
except ResponseError as err:
print(err)
#Copy object with new name
print("------ After Copy: ------------")
try:
client.copy_object('posts-bucket', 'posts9.json',
'/mybucket/posts.json')
except ResponseError as err:
print(err)
list_objects(client, 'posts-bucket')
# 第一次运行报错 minio.error.PreconditionFailed: PreconditionFailed: message: At least one of the preconditions you specified did not hold.
# 不在copy_object函数加上最后一个参数就不报错了: copy_conditions,metadata=metadata
copy_conditions = CopyConditions()
# Set modified condition, copy object modified since 2014 April.
t = (2014, 4, 0, 0, 0, 0, 0, 0, 0)
mod_since = datetime.utcfromtimestamp(time.mktime(t))
copy_conditions.set_modified_since(mod_since)
# Set unmodified condition, copy object unmodified since 2014 April.
copy_conditions.set_unmodified_since(mod_since)
# Set matching ETag condition, copy object which matches the following ETag.
copy_conditions.set_match_etag("31624deb84149d2f8ef9c385918b653a")
# Set matching ETag except condition, copy object which does not match the following ETag.
copy_conditions.set_match_etag_except("31624deb84149d2f8ef9c385918b653a")
# Set metadata
metadata = {"test-key": "test-data"}
minioClient = Minio('ggtaiwanmini.117503445.top:9000',
access_key='admin',
secret_key='admin123',
secure=False)
try:
copy_result = minioClient.copy_object(
"maylogs", "newcopy.txt", "/liqiqi/min-io.txt") #这里加不加最后一个参数有待讨论
print(copy_result)
except ResponseError as err:
print(err)
def test_object_is_not_empty_string(self):
client = Minio('localhost:9000')
client.copy_object('hello', ' \t \n ', '')
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and
# my-objectname are dummy values, please replace them with original values.
from datetime import datetime, timezone
from minio import CopyConditions, Minio
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEY',
secret_key='YOUR-SECRETKEY')
# copy an object from a bucket to another.
result = client.copy_object(
"my-bucket",
"my-object",
"/my-sourcebucket/my-sourceobject",
)
print(result.object_name, result.version_id)
# copy an object with condition.
copy_conditions = CopyConditions()
# Set modified condition, copy object modified since 1st April 2014.
mod_since = datetime(2014, 4, 1, tzinfo=timezone.utc)
copy_conditions.set_modified_since(mod_since)
# Set unmodified condition, copy object unmodified since 1st April 2014.
# copy_conditions.set_unmodified_since(mod_since)
# Set matching ETag condition, copy object which matches the following ETag.
# copy_conditions.set_match_etag("31624deb84149d2f8ef9c385918b653a")
class MinioClientWrapper:
"""Wrapper around minio"""
def __init__(
self,
endpoint: str,
access_key: str = None,
secret_key: str = None,
secure: bool = False,
):
self.__metadata_prefix = "x-amz-meta-"
self.endpoint = endpoint
self.access_key = access_key
self.secret_key = secret_key
self.secure = secure
self.endpoint_url = ("https://" if secure else "http://") + endpoint
try:
self._minio = Minio(
endpoint, access_key=access_key, secret_key=secret_key, secure=secure
)
except MinioException:
logging.exception("Could not create minio client")
raise
def __remove_objects_recursively(self, bucket_name):
to_del = [
obj.object_name for obj in self.list_objects(bucket_name, recursive=True)
]
self.remove_objects(bucket_name, to_del)
def create_bucket(self, bucket_name, delete_contents_if_exists=False):
try:
if not self.exists_bucket(bucket_name):
self._minio.make_bucket(bucket_name)
elif delete_contents_if_exists:
return self.__remove_objects_recursively(bucket_name)
except MinioException:
logging.exception("Could not create bucket")
return False
# it probably already exists and is
return True
def remove_bucket(self, bucket_name, delete_contents=False):
try:
if self.exists_bucket(bucket_name):
if delete_contents:
self.__remove_objects_recursively(bucket_name)
self._minio.remove_bucket(bucket_name)
except MinioException:
logging.exception("Could not remove bucket")
return False
return True
def exists_bucket(self, bucket_name):
try:
return self._minio.bucket_exists(bucket_name)
except MinioException:
logging.exception("Could not check bucket for existence")
return False
def upload_file(self, bucket_name, object_name, filepath, metadata=None):
"""Note
metadata are special, you need to use the
'X-Amz-Meta' standard, i.e:
- key and value must be strings
- and the keys are case insensitive:
key1 -- > Key1
key_one --> Key_one
key-one --> Key-One
"""
try:
_metadata = {}
if metadata is not None:
for key in metadata.keys():
_metadata[self.__metadata_prefix + key] = metadata[key]
self._minio.fput_object(
bucket_name, object_name, filepath, metadata=_metadata
)
except MinioException:
logging.exception("Could not upload file")
return False
return True
def download_file(self, bucket_name, object_name, filepath):
try:
self._minio.fget_object(bucket_name, object_name, filepath)
except MinioException:
logging.exception("Could not download file")
return False
return True
def get_metadata(self, bucket_name, object_name):
try:
obj = self._minio.stat_object(bucket_name, object_name)
assert obj.metadata # nosec
return dict(obj.metadata)
except MinioException:
logging.exception("Could not get metadata")
return {}
def list_objects(
self, bucket_name: str, prefix: Optional[str] = None, recursive: bool = False
) -> Iterator[Object]:
try:
return self._minio.list_objects(
bucket_name=bucket_name, prefix=prefix, recursive=recursive
)
except MinioException:
logging.exception("Could not list objects")
return []
def remove_objects(self, bucket_name: str, objects: List[str]):
try:
delete = [DeleteObject(name, version_id=None) for name in objects]
iter_errors: Iterator[DeleteError] = self._minio.remove_objects(
bucket_name, delete
)
for err in iter_errors:
log.error(
"Failed to delete '%s' [version=%s]: %s (code: %s)",
err.name,
err.version_id,
err.message,
err.code,
)
except MinioException:
logging.exception("Could remove objects")
return False
return True
def exists_object(self, bucket_name, object_name, recursive=False):
"""This seems to be pretty heavy, should be used with care"""
try:
for obj in self.list_objects(bucket_name, recursive=recursive):
if obj.object_name == object_name:
return True
except MinioException:
logging.exception("Could check object for existence")
return False
return False
def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3)):
try:
return self._minio.presigned_put_object(
bucket_name, object_name, expires=dt
)
except MinioException:
logging.exception("Could create presigned put url")
return ""
def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3)):
try:
return self._minio.presigned_get_object(
bucket_name, object_name, expires=dt
)
except MinioException:
logging.exception("Could create presigned get url")
return ""
def copy_object(
self,
to_bucket_name: str,
to_object_name: str,
from_bucket: str,
from_object: str,
):
try:
# ValueError for arguments
result: ObjectWriteResult = self._minio.copy_object(
bucket_name=to_bucket_name,
object_name=to_object_name,
source=CopySource(from_bucket, from_object),
)
return result.bucket_name == to_bucket_name
except MinioException:
logging.exception("Could not copy")
return False
def test_object_is_string(self):
client = Minio('localhost:9000')
client.copy_object('hello', 12, 12)
from datetime import datetime, timezone
from minio import Minio
from minio.commonconfig import REPLACE, CopySource
client = Minio(
"play.min.io",
access_key="Q3AM3UQ867SPQQA43P2F",
secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
)
# copy an object from a bucket to another.
result = client.copy_object(
"my-bucket",
"my-object",
CopySource("my-sourcebucket", "my-sourceobject"),
)
print(result.object_name, result.version_id)
# copy an object with condition.
result = client.copy_object(
"my-bucket",
"my-object",
CopySource(
"my-sourcebucket",
"my-sourceobject",
modified_since=datetime(2014, 4, 1, tzinfo=timezone.utc),
),
)
print(result.object_name, result.version_id)
def test_valid_copy_source(self):
client = Minio('localhost:9000')
client.copy_object('hello', '1', '/testbucket/object')
'posts2.json',
'./posts.json',
content_type='application/json')
except MinioException as err:
print("Minio exception:", err)
except InvalidResponseError as err:
print("Invalid response error:", err)
for bucket in client.list_buckets():
print(bucket.name, bucket.creation_date)
list_objects(client, bucket.name)
# Copy object with new name
print("------ After Copy: ------------")
try:
client.copy_object('posts-bucket', 'postFromMyblogs.json',
CopySource('myblogs', 'posts.json'))
except MinioException as err:
print("Minio exception:", err)
except InvalidResponseError as err:
print("Invalid response error:", err)
list_objects(client, 'posts-bucket')
#Get all objects from 'posts-bucket'
for bucket in client.list_buckets():
for obj in client.list_objects(bucket.name, recursive=True):
try:
print(
client.fget_object(bucket.name, obj.object_name,
'./results/' + obj.object_name))
except MinioException as err:
print("Minio exception:", err)
class UpdateTableFileCommand:
def __init__(self, table, file_url, **options):
self.table = table
self.file_url = file_url
self.file_url_info = urlparse(file_url)
self.hasher = hashlib.sha512()
self.file_size = 0
minio_endpoint = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc
self.should_upload = minio_endpoint != self.file_url_info.netloc
self.minio = Minio(
minio_endpoint, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY
)
self._output_file = None
self.delete_source = options["delete_source"]
@property
def output_file(self):
if not self._output_file:
self._output_file = NamedTemporaryFile(delete=False)
return self._output_file
def read_file_chunks(self, chunk_size):
response = requests.get(self.file_url, stream=True)
num_chunks = (
math.ceil(int(response.headers["Content-Length"]) / chunk_size)
if response.headers.get("Content-Length")
else None
)
chunks = response.iter_content(chunk_size=chunk_size)
for chunk in tqdm(chunks, desc=f"Downloading {self.file_url} chunks...", total=num_chunks):
self.file_size += len(chunk)
self.hasher.update(chunk)
yield chunk
def process_file_chunk(self, chunk, chunk_size):
if self.should_upload:
self.output_file.write(chunk)
def finish_process(self):
source = self.file_url_info.path # /BUCKET_NAME/OBJ_PATH
suffix = "".join(Path(source).suffixes)
dest_name = f"{self.table.dataset.slug}/{self.table.name}{suffix}"
bucket = settings.MINIO_STORAGE_DATASETS_BUCKET_NAME
is_same_file = source == f"/{bucket}/{dest_name}"
if self.should_upload:
self.output_file.close()
progress = MinioProgress()
self.log(f"Uploading file to bucket: {bucket}")
content_type, encoding = mimetypes.guess_type(dest_name)
if encoding == "gzip":
# quando é '.csv.gz' o retorno de guess_type é ('text/csv', 'gzip')
content_type = "application/gzip"
elif encoding is None:
content_type = "text/plain"
self.minio.fput_object(
bucket, dest_name, self.output_file.name, progress=progress, content_type=content_type
)
elif not is_same_file:
self.log(f"Copying {source} to bucket {bucket}")
self.minio.copy_object(bucket, dest_name, source)
if self.delete_source:
self.log(f"Deleting {source}")
split_source = source.split("/")
source_bucket, source_obj = split_source[1], "/".join(split_source[2:])
self.minio.remove_object(source_bucket, source_obj)
else:
self.log(f"Using {source} as the dataset file.", end="")
os.remove(self.output_file.name)
return f"{settings.AWS_S3_ENDPOINT_URL}{bucket}/{dest_name}"
@classmethod
def execute(cls, dataset_slug, tablename, file_url, **options):
table = Table.with_hidden.for_dataset(dataset_slug).named(tablename)
self = cls(table, file_url, **options)
chunk_size = settings.MINIO_DATASET_DOWNLOAD_CHUNK_SIZE
for chunk in self.read_file_chunks(chunk_size):
self.process_file_chunk(chunk, chunk_size)
new_file_url = self.finish_process()
table_file, created = self.create_table_file(new_file_url)
table_file_url = f"https://{settings.APP_HOST}{table_file.admin_url}"
if created:
self.log(f"\nNew TableFile entry: {table_file_url}")
else:
self.log(f"\nUsing existing TableFile entry: {table_file_url}")
self.log(f"File hash: {table_file.sha512sum}")
self.log(f"File size: {table_file.readable_size}")
def create_table_file(self, file_url):
filename = Path(urlparse(file_url).path).name
table_file, created = TableFile.objects.get_or_create(
table=self.table,
file_url=file_url,
sha512sum=self.hasher.hexdigest(),
size=str(self.file_size),
filename=filename,
)
return table_file, created
def log(self, msg, *args, **kwargs):
print(msg, *args, **kwargs)
def test_length_is_string(self):
client = Minio('localhost:9000')
client.copy_object('..hello', '1', '/testbucket/object')
class S3Client:
""" Wrapper around minio
"""
def __init__(self,
endpoint,
access_key=None,
secret_key=None,
secure=False):
self.__metadata_prefix = "x-amz-meta-"
self.client = None
try:
self.client = Minio(endpoint,
access_key=access_key,
secret_key=secret_key,
secure=secure)
except ResponseError as _err:
logging.exception("Could not create minio client")
def __remove_objects_recursively(self, bucket_name):
objs = self.list_objects(bucket_name, recursive=True)
to_del = []
for obj in objs:
to_del.append(obj.object_name)
self.remove_objects(bucket_name, to_del)
def create_bucket(self, bucket_name, delete_contents_if_exists=False):
try:
if not self.exists_bucket(bucket_name):
self.client.make_bucket(bucket_name)
elif delete_contents_if_exists:
return self.__remove_objects_recursively(bucket_name)
except ResponseError as _err:
logging.exception("Could not create bucket")
return False
# it probably already exists and is
return True
def remove_bucket(self, bucket_name, delete_contents=False):
try:
if self.exists_bucket(bucket_name):
if delete_contents:
self.__remove_objects_recursively(bucket_name)
self.client.remove_bucket(bucket_name)
except ResponseError as _err:
logging.exception("Could not remove bucket")
return False
return True
def exists_bucket(self, bucket_name):
try:
return self.client.bucket_exists(bucket_name)
except ResponseError as _err:
logging.exception("Could not check bucket for existence")
return False
def list_buckets(self):
try:
return self.client.list_buckets()
except ResponseError as _err:
logging.exception("Could not list bucket")
return []
def upload_file(self, bucket_name, object_name, filepath, metadata=None):
""" Note
metadata are special, you need to use the
'X-Amz-Meta' standard, i.e:
- key and value must be strings
- and the keys are case insensitive:
key1 -- > Key1
key_one --> Key_one
key-one --> Key-One
"""
try:
_metadata = {}
if metadata is not None:
for key in metadata.keys():
_metadata[self.__metadata_prefix + key] = metadata[key]
self.client.fput_object(bucket_name,
object_name,
filepath,
metadata=_metadata)
except ResponseError as _err:
logging.exception("Could not upload file")
return False
return True
def download_file(self, bucket_name, object_name, filepath):
try:
self.client.fget_object(bucket_name, object_name, filepath)
except ResponseError as _err:
logging.exception("Could not download file")
return False
return True
def get_metadata(self, bucket_name, object_name):
try:
obj = self.client.stat_object(bucket_name, object_name)
_metadata = obj.metadata
metadata = {}
for key in _metadata.keys():
_key = key[len(self.__metadata_prefix):]
metadata[_key] = _metadata[key]
return metadata
except ResponseError as _err:
logging.exception("Could not get metadata")
return {}
def list_objects(self, bucket_name, prefix=None, recursive=False):
try:
return self.client.list_objects(bucket_name,
prefix=prefix,
recursive=recursive)
except ResponseError as _err:
logging.exception("Could not list objects")
return []
def list_objects_v2(self, bucket_name, recursive=False):
try:
return self.client.list_objects_v2(bucket_name,
recursive=recursive)
except ResponseError as _err:
logging.exception("Could not list objects")
return []
def remove_objects(self, bucket_name, objects):
try:
for del_err in self.client.remove_objects(bucket_name, objects):
msg = "Deletion Error: {}".format(del_err)
log.debug(msg)
except ResponseError as _err:
logging.exception("Could remove objects")
return False
return True
def exists_object(self, bucket_name, object_name, recursive=False):
''' This seems to be pretty heavy, should be used with care
'''
try:
objects = self.list_objects(bucket_name, recursive=recursive)
for obj in objects:
if obj.object_name == object_name:
return True
except ResponseError as _err:
logging.exception("Could check object for existence")
return False
return False
def search(self,
bucket_name,
query,
recursive=True,
include_metadata=False):
results = []
objs = self.list_objects(bucket_name, recursive=recursive)
_query = re.compile(query, re.IGNORECASE)
for obj in objs:
if _query.search(obj.object_name):
results.append(obj)
if include_metadata:
metadata = self.get_metadata(bucket_name, obj.object_name)
for key in metadata.keys():
if _query.search(key) or _query.search(metadata[key]):
results.append(obj)
for r in results:
msg = "Object {} in bucket {} matches query {}".format(
r.object_name, r.bucket_name, query)
log.debug(msg)
return results
def create_presigned_put_url(self,
bucket_name,
object_name,
dt=timedelta(days=3)):
try:
return self.client.presigned_put_object(bucket_name,
object_name,
expires=dt)
except ResponseError as _err:
logging.exception("Could create presigned put url")
return ""
def create_presigned_get_url(self,
bucket_name,
object_name,
dt=timedelta(days=3)):
try:
return self.client.presigned_get_object(bucket_name,
object_name,
expires=dt)
except ResponseError as _err:
logging.exception("Could create presigned get url")
return ""
def copy_object(self, to_bucket_name, to_object_name,
from_bucket_object_name):
try:
ret = self.client.copy_object(to_bucket_name, to_object_name,
from_bucket_object_name,
CopyConditions())
print(ret)
return True
except ResponseError as _err:
logging.exception("Could not copy")
return False
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEY',
secret_key='YOUR-SECRETKEY')
# client.trace_on(sys.stderr)
copy_conditions = CopyConditions()
# Set modified condition, copy object modified since 2014 April.
t = (2014, 4, 0, 0, 0, 0, 0, 0, 0)
mod_since = datetime.utcfromtimestamp(time.mktime(t))
copy_conditions.set_modified_since(mod_since)
# Set unmodified condition, copy object unmodified since 2014 April.
# copy_conditions.set_unmodified_since(mod_since)
# Set matching ETag condition, copy object which matches the following ETag.
# copy_conditions.set_match_etag("31624deb84149d2f8ef9c385918b653a")
# Set matching ETag except condition, copy object which does not match the
# following ETag.
# copy_conditions.set_match_etag_except("31624deb84149d2f8ef9c385918b653a")
try:
copy_result = client.copy_object("my-bucket", "my-object",
"/my-sourcebucket/my-sourceobject",
copy_conditions)
print(copy_result)
except ResponseError as err:
print(err)
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and
# my-objectname are dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEY',
secret_key='YOUR-SECRETKEY')
# Set the metadata
metadata = {"test_meta_key": "test_meta_value"}
try:
copy_result = client.copy_object("my-bucket",
"my-object",
"/my-sourcebucket/my-sourceobject",
metadata=metadata)
print(copy_result)
except ResponseError as err:
print(err)
import uuid
ACCESS_KEY = "xx"
SECRET_KEY = "xx"
#Initialise MinioClient
minioClient = Minio('127.0.0.1:9000',
access_key=ACCESS_KEY,
secret_key=SECRET_KEY,
secure=False)
# Insert output from Deepspeech into this variable
metadata = {"Transcript": "xx", "Entities": "[[['xx','xx']],[['xx','xx']]]"}
# Append metadata onto original video and upload to new Minio Bucket
try:
copy_result = minioClient.copy_object("output_Bucket",
"updatedFileName",
"original_Bucket/fileToBeUpdated",
metadata=metadata)
except ResponseError as err:
print(err)
# Error-Checking to validate metadata has been appended to the corresponding video
try:
print(
minioClient.fget_object(bucket_name="output_Bucket",
object_name="updatedFileName"))
except ResponseError as err:
print(err)
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('play.minio.io:9000',
'Q3AM3UQ867SPQQA43P2F',
'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG')
_http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()
)
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
client.make_bucket(bucket_name)
is_s3 = client._endpoint_url.startswith("s3.amazonaws")
if is_s3:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
## Check if return codes a valid from server.
if is_s3:
try:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
except BucketAlreadyOwnedByYou as err:
pass
except BucketAlreadyExists as err:
pass
except ResponseError as err:
raise
# Check if bucket was created properly.
client.bucket_exists(bucket_name)
if is_s3:
client.bucket_exists(bucket_name+'.unique')
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
_, _ = bucket.name, bucket.creation_date
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data,
file_stat.st_size)
file_data.close()
with open('largefile', 'wb') as file_data:
for i in range(0, 104857):
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Fput a file
client.fput_object(bucket_name, object_name+'-f', 'testfile')
if is_s3:
client.fput_object(bucket_name, object_name+'-f', 'testfile',
metadata={'x-amz-storage-class': 'STANDARD_IA'})
# Fput a large file.
client.fput_object(bucket_name, object_name+'-large', 'largefile')
if is_s3:
client.fput_object(bucket_name, object_name+'-large', 'largefile',
metadata={'x-amz-storage-class': 'STANDARD_IA'})
# Copy a file
client.copy_object(bucket_name, object_name+'-copy',
'/'+bucket_name+'/'+object_name+'-f')
try:
copy_conditions = CopyConditions()
copy_conditions.set_match_etag('test-etag')
client.copy_object(bucket_name, object_name+'-copy',
'/'+bucket_name+'/'+object_name+'-f',
copy_conditions)
except PreconditionFailed as err:
if err.message != 'At least one of the preconditions you specified did not hold.':
raise
# Fetch stats on your object.
client.stat_object(bucket_name, object_name)
# Fetch stats on your object.
client.stat_object(bucket_name, object_name+'-f')
# Fetch stats on your large object.
client.stat_object(bucket_name, object_name+'-large')
# Fetch stats on your object.
client.stat_object(bucket_name, object_name+'-copy')
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
client.fget_object(bucket_name, object_name, 'newfile-f')
client.fput_object(bucket_name, object_name+'-f', 'testfile',
metadata={'x-amz-meta-testing': 'value'})
stat = client.fget_object(bucket_name, object_name+'-f', 'newfile-f-custom')
if not stat.metadata.has_key('X-Amz-Meta-Testing'):
raise ValueError('Metadata key \'x-amz-meta-testing\' not found')
value = stat.metadata['X-Amz-Meta-Testing']
if value != 'value':
raise ValueError('Metadata key has unexpected'
' value {0}'.format(value))
# List all object paths in bucket.
print("Listing using ListObjects")
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
_, _, _, _, _, _ = obj.bucket_name, obj.object_name, \
obj.last_modified, \
obj.etag, obj.size, \
obj.content_type
# List all object paths in bucket using V2 API.
print("Listing using ListObjectsV2")
objects = client.list_objects_v2(bucket_name, recursive=True)
for obj in objects:
_, _, _, _, _, _ = obj.bucket_name, obj.object_name, \
obj.last_modified, \
obj.etag, obj.size, \
obj.content_type
presigned_get_object_url = client.presigned_get_object(bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
raise ResponseError(response,
'GET',
bucket_name,
object_name).get_exception()
presigned_put_object_url = client.presigned_put_object(bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
raise ResponseError(response,
'PUT',
bucket_name,
object_name).get_exception()
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow()+timedelta(days=10)
policy.set_expires(expires_date)
client.presigned_post_policy(policy)
# Remove all objects.
client.remove_object(bucket_name, object_name)
client.remove_object(bucket_name, object_name+'-f')
client.remove_object(bucket_name, object_name+'-large')
client.remove_object(bucket_name, object_name+'-copy')
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Set read-only policy successfully.
client.set_bucket_policy(bucket_name, '1/', Policy.READ_ONLY)
# Set read-write policy successfully.
client.set_bucket_policy(bucket_name, '1/', Policy.READ_WRITE)
# Reset policy to NONE.
client.set_bucket_policy(bucket_name, '', Policy.NONE)
# Validate if the policy is reverted back to NONE.
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Upload some new objects to prepare for multi-object delete test.
print("Prepare for remove_objects() test.")
object_names = []
for i in range(10):
curr_object_name = object_name+"-{}".format(i)
# print("object-name: {}".format(curr_object_name))
client.fput_object(bucket_name, curr_object_name, "testfile")
object_names.append(curr_object_name)
# delete the objects in a single library call.
print("Performing remove_objects() test.")
del_errs = client.remove_objects(bucket_name, object_names)
had_errs = False
for del_err in del_errs:
had_errs = True
print("Remove objects err is {}".format(del_err))
if had_errs:
print("Removing objects FAILED - it had unexpected errors.")
raise
else:
print("Removing objects worked as expected.")
# Remove a bucket. This operation will only work if your bucket is empty.
print("Deleting buckets and finishing tests.")
client.remove_bucket(bucket_name)
if client._endpoint_url.startswith("s3.amazonaws"):
client.remove_bucket(bucket_name+'.unique')
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
os.remove('largefile')
os.remove('newfile-f-custom')
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name + '.unique', location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name + '.unique', location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name + '.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data,
file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name + '-f', 'testfile'))
# Copy a file
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f'))
try:
copy_conditions = CopyConditions()
copy_conditions.set_match_etag('test-etag')
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f',
copy_conditions))
except ResponseError as err:
if err.code != 'PreconditionFailed':
raise
if err.message != 'At least one of the pre-conditions you specified did not hold':
raise
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-f'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-copy'))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
print("Listing using ListObjects API")
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
# List all object paths in bucket using V2 API.
print("Listing using ListObjectsV2 API")
objects = client.list_objects_v2(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(
bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(
bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow() + timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name + '-f'))
print(client.remove_object(bucket_name, object_name + '-copy'))
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Set read-write policy successfully.
client.set_bucket_policy(bucket_name, '', Policy.READ_WRITE)
# Reset policy to NONE.
client.set_bucket_policy(bucket_name, '', Policy.NONE)
# Validate if the policy is reverted back to NONE.
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Upload some new objects to prepare for multi-object delete test.
print("Prepare for remove_objects() test.")
object_names = []
for i in range(10):
curr_object_name = object_name + "-{}".format(i)
print("object-name: {}".format(curr_object_name))
print(client.fput_object(bucket_name, curr_object_name, "testfile"))
object_names.append(curr_object_name)
# delete the objects in a single library call.
print("Performing remove_objects() test.")
del_errs = client.remove_objects(bucket_name, object_names)
had_errs = False
for del_err in del_errs:
had_errs = True
print("Err is {}".format(del_err))
if had_errs:
print("remove_objects() FAILED - it had unexpected errors.")
else:
print("remove_objects() worked as expected.")
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name + '.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
