def main():
"""
Purpose:
Read an .avro File
"""
logging.info("Starting Create Bucket in Minio")
opts = get_options()
minio_url =\
minio_connection_helpers.build_minio_url(opts.minio_host, opts.minio_port)
minio_client = minio_connection_helpers.connect_to_minio(
minio_url, opts.access_key, opts.secret_key)
buckets = minio_bucket_helpers.get_buckets(minio_client)
bucket_names = minio_bucket_helpers.get_bucket_names(minio_client)
if opts.bucket_name not in bucket_names:
logging.info(
f"{opts.bucket_name} doesn't exist, expecting an exception")
try:
minio_bucket_helpers.delete_bucket(minio_client, opts.bucket_name)
except BucketDoesntExist as ba_err:
logging.error(f"Got expected Error: {ba_err}")
except Exception as err:
logging.error(f"Unexpected Error Deleting Bucket: {err}")
raise err
import pdb
pdb.set_trace()
logging.info("Create Bucket in Minio Complete")
def __init__(self, minio_host, access_key, secret_key, minio_port=9000):
"""
Purpose:
Initilize the MinioClient Class.
Args:
minio_host (String): Host for Minio
access_key (String): Access Key for Minio
secret_key (String): Secret Key for Minio
minio_port (Int): Port for Minio (Defaults to 9000)
Returns:
N/A
"""
logging.info(
f"Initializing MinioClient Object Connected to {minio_host}:{minio_port}"
)
self.minio_host = minio_host
self.minio_port = minio_port
self.access_key = access_key
self.secret_key = secret_key
self.minio_url = f"http://{minio_host}:{minio_port}"
self.minio_url = minio_connection_helpers.build_minio_url(
self.minio_host, self.minio_port)
self.minio_client = minio_connection_helpers.connect_to_minio(
self.minio_url, self.access_key, self.secret_key)
def main():
"""
Purpose:
Read an .avro File
"""
logging.info("Starting Get Objects from Buckets in Minio")
opts = get_options()
minio_url =\
minio_connection_helpers.build_minio_url(opts.minio_host, opts.minio_port)
minio_client = minio_connection_helpers.connect_to_minio(
minio_url, opts.access_key, opts.secret_key)
if not opts.object_names:
objects = minio_object_helpers.get_objects(minio_client,
opts.bucket_name)
object_names = minio_object_helpers.get_object_names(
minio_client, opts.bucket_name)
else:
object_names = opts.object_names
for object_name in object_names:
object_exists = minio_object_helpers.is_object_in_bucket(
minio_client, opts.bucket_name, object_name)
if not object_exists:
logging.error(f"{object_name} doesnt exist in {opts.bucket_name}")
continue
if opts.download_to_memory:
minio_object = minio_object_helpers.download_object_to_memory(
minio_client, opts.bucket_name, object_name)
import pdb
pdb.set_trace()
else:
minio_object_helpers.download_object_to_file(
minio_client,
opts.bucket_name,
object_name,
filename=f"{opts.download_dir}/{object_name}",
)
import pdb
pdb.set_trace()
logging.info("Get Objects from Buckets in Minio Complete")
def main():
"""
Purpose:
Read an .avro File
"""
logging.info("Starting Connect to Minio")
opts = get_options()
minio_client_1 = minio_client.MinioClient(opts.minio_host,
opts.access_key,
opts.secret_key,
minio_port=opts.minio_port)
minio_url =\
minio_connection_helpers.build_minio_url(opts.minio_host, opts.minio_port)
minio_client_2 = minio_connection_helpers.connect_to_minio(
minio_url, opts.access_key, opts.secret_key)
import pdb
pdb.set_trace()
logging.info("Connect to Minio Complete")