def __init__(self, **kwargs: Any) -> None:
"""Initializes the FileOpBase instance"""
import minio
from minio.credentials import providers
self.filepath = kwargs["filepath"]
self.input_params = kwargs or []
self.cos_endpoint = urlparse(self.input_params.get("cos-endpoint"))
self.cos_bucket = self.input_params.get("cos-bucket")
# Infer secure from the endpoint's scheme.
self.secure = self.cos_endpoint.scheme == "https"
# get minio credentials provider
if "cos-user" in self.input_params and "cos-password" in self.input_params:
cred_provider = providers.StaticProvider(
access_key=self.input_params.get("cos-user"),
secret_key=self.input_params.get("cos-password"),
)
elif "AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ:
cred_provider = providers.EnvAWSProvider()
elif "AWS_ROLE_ARN" in os.environ and "AWS_WEB_IDENTITY_TOKEN_FILE" in os.environ:
cred_provider = providers.IamAwsProvider()
else:
raise RuntimeError(
"No minio credentials provider can be initialised for current configs. "
"Please validate your runtime configuration details and retry."
)
# get minio client
self.cos_client = minio.Minio(self.cos_endpoint.netloc, secure=self.secure, credentials=cred_provider)
def __init__(self,
config=None,
endpoint=None,
access_key=None,
secret_key=None,
bucket=None,
**kwargs):
super().__init__(**kwargs)
cred_provider = None
if config is None:
# The client was invoked by an entity that does not utilize
# runtime configurations.
if access_key is None or secret_key is None:
# use env variables for authentication
if (len(os.environ.get("AWS_ACCESS_KEY_ID", "").strip()) == 0
or len(
os.environ.get("AWS_SECRET_ACCESS_KEY",
"").strip()) == 0):
raise RuntimeError(
"Cannot connect to object storage. No credentials "
" were provided and environment variables "
" AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are not "
" properly defined.")
else:
cred_provider = providers.EnvAWSProvider()
else:
# use provided username and password for authentication
cred_provider = providers.StaticProvider(
access_key=access_key,
secret_key=secret_key,
)
self.endpoint = endpoint
self.bucket = bucket
else:
auth_type = config.metadata["cos_auth_type"]
self.endpoint = urlparse(config.metadata["cos_endpoint"])
self.bucket = config.metadata["cos_bucket"]
if auth_type in ["USER_CREDENTIALS", "KUBERNETES_SECRET"]:
cred_provider = providers.StaticProvider(
access_key=config.metadata["cos_username"],
secret_key=config.metadata["cos_password"],
)
elif auth_type == "AWS_IAM_ROLES_FOR_SERVICE_ACCOUNTS":
if os.environ.get("AWS_ROLE_ARN") is None or os.environ.get(
"AWS_WEB_IDENTITY_TOKEN_FILE") is None:
raise RuntimeError(
"Cannot connect to object storage. "
f"Authentication provider '{auth_type}' requires "
"environment variables AWS_ROLE_ARN and AWS_IAM_ROLES_FOR_SERVICE_ACCOUNTS."
)
# Verify that AWS_WEB_IDENTITY_TOKEN_FILE exists
if Path(os.environ["AWS_WEB_IDENTITY_TOKEN_FILE"]).is_file(
) is False:
raise RuntimeError(
"Cannot connect to object storage. The value of environment "
"variable AWS_IAM_ROLES_FOR_SERVICE_ACCOUNTS references "
f"'{os.environ['AWS_WEB_IDENTITY_TOKEN_FILE']}', which is not a valid file."
)
cred_provider = providers.IamAwsProvider()
else:
raise RuntimeError(
"Cannot connect to object storage. "
f"Authentication provider '{auth_type}' is not supported.")
# Infer secure from the endpoint's scheme.
self.secure = self.endpoint.scheme == "https"
# get minio client
self.client = minio.Minio(self.endpoint.netloc,
secure=self.secure,
credentials=cred_provider)
# Make a bucket with the make_bucket API call.
try:
if not self.client.bucket_exists(self.bucket):
self.client.make_bucket(self.bucket)
except S3Error as ex:
# unpack the S3Error based off error codes
# https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
if ex.code == "BucketAlreadyOwnedByYou":
self.log.warning("Object Storage bucket already owned by you",
exc_info=True)
elif ex.code == "BucketAlreadyExists":
self.log.warning("Object Storage bucket already exists",
exc_info=True)
elif ex.code == "SignatureDoesNotMatch":
self.log.error("Incorrect Object Storage password supplied")
elif ex.code == "InvalidAccessKeyId":
self.log.error("Incorrect Object Storage username supplied")
else:
self.log.error(f"Object Storage error: {ex.code}",
exc_info=True)
raise ex from ex
except ValueError as ex:
# providers.IamAwsProvider raises this if something bad happened
if isinstance(cred_provider, providers.IamAwsProvider):
raise RuntimeError(
f"Cannot connect to object storage: {ex}. Verify that "
f"environment variable AWS_WEB_IDENTITY_TOKEN_FILE contains a valid value."
)
else:
raise ex