def __init__(self, **kwargs: Any) -> None: """Initializes the FileOpBase instance""" import minio from minio.credentials import providers self.filepath = kwargs["filepath"] self.input_params = kwargs or [] self.cos_endpoint = urlparse(self.input_params.get("cos-endpoint")) self.cos_bucket = self.input_params.get("cos-bucket") # Infer secure from the endpoint's scheme. self.secure = self.cos_endpoint.scheme == "https" # get minio credentials provider if "cos-user" in self.input_params and "cos-password" in self.input_params: cred_provider = providers.StaticProvider( access_key=self.input_params.get("cos-user"), secret_key=self.input_params.get("cos-password"), ) elif "AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ: cred_provider = providers.EnvAWSProvider() elif "AWS_ROLE_ARN" in os.environ and "AWS_WEB_IDENTITY_TOKEN_FILE" in os.environ: cred_provider = providers.IamAwsProvider() else: raise RuntimeError( "No minio credentials provider can be initialised for current configs. " "Please validate your runtime configuration details and retry." ) # get minio client self.cos_client = minio.Minio(self.cos_endpoint.netloc, secure=self.secure, credentials=cred_provider)
def __init__(self, config=None, endpoint=None, access_key=None, secret_key=None, bucket=None, **kwargs): super().__init__(**kwargs) cred_provider = None if config is None: # The client was invoked by an entity that does not utilize # runtime configurations. if access_key is None or secret_key is None: # use env variables for authentication if (len(os.environ.get("AWS_ACCESS_KEY_ID", "").strip()) == 0 or len( os.environ.get("AWS_SECRET_ACCESS_KEY", "").strip()) == 0): raise RuntimeError( "Cannot connect to object storage. No credentials " " were provided and environment variables " " AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are not " " properly defined.") else: cred_provider = providers.EnvAWSProvider() else: # use provided username and password for authentication cred_provider = providers.StaticProvider( access_key=access_key, secret_key=secret_key, ) self.endpoint = endpoint self.bucket = bucket else: auth_type = config.metadata["cos_auth_type"] self.endpoint = urlparse(config.metadata["cos_endpoint"]) self.bucket = config.metadata["cos_bucket"] if auth_type in ["USER_CREDENTIALS", "KUBERNETES_SECRET"]: cred_provider = providers.StaticProvider( access_key=config.metadata["cos_username"], secret_key=config.metadata["cos_password"], ) elif auth_type == "AWS_IAM_ROLES_FOR_SERVICE_ACCOUNTS": if os.environ.get("AWS_ROLE_ARN") is None or os.environ.get( "AWS_WEB_IDENTITY_TOKEN_FILE") is None: raise RuntimeError( "Cannot connect to object storage. " f"Authentication provider '{auth_type}' requires " "environment variables AWS_ROLE_ARN and AWS_IAM_ROLES_FOR_SERVICE_ACCOUNTS." ) # Verify that AWS_WEB_IDENTITY_TOKEN_FILE exists if Path(os.environ["AWS_WEB_IDENTITY_TOKEN_FILE"]).is_file( ) is False: raise RuntimeError( "Cannot connect to object storage. The value of environment " "variable AWS_IAM_ROLES_FOR_SERVICE_ACCOUNTS references " f"'{os.environ['AWS_WEB_IDENTITY_TOKEN_FILE']}', which is not a valid file." ) cred_provider = providers.IamAwsProvider() else: raise RuntimeError( "Cannot connect to object storage. " f"Authentication provider '{auth_type}' is not supported.") # Infer secure from the endpoint's scheme. self.secure = self.endpoint.scheme == "https" # get minio client self.client = minio.Minio(self.endpoint.netloc, secure=self.secure, credentials=cred_provider) # Make a bucket with the make_bucket API call. try: if not self.client.bucket_exists(self.bucket): self.client.make_bucket(self.bucket) except S3Error as ex: # unpack the S3Error based off error codes # https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html if ex.code == "BucketAlreadyOwnedByYou": self.log.warning("Object Storage bucket already owned by you", exc_info=True) elif ex.code == "BucketAlreadyExists": self.log.warning("Object Storage bucket already exists", exc_info=True) elif ex.code == "SignatureDoesNotMatch": self.log.error("Incorrect Object Storage password supplied") elif ex.code == "InvalidAccessKeyId": self.log.error("Incorrect Object Storage username supplied") else: self.log.error(f"Object Storage error: {ex.code}", exc_info=True) raise ex from ex except ValueError as ex: # providers.IamAwsProvider raises this if something bad happened if isinstance(cred_provider, providers.IamAwsProvider): raise RuntimeError( f"Cannot connect to object storage: {ex}. Verify that " f"environment variable AWS_WEB_IDENTITY_TOKEN_FILE contains a valid value." ) else: raise ex