def __init__(
self,
name: str,
datasource_name: str,
bucket: str,
execution_engine: Optional[ExecutionEngine] = None,
default_regex: Optional[dict] = None,
sorters: Optional[list] = None,
prefix: str = "",
delimiter: str = "/",
max_keys: int = 1000,
boto3_options: Optional[dict] = None,
batch_spec_passthrough: Optional[dict] = None,
) -> None:
"""
InferredAssetS3DataConnector for connecting to S3.
Args:
name (str): required name for data_connector
datasource_name (str): required name for datasource
bucket (str): bucket for S3
execution_engine (ExecutionEngine): optional reference to ExecutionEngine
default_regex (dict): optional regex configuration for filtering data_references
sorters (list): optional list of sorters for sorting data_references
prefix (str): S3 prefix
delimiter (str): S3 delimiter
max_keys (int): S3 max_keys (default is 1000)
boto3_options (dict): optional boto3 options
batch_spec_passthrough (dict): dictionary with keys that will be added directly to batch_spec
"""
logger.debug(f'Constructing InferredAssetS3DataConnector "{name}".')
super().__init__(
name=name,
datasource_name=datasource_name,
execution_engine=execution_engine,
default_regex=default_regex,
sorters=sorters,
batch_spec_passthrough=batch_spec_passthrough,
)
self._bucket = bucket
self._prefix = ConfiguredAssetS3DataConnector.sanitize_prefix_for_s3(
prefix)
self._delimiter = delimiter
self._max_keys = max_keys
if boto3_options is None:
boto3_options = {}
try:
self._s3 = boto3.client("s3", **boto3_options)
except (TypeError, AttributeError):
raise ImportError(
"Unable to load boto3 (it is required for InferredAssetS3DataConnector)."
)
def check_sameness(prefix, expected_output):
s3_sanitized = ConfiguredAssetS3DataConnector.sanitize_prefix_for_s3(prefix)
file_system_sanitized = FilePathDataConnector.sanitize_prefix(prefix)
if os.sep == "\\": # Fix to ensure tests work on Windows
file_system_sanitized = file_system_sanitized.replace("\\", "/")
assert file_system_sanitized == expected_output, (
f"Expected output does not match original sanitization behavior, got "
f"{file_system_sanitized} instead of {expected_output}"
)
assert (
s3_sanitized == expected_output == file_system_sanitized
), f'S3 sanitized result is incorrect, "{s3_sanitized} instead of {expected_output}'