def __init__( self, name: str, datasource_name: str, bucket: str, execution_engine: Optional[ExecutionEngine] = None, default_regex: Optional[dict] = None, sorters: Optional[list] = None, prefix: str = "", delimiter: str = "/", max_keys: int = 1000, boto3_options: Optional[dict] = None, batch_spec_passthrough: Optional[dict] = None, ) -> None: """ InferredAssetS3DataConnector for connecting to S3. Args: name (str): required name for data_connector datasource_name (str): required name for datasource bucket (str): bucket for S3 execution_engine (ExecutionEngine): optional reference to ExecutionEngine default_regex (dict): optional regex configuration for filtering data_references sorters (list): optional list of sorters for sorting data_references prefix (str): S3 prefix delimiter (str): S3 delimiter max_keys (int): S3 max_keys (default is 1000) boto3_options (dict): optional boto3 options batch_spec_passthrough (dict): dictionary with keys that will be added directly to batch_spec """ logger.debug(f'Constructing InferredAssetS3DataConnector "{name}".') super().__init__( name=name, datasource_name=datasource_name, execution_engine=execution_engine, default_regex=default_regex, sorters=sorters, batch_spec_passthrough=batch_spec_passthrough, ) self._bucket = bucket self._prefix = ConfiguredAssetS3DataConnector.sanitize_prefix_for_s3( prefix) self._delimiter = delimiter self._max_keys = max_keys if boto3_options is None: boto3_options = {} try: self._s3 = boto3.client("s3", **boto3_options) except (TypeError, AttributeError): raise ImportError( "Unable to load boto3 (it is required for InferredAssetS3DataConnector)." )
def check_sameness(prefix, expected_output): s3_sanitized = ConfiguredAssetS3DataConnector.sanitize_prefix_for_s3(prefix) file_system_sanitized = FilePathDataConnector.sanitize_prefix(prefix) if os.sep == "\\": # Fix to ensure tests work on Windows file_system_sanitized = file_system_sanitized.replace("\\", "/") assert file_system_sanitized == expected_output, ( f"Expected output does not match original sanitization behavior, got " f"{file_system_sanitized} instead of {expected_output}" ) assert ( s3_sanitized == expected_output == file_system_sanitized ), f'S3 sanitized result is incorrect, "{s3_sanitized} instead of {expected_output}'