def _copy_objects( batch: List[Tuple[str, str]], use_threads: bool, boto3_session: boto3.Session, s3_additional_kwargs: Optional[Dict[str, Any]], ) -> None: _logger.debug("len(batch): %s", len(batch)) client_s3: boto3.client = _utils.client(service_name="s3", session=boto3_session) resource_s3: boto3.resource = _utils.resource(service_name="s3", session=boto3_session) if s3_additional_kwargs is None: boto3_kwargs: Optional[Dict[str, Any]] = None else: boto3_kwargs = get_botocore_valid_kwargs( function_name="copy_object", s3_additional_kwargs=s3_additional_kwargs) for source, target in batch: source_bucket, source_key = _utils.parse_path(path=source) copy_source: Dict[str, str] = { "Bucket": source_bucket, "Key": source_key } target_bucket, target_key = _utils.parse_path(path=target) resource_s3.meta.client.copy( CopySource=copy_source, Bucket=target_bucket, Key=target_key, SourceClient=client_s3, ExtraArgs=boto3_kwargs, Config=TransferConfig(num_download_attempts=10, use_threads=use_threads), )
def create_athena_bucket(boto3_session: Optional[boto3.Session] = None) -> str: """Create the default Athena bucket if it doesn't exist. Parameters ---------- boto3_session : boto3.Session(), optional Boto3 Session. The default boto3 session will be used if boto3_session receive None. Returns ------- str Bucket s3 path (E.g. s3://aws-athena-query-results-ACCOUNT-REGION/) Examples -------- >>> import awswrangler as wr >>> wr.athena.create_athena_bucket() 's3://aws-athena-query-results-ACCOUNT-REGION/' """ session: boto3.Session = _utils.ensure_session(session=boto3_session) account_id: str = sts.get_account_id(boto3_session=session) region_name: str = str(session.region_name).lower() s3_output = f"s3://aws-athena-query-results-{account_id}-{region_name}/" s3_resource = _utils.resource(service_name="s3", session=session) s3_resource.Bucket(s3_output) return s3_output
def create_athena_bucket(boto3_session: Optional[boto3.Session] = None) -> str: """Create the default Athena bucket if it doesn't exist. Parameters ---------- boto3_session : boto3.Session(), optional Boto3 Session. The default boto3 session will be used if boto3_session receive None. Returns ------- str Bucket s3 path (E.g. s3://aws-athena-query-results-ACCOUNT-REGION/) Examples -------- >>> import awswrangler as wr >>> wr.athena.create_athena_bucket() 's3://aws-athena-query-results-ACCOUNT-REGION/' """ session: boto3.Session = _utils.ensure_session(session=boto3_session) account_id: str = sts.get_account_id(boto3_session=session) region_name: str = str(session.region_name).lower() bucket_name = f"aws-athena-query-results-{account_id}-{region_name}" path = f"s3://{bucket_name}/" resource = _utils.resource(service_name="s3", session=session) bucket = resource.Bucket(bucket_name) args = {} if region_name == "us-east-1" else { "CreateBucketConfiguration": { "LocationConstraint": region_name } } try: bucket.create(**args) except resource.meta.client.exceptions.BucketAlreadyOwnedByYou as err: _logger.debug("Bucket %s already exists.", err.response["Error"]["BucketName"]) except botocore.exceptions.ClientError as err: if err.response["Error"]["Code"] == "OperationAborted": _logger.debug( "A conflicting conditional operation is currently in progress against this resource." ) bucket.wait_until_exists() return path
def _copy_objects(batch: List[Tuple[str, str]], use_threads: bool, boto3_session: boto3.Session) -> None: _logger.debug("len(batch): %s", len(batch)) client_s3: boto3.client = _utils.client(service_name="s3", session=boto3_session) resource_s3: boto3.resource = _utils.resource(service_name="s3", session=boto3_session) for source, target in batch: source_bucket, source_key = _utils.parse_path(path=source) copy_source: Dict[str, str] = { "Bucket": source_bucket, "Key": source_key } target_bucket, target_key = _utils.parse_path(path=target) resource_s3.meta.client.copy( CopySource=copy_source, Bucket=target_bucket, Key=target_key, SourceClient=client_s3, Config=TransferConfig(num_download_attempts=15, use_threads=use_threads), )
def get_table( table_name: str, boto3_session: Optional[boto3.Session] = None, ) -> boto3.resource: """Get DynamoDB table object for specified table name. Parameters ---------- table_name : str Name of the Amazon DynamoDB table. boto3_session : boto3.Session(), optional Boto3 Session. The default boto3 Session will be used if boto3_session receive None. Returns ------- dynamodb_table : boto3.resources.dynamodb.Table Boto3 DynamoDB.Table object. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table """ dynamodb_resource = _utils.resource(service_name="dynamodb", session=boto3_session) dynamodb_table = dynamodb_resource.Table(table_name) return dynamodb_table