def _copy_objects(
batch: List[Tuple[str, str]],
use_threads: bool,
boto3_session: boto3.Session,
s3_additional_kwargs: Optional[Dict[str, Any]],
) -> None:
_logger.debug("len(batch): %s", len(batch))
client_s3: boto3.client = _utils.client(service_name="s3",
session=boto3_session)
resource_s3: boto3.resource = _utils.resource(service_name="s3",
session=boto3_session)
if s3_additional_kwargs is None:
boto3_kwargs: Optional[Dict[str, Any]] = None
else:
boto3_kwargs = get_botocore_valid_kwargs(
function_name="copy_object",
s3_additional_kwargs=s3_additional_kwargs)
for source, target in batch:
source_bucket, source_key = _utils.parse_path(path=source)
copy_source: Dict[str, str] = {
"Bucket": source_bucket,
"Key": source_key
}
target_bucket, target_key = _utils.parse_path(path=target)
resource_s3.meta.client.copy(
CopySource=copy_source,
Bucket=target_bucket,
Key=target_key,
SourceClient=client_s3,
ExtraArgs=boto3_kwargs,
Config=TransferConfig(num_download_attempts=10,
use_threads=use_threads),
)
def create_athena_bucket(boto3_session: Optional[boto3.Session] = None) -> str:
"""Create the default Athena bucket if it doesn't exist.
Parameters
----------
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
str
Bucket s3 path (E.g. s3://aws-athena-query-results-ACCOUNT-REGION/)
Examples
--------
>>> import awswrangler as wr
>>> wr.athena.create_athena_bucket()
's3://aws-athena-query-results-ACCOUNT-REGION/'
"""
session: boto3.Session = _utils.ensure_session(session=boto3_session)
account_id: str = sts.get_account_id(boto3_session=session)
region_name: str = str(session.region_name).lower()
s3_output = f"s3://aws-athena-query-results-{account_id}-{region_name}/"
s3_resource = _utils.resource(service_name="s3", session=session)
s3_resource.Bucket(s3_output)
return s3_output
def create_athena_bucket(boto3_session: Optional[boto3.Session] = None) -> str:
"""Create the default Athena bucket if it doesn't exist.
Parameters
----------
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
str
Bucket s3 path (E.g. s3://aws-athena-query-results-ACCOUNT-REGION/)
Examples
--------
>>> import awswrangler as wr
>>> wr.athena.create_athena_bucket()
's3://aws-athena-query-results-ACCOUNT-REGION/'
"""
session: boto3.Session = _utils.ensure_session(session=boto3_session)
account_id: str = sts.get_account_id(boto3_session=session)
region_name: str = str(session.region_name).lower()
bucket_name = f"aws-athena-query-results-{account_id}-{region_name}"
path = f"s3://{bucket_name}/"
resource = _utils.resource(service_name="s3", session=session)
bucket = resource.Bucket(bucket_name)
args = {} if region_name == "us-east-1" else {
"CreateBucketConfiguration": {
"LocationConstraint": region_name
}
}
try:
bucket.create(**args)
except resource.meta.client.exceptions.BucketAlreadyOwnedByYou as err:
_logger.debug("Bucket %s already exists.",
err.response["Error"]["BucketName"])
except botocore.exceptions.ClientError as err:
if err.response["Error"]["Code"] == "OperationAborted":
_logger.debug(
"A conflicting conditional operation is currently in progress against this resource."
)
bucket.wait_until_exists()
return path
def _copy_objects(batch: List[Tuple[str, str]], use_threads: bool,
boto3_session: boto3.Session) -> None:
_logger.debug("len(batch): %s", len(batch))
client_s3: boto3.client = _utils.client(service_name="s3",
session=boto3_session)
resource_s3: boto3.resource = _utils.resource(service_name="s3",
session=boto3_session)
for source, target in batch:
source_bucket, source_key = _utils.parse_path(path=source)
copy_source: Dict[str, str] = {
"Bucket": source_bucket,
"Key": source_key
}
target_bucket, target_key = _utils.parse_path(path=target)
resource_s3.meta.client.copy(
CopySource=copy_source,
Bucket=target_bucket,
Key=target_key,
SourceClient=client_s3,
Config=TransferConfig(num_download_attempts=15,
use_threads=use_threads),
)
def get_table(
table_name: str,
boto3_session: Optional[boto3.Session] = None,
) -> boto3.resource:
"""Get DynamoDB table object for specified table name.
Parameters
----------
table_name : str
Name of the Amazon DynamoDB table.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 Session will be used if boto3_session receive None.
Returns
-------
dynamodb_table : boto3.resources.dynamodb.Table
Boto3 DynamoDB.Table object.
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table
"""
dynamodb_resource = _utils.resource(service_name="dynamodb",
session=boto3_session)
dynamodb_table = dynamodb_resource.Table(table_name)
return dynamodb_table
