def get_aws_s3_signed_url(request) -> JsonResponse:
"""Generate a pre-signed S3 POST URL."""
body = json.loads(request.body)
s3_client = boto3.client(
's3',
'eu-central-1', # Not specifying this breaks.
config=BotoConfig(s3={'addressing_style': 'path'}),
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
)
path = get_upload_to_hashed_path(None, body['filename'])
try:
response = s3_client.generate_presigned_post(
settings.AWS_UPLOADS_BUCKET_NAME,
str(path),
Fields=None,
Conditions=None,
ExpiresIn=600,
)
except ClientError as e:
logging.error(e)
raise SuspiciousOperation("Error while creating signed url")
return JsonResponse({'method': 'POST', 'url': response['url'], 'fields': response['fields']})
def __init__(
self,
region_name=None,
bucket_name=None,
accelerated_transfer=True,
**kwargs,
):
self._region_name = region_name or S3_UPLOADS_REGION
self._bucket_name = bucket_name or S3_UPLOADS_BUCKET
self.client = boto3.client(
's3',
region_name=self._region_name,
aws_access_key_id=AWS_MANAGER_PUBLIC_KEY,
aws_secret_access_key=AWS_MANAGER_PRIVATE_KEY,
config=BotoConfig(s3={
"use_accelerate_endpoint": accelerated_transfer,
}))
self._config = TransferConfig(
multipart_threshold=25 * 1024 * 1024, # 25 MB
max_concurrency=15,
num_download_attempts=5,
io_chunksize=1024 * 1024, # 1 MB
)
self.transfer = BotoTransfer(self.client, self._config)
def get_s3_client() -> Any:
"""Get the boto3 s3 client."""
client_name = 's3'
if client_name not in CLIENTS:
session = get_session()
CLIENTS[client_name] = session.client(client_name, config=BotoConfig(
max_pool_connections=1,
retries=dict(max_attempts=2)
))
return CLIENTS[client_name]
def _securityTokenService(self,
clientType,
roleSessionName,
regionName,
timeout=None):
session = boto3.Session(profile_name=self.config.s3_aws_profile,
region_name=regionName)
if self.config.s3_aws_role == None:
if timeout != None:
botoConfig = BotoConfig(retries={'max_attempts': 0},
read_timeout=timeout,
connect_timeout=timeout)
client = session.client(clientType, config=botoConfig)
else:
client = session.client(clientType)
else:
stsClient = session.client('sts')
assumedRoleObject = stsClient.assume_role(
RoleArn=self.config.s3_aws_role,
RoleSessionName=roleSessionName,
DurationSeconds=12 * 60 * 60)
credentials = assumedRoleObject['Credentials']
if timeout != None:
botoConfig = BotoConfig(retries={'max_attempts': 0},
read_timeout=timeout,
connect_timeout=timeout)
client = boto3.client(
clientType,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
region_name=regionName,
config=botoConfig)
else:
client = boto3.client(
clientType,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
region_name=regionName)
#print("Created role %s based session for %s." % (self.config.s3_aws_role, clientType))
return client
def client(self,
service: str,
profile: str = "default",
region: str = None) -> boto3.client:
region = self._get_region(region, profile)
session = self.session(profile, region)
kwargs = {"config": BotoConfig(retries={"max_attempts": 20})}
if service in REGIONAL_ENDPOINT_SERVICES:
kwargs.update(
{"endpoint_url": self._get_endpoint_url(service, region)})
return self._cache_lookup(
self._client_cache,
[profile, region, service],
session.client,
[service],
kwargs,
)
def _emr_client(self):
# Use an increased number of retries since DescribeStep calls have a pretty low rate limit.
config = BotoConfig(retries={"max_attempts": 10, "mode": "standard"})
return boto3.client("emr", region_name=self._region, config=config)