def get_credential_to_access_bucket(self, aws_creds, expires_in):
s3_buckets = get_value(
flask.current_app.config,
"S3_BUCKETS",
InternalError("buckets not configured"),
)
if len(aws_creds) == 0 and len(s3_buckets) == 0:
raise InternalError("no bucket is configured")
if len(aws_creds) == 0 and len(s3_buckets) > 0:
raise InternalError("credential for buckets is not configured")
bucket_cred = None
for pattern in s3_buckets:
if re.match("^" + pattern + "$", self.parsed_url.netloc):
bucket_cred = s3_buckets[pattern]
break
if bucket_cred is None:
raise Unauthorized("permission denied for bucket")
cred_key = get_value(
bucket_cred, "cred",
InternalError("credential of that bucket is missing"))
if cred_key == "*":
return {"aws_access_key_id": "*"}
if "role-arn" not in bucket_cred:
return get_value(
aws_creds,
cred_key,
InternalError("aws credential of that bucket is not found"),
)
else:
return S3IndexedFileLocation.assume_role(aws_creds, bucket_cred,
cred_key, expires_in)
def assume_role(cls, bucket_cred, expires_in, aws_creds_config):
role_arn = get_value(
bucket_cred, "role-arn",
InternalError("role-arn of that bucket is missing"))
assumed_role = flask.current_app.boto.assume_role(
role_arn, expires_in, aws_creds_config)
cred = get_value(assumed_role, "Credentials",
InternalError("fail to assume role"))
return {
"aws_access_key_id":
get_value(
cred,
"AccessKeyId",
InternalError("outdated format. AccessKeyId missing"),
),
"aws_secret_access_key":
get_value(
cred,
"SecretAccessKey",
InternalError("outdated format. SecretAccessKey missing"),
),
"aws_session_token":
get_value(
cred,
"SessionToken",
InternalError("outdated format. Sesssion token missing"),
),
}
def get_signed_url(
self, action, expires_in, public_data=False, force_signed_url=True, **kwargs
):
aws_creds = get_value(
config, "AWS_CREDENTIALS", InternalError("credentials not configured")
)
s3_buckets = get_value(
config, "S3_BUCKETS", InternalError("buckets not configured")
)
bucket_name = self.bucket_name()
bucket = s3_buckets.get(bucket_name)
if bucket and bucket.get("endpoint_url"):
http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
self.parsed_url.netloc, self.parsed_url.path.strip("/")
)
else:
http_url = "https://{}.s3.amazonaws.com/{}".format(
self.parsed_url.netloc, self.parsed_url.path.strip("/")
)
credential = S3IndexedFileLocation.get_credential_to_access_bucket(
bucket_name, aws_creds, expires_in
)
# if it's public and we don't need to force the signed url, just return the raw
# s3 url
aws_access_key_id = get_value(
credential,
"aws_access_key_id",
InternalError("aws configuration not found"),
)
# `aws_access_key_id == "*"` is a special case to support public buckets
# where we do *not* want to try signing at all. the other case is that the
# data is public and user requested to not sign the url
if aws_access_key_id == "*" or (public_data and not force_signed_url):
return http_url
region = self.get_bucket_region()
if not region and not bucket.get("endpoint_url"):
region = flask.current_app.boto.get_bucket_region(
self.parsed_url.netloc, credential
)
user_info = _get_user_info()
url = generate_aws_presigned_url(
http_url,
ACTION_DICT["s3"][action],
credential,
"s3",
region,
expires_in,
user_info,
)
return url
def generate_presigne_url_for_part_upload(self, uploadId, partNumber, expires_in):
"""
Generate presigned url for uploading object part given uploadId and part number
Args:
uploadId(str): uploadID of the multipart upload
partNumber(int): part number
expires(int): expiration time
Returns:
presigned_url(str)
"""
aws_creds = get_value(
config, "AWS_CREDENTIALS", InternalError("credentials not configured")
)
credential = S3IndexedFileLocation.get_credential_to_access_bucket(
self.bucket_name(), aws_creds, expires_in
)
region = self.get_bucket_region()
if not region:
region = flask.current_app.boto.get_bucket_region(
self.parsed_url.netloc, credential
)
return multipart_upload.generate_presigned_url_for_uploading_part(
self.parsed_url.netloc,
self.parsed_url.path.strip("/"),
credential,
uploadId,
partNumber,
region,
expires_in,
)
def get_credential_to_access_bucket(
cls, bucket_name, aws_creds, expires_in, boto=None
):
s3_buckets = get_value(
config, "S3_BUCKETS", InternalError("buckets not configured")
)
if len(aws_creds) == 0 and len(s3_buckets) == 0:
raise InternalError("no bucket is configured")
if len(aws_creds) == 0 and len(s3_buckets) > 0:
raise InternalError("credential for buckets is not configured")
bucket_cred = s3_buckets.get(bucket_name)
if bucket_cred is None:
raise Unauthorized("permission denied for bucket")
cred_key = get_value(
bucket_cred, "cred", InternalError("credential of that bucket is missing")
)
# this is a special case to support public buckets where we do *not* want to
# try signing at all
if cred_key == "*":
return {"aws_access_key_id": "*"}
if "role-arn" not in bucket_cred:
return get_value(
aws_creds,
cred_key,
InternalError("aws credential of that bucket is not found"),
)
else:
aws_creds_config = get_value(
aws_creds,
cred_key,
InternalError("aws credential of that bucket is not found"),
)
return S3IndexedFileLocation.assume_role(
bucket_cred, expires_in, aws_creds_config, boto
)
def assume_role(cls, bucket_cred, expires_in, aws_creds_config, boto=None):
"""
Args:
bucket_cred
expires_in
aws_creds_config
boto (optional): provide `boto` when calling this function
outside of application context, to avoid errors when
using `flask.current_app`.
"""
boto = boto or flask.current_app.boto
role_arn = get_value(
bucket_cred, "role-arn",
InternalError("role-arn of that bucket is missing"))
assumed_role = boto.assume_role(role_arn, expires_in, aws_creds_config)
cred = get_value(assumed_role, "Credentials",
InternalError("fail to assume role"))
return {
"aws_access_key_id":
get_value(
cred,
"AccessKeyId",
InternalError("outdated format. AccessKeyId missing"),
),
"aws_secret_access_key":
get_value(
cred,
"SecretAccessKey",
InternalError("outdated format. SecretAccessKey missing"),
),
"aws_session_token":
get_value(
cred,
"SessionToken",
InternalError("outdated format. Sesssion token missing"),
),
}
def bucket_name(self):
"""
Return:
Optional[str]: bucket name or None if not not in cofig
"""
s3_buckets = get_value(
flask.current_app.config,
"S3_BUCKETS",
InternalError("buckets not configured"),
)
for bucket in s3_buckets:
if re.match("^" + bucket + "$", self.parsed_url.netloc):
return bucket
return None
def get_bucket_region(self):
s3_buckets = get_value(config, "S3_BUCKETS",
InternalError("buckets not configured"))
if len(s3_buckets) == 0:
return None
bucket_cred = s3_buckets.get(self.bucket_name())
if bucket_cred is None:
return None
if "region" not in bucket_cred:
return None
else:
return bucket_cred["region"]
def get_signed_url(self, action, expires_in, public_data=False):
aws_creds = get_value(
flask.current_app.config,
"AWS_CREDENTIALS",
InternalError("credentials not configured"),
)
http_url = "https://{}.s3.amazonaws.com/{}".format(
self.parsed_url.netloc, self.parsed_url.path.strip("/"))
config = self.get_credential_to_access_bucket(aws_creds, expires_in)
aws_access_key_id = get_value(
config, "aws_access_key_id",
InternalError("aws configuration not found"))
if aws_access_key_id == "*":
return http_url
region = flask.current_app.boto.get_bucket_region(
self.parsed_url.netloc, config)
user_info = {}
if not public_data:
user_info = S3IndexedFileLocation.get_user_info()
url = generate_aws_presigned_url(
http_url,
ACTION_DICT["s3"][action],
config,
"s3",
region,
expires_in,
user_info,
)
return url
def init_multipart_upload(self, expires_in):
"""
Initialize multipart upload
Args:
expires(int): expiration time
Returns:
UploadId(str)
"""
aws_creds = get_value(config, "AWS_CREDENTIALS",
InternalError("credentials not configured"))
credentials = S3IndexedFileLocation.get_credential_to_access_bucket(
self.bucket_name(), aws_creds, expires_in)
return multipart_upload.initilize_multipart_upload(
self.parsed_url.netloc, self.parsed_url.path.strip("/"),
credentials)
def complete_multipart_upload(self, uploadId, parts, expires_in):
"""
Complete multipart upload.
Args:
uploadId(str): upload id of the current upload
parts(list(set)): List of part infos
[{"Etag": "1234567", "PartNumber": 1}, {"Etag": "4321234", "PartNumber": 2}]
"""
aws_creds = get_value(config, "AWS_CREDENTIALS",
InternalError("credentials not configured"))
credentials = S3IndexedFileLocation.get_credential_to_access_bucket(
self.bucket_name(), aws_creds, expires_in)
multipart_upload.complete_multipart_upload(
self.parsed_url.netloc,
self.parsed_url.path.strip("/"),
credentials,
uploadId,
parts,
)
