def get_credential_to_access_bucket(self, aws_creds, expires_in): s3_buckets = get_value( flask.current_app.config, "S3_BUCKETS", InternalError("buckets not configured"), ) if len(aws_creds) == 0 and len(s3_buckets) == 0: raise InternalError("no bucket is configured") if len(aws_creds) == 0 and len(s3_buckets) > 0: raise InternalError("credential for buckets is not configured") bucket_cred = None for pattern in s3_buckets: if re.match("^" + pattern + "$", self.parsed_url.netloc): bucket_cred = s3_buckets[pattern] break if bucket_cred is None: raise Unauthorized("permission denied for bucket") cred_key = get_value( bucket_cred, "cred", InternalError("credential of that bucket is missing")) if cred_key == "*": return {"aws_access_key_id": "*"} if "role-arn" not in bucket_cred: return get_value( aws_creds, cred_key, InternalError("aws credential of that bucket is not found"), ) else: return S3IndexedFileLocation.assume_role(aws_creds, bucket_cred, cred_key, expires_in)
def assume_role(cls, bucket_cred, expires_in, aws_creds_config): role_arn = get_value( bucket_cred, "role-arn", InternalError("role-arn of that bucket is missing")) assumed_role = flask.current_app.boto.assume_role( role_arn, expires_in, aws_creds_config) cred = get_value(assumed_role, "Credentials", InternalError("fail to assume role")) return { "aws_access_key_id": get_value( cred, "AccessKeyId", InternalError("outdated format. AccessKeyId missing"), ), "aws_secret_access_key": get_value( cred, "SecretAccessKey", InternalError("outdated format. SecretAccessKey missing"), ), "aws_session_token": get_value( cred, "SessionToken", InternalError("outdated format. Sesssion token missing"), ), }
def get_signed_url( self, action, expires_in, public_data=False, force_signed_url=True, **kwargs ): aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") ) s3_buckets = get_value( config, "S3_BUCKETS", InternalError("buckets not configured") ) bucket_name = self.bucket_name() bucket = s3_buckets.get(bucket_name) if bucket and bucket.get("endpoint_url"): http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") ) else: http_url = "https://{}.s3.amazonaws.com/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") ) credential = S3IndexedFileLocation.get_credential_to_access_bucket( bucket_name, aws_creds, expires_in ) # if it's public and we don't need to force the signed url, just return the raw # s3 url aws_access_key_id = get_value( credential, "aws_access_key_id", InternalError("aws configuration not found"), ) # `aws_access_key_id == "*"` is a special case to support public buckets # where we do *not* want to try signing at all. the other case is that the # data is public and user requested to not sign the url if aws_access_key_id == "*" or (public_data and not force_signed_url): return http_url region = self.get_bucket_region() if not region and not bucket.get("endpoint_url"): region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, credential ) user_info = _get_user_info() url = generate_aws_presigned_url( http_url, ACTION_DICT["s3"][action], credential, "s3", region, expires_in, user_info, ) return url
def generate_presigne_url_for_part_upload(self, uploadId, partNumber, expires_in): """ Generate presigned url for uploading object part given uploadId and part number Args: uploadId(str): uploadID of the multipart upload partNumber(int): part number expires(int): expiration time Returns: presigned_url(str) """ aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") ) credential = S3IndexedFileLocation.get_credential_to_access_bucket( self.bucket_name(), aws_creds, expires_in ) region = self.get_bucket_region() if not region: region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, credential ) return multipart_upload.generate_presigned_url_for_uploading_part( self.parsed_url.netloc, self.parsed_url.path.strip("/"), credential, uploadId, partNumber, region, expires_in, )
def get_credential_to_access_bucket( cls, bucket_name, aws_creds, expires_in, boto=None ): s3_buckets = get_value( config, "S3_BUCKETS", InternalError("buckets not configured") ) if len(aws_creds) == 0 and len(s3_buckets) == 0: raise InternalError("no bucket is configured") if len(aws_creds) == 0 and len(s3_buckets) > 0: raise InternalError("credential for buckets is not configured") bucket_cred = s3_buckets.get(bucket_name) if bucket_cred is None: raise Unauthorized("permission denied for bucket") cred_key = get_value( bucket_cred, "cred", InternalError("credential of that bucket is missing") ) # this is a special case to support public buckets where we do *not* want to # try signing at all if cred_key == "*": return {"aws_access_key_id": "*"} if "role-arn" not in bucket_cred: return get_value( aws_creds, cred_key, InternalError("aws credential of that bucket is not found"), ) else: aws_creds_config = get_value( aws_creds, cred_key, InternalError("aws credential of that bucket is not found"), ) return S3IndexedFileLocation.assume_role( bucket_cred, expires_in, aws_creds_config, boto )
def assume_role(cls, bucket_cred, expires_in, aws_creds_config, boto=None): """ Args: bucket_cred expires_in aws_creds_config boto (optional): provide `boto` when calling this function outside of application context, to avoid errors when using `flask.current_app`. """ boto = boto or flask.current_app.boto role_arn = get_value( bucket_cred, "role-arn", InternalError("role-arn of that bucket is missing")) assumed_role = boto.assume_role(role_arn, expires_in, aws_creds_config) cred = get_value(assumed_role, "Credentials", InternalError("fail to assume role")) return { "aws_access_key_id": get_value( cred, "AccessKeyId", InternalError("outdated format. AccessKeyId missing"), ), "aws_secret_access_key": get_value( cred, "SecretAccessKey", InternalError("outdated format. SecretAccessKey missing"), ), "aws_session_token": get_value( cred, "SessionToken", InternalError("outdated format. Sesssion token missing"), ), }
def bucket_name(self): """ Return: Optional[str]: bucket name or None if not not in cofig """ s3_buckets = get_value( flask.current_app.config, "S3_BUCKETS", InternalError("buckets not configured"), ) for bucket in s3_buckets: if re.match("^" + bucket + "$", self.parsed_url.netloc): return bucket return None
def get_bucket_region(self): s3_buckets = get_value(config, "S3_BUCKETS", InternalError("buckets not configured")) if len(s3_buckets) == 0: return None bucket_cred = s3_buckets.get(self.bucket_name()) if bucket_cred is None: return None if "region" not in bucket_cred: return None else: return bucket_cred["region"]
def get_signed_url(self, action, expires_in, public_data=False): aws_creds = get_value( flask.current_app.config, "AWS_CREDENTIALS", InternalError("credentials not configured"), ) http_url = "https://{}.s3.amazonaws.com/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/")) config = self.get_credential_to_access_bucket(aws_creds, expires_in) aws_access_key_id = get_value( config, "aws_access_key_id", InternalError("aws configuration not found")) if aws_access_key_id == "*": return http_url region = flask.current_app.boto.get_bucket_region( self.parsed_url.netloc, config) user_info = {} if not public_data: user_info = S3IndexedFileLocation.get_user_info() url = generate_aws_presigned_url( http_url, ACTION_DICT["s3"][action], config, "s3", region, expires_in, user_info, ) return url
def init_multipart_upload(self, expires_in): """ Initialize multipart upload Args: expires(int): expiration time Returns: UploadId(str) """ aws_creds = get_value(config, "AWS_CREDENTIALS", InternalError("credentials not configured")) credentials = S3IndexedFileLocation.get_credential_to_access_bucket( self.bucket_name(), aws_creds, expires_in) return multipart_upload.initilize_multipart_upload( self.parsed_url.netloc, self.parsed_url.path.strip("/"), credentials)
def complete_multipart_upload(self, uploadId, parts, expires_in): """ Complete multipart upload. Args: uploadId(str): upload id of the current upload parts(list(set)): List of part infos [{"Etag": "1234567", "PartNumber": 1}, {"Etag": "4321234", "PartNumber": 2}] """ aws_creds = get_value(config, "AWS_CREDENTIALS", InternalError("credentials not configured")) credentials = S3IndexedFileLocation.get_credential_to_access_bucket( self.bucket_name(), aws_creds, expires_in) multipart_upload.complete_multipart_upload( self.parsed_url.netloc, self.parsed_url.path.strip("/"), credentials, uploadId, parts, )