def from_headers(cls,
headers,
bucket_owner,
object_owner=None,
as_private=True,
inherit_grants=None):
"""
Convert HTTP headers to an ACL instance.
"""
grants = []
if inherit_grants:
grants += inherit_grants
try:
for key, value in headers.items():
if key.lower().startswith('x-amz-grant-'):
permission = key[len('x-amz-grant-'):]
permission = permission.upper().replace('-', '_')
if permission not in PERMISSIONS:
continue
for grantee in value.split(','):
grants.append(
Grant(Grantee.from_header(grantee), permission))
if 'x-amz-acl' in headers:
try:
acl = headers['x-amz-acl']
if len(grants) > 0:
err_msg = 'Specifying both Canned ACLs and Header ' \
'Grants is not allowed'
raise InvalidRequest(err_msg)
# skip empty ACL headers sent by clients
if len(acl):
grantees = canned_acl_grantees(bucket_owner,
object_owner)[acl]
for permission, grantee in grantees:
grants.append(Grant(grantee, permission))
except KeyError:
# expects canned_acl_grantees()[] raises KeyError
raise InvalidArgument('x-amz-acl', headers['x-amz-acl'])
except (KeyError, ValueError):
# TODO: think about we really catch this except sequence
raise InvalidRequest()
if len(grants) == 0:
# No ACL headers
if as_private:
return ACLPrivate(bucket_owner, object_owner)
else:
return None
return cls(object_owner or bucket_owner, grants)
def check_md5(self, body):
if 'HTTP_CONTENT_MD5' not in self.environ:
raise InvalidRequest('Missing required header for this request: '
'Content-MD5')
digest = md5.new(body).digest().encode('base64').strip()
if self.environ['HTTP_CONTENT_MD5'] != digest:
raise InvalidDigest(content_md5=self.environ['HTTP_CONTENT_MD5'])
def check_copy_source(self, app):
"""
check_copy_source checks the copy source existence and if copying an
object to itself, for illegal request parameters
:returns: the source HEAD response
"""
try:
src_path = self.headers['X-Amz-Copy-Source']
except KeyError:
return None
if '?' in src_path:
src_path, qs = src_path.split('?', 1)
query = parse_qsl(qs, True)
if not query:
pass # ignore it
elif len(query) > 1 or query[0][0] != 'versionId':
raise InvalidArgument('X-Amz-Copy-Source',
self.headers['X-Amz-Copy-Source'],
'Unsupported copy source parameter.')
elif query[0][1] != 'null':
# TODO: once we support versioning, we'll need to translate
# src_path to the proper location in the versions container
raise S3NotImplemented('Versioning is not yet supported')
self.headers['X-Amz-Copy-Source'] = src_path
src_path = unquote(src_path)
src_path = src_path if src_path.startswith('/') else ('/' + src_path)
src_bucket, src_obj = split_path(src_path, 0, 2, True)
headers = swob.HeaderKeyDict()
headers.update(self._copy_source_headers())
src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj,
headers=headers)
if src_resp.status_int == 304: # pylint: disable-msg=E1101
raise PreconditionFailed()
self.headers['X-Amz-Copy-Source'] = \
'/' + self.headers['X-Amz-Copy-Source'].lstrip('/')
source_container, source_obj = \
split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True)
if (self.container_name == source_container and
self.object_name == source_obj and
self.headers.get('x-amz-metadata-directive',
'COPY') == 'COPY'):
raise InvalidRequest("This copy request is illegal "
"because it is trying to copy an "
"object to itself without "
"changing the object's metadata, "
"storage class, website redirect "
"location or encryption "
"attributes.")
return src_resp
def from_headers(cls,
headers,
bucket_owner,
object_owner=None,
as_private=True):
"""
Convert HTTP headers to an ACL instance.
"""
grants = []
try:
for key, value in headers.items():
if key.lower().startswith('x-amz-grant-'):
permission = key[len('x-amz-grant-'):]
permission = permission.upper().replace('-', '_')
if permission not in PERMISSIONS:
continue
for grantee in value.split(','):
grants.append(
Grant(Grantee.from_header(grantee), permission))
if 'x-amz-acl' in headers:
acl = headers['x-amz-acl']
if len(grants) > 0:
err_msg = 'Specifying both Canned ACLs and Header ' \
'Grants is not allowed'
raise InvalidRequest(err_msg)
grantees = canned_acl_grantees(bucket_owner, object_owner)[acl]
for permission, grantee in grantees:
grants.append(Grant(grantee, permission))
except (KeyError, ValueError):
raise InvalidRequest()
if len(grants) == 0:
# No ACL headers
if as_private:
return ACLPrivate(bucket_owner, object_owner)
else:
return None
return cls(object_owner or bucket_owner, grants)
def _canonical_request(self):
# prepare 'canonical_request'
# Example requests are like following:
#
# GET
# /
# Action=ListUsers&Version=2010-05-08
# content-type:application/x-www-form-urlencoded; charset=utf-8
# host:iam.amazonaws.com
# x-amz-date:20150830T123600Z
#
# content-type;host;x-amz-date
# e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
#
# 1. Add verb like: GET
cr = [self.method.upper()]
# 2. Add path like: /
path = self._canonical_uri()
cr.append(path)
# 3. Add query like: Action=ListUsers&Version=2010-05-08
cr.append(self._canonical_query_string())
# 4. Add headers like:
# content-type:application/x-www-form-urlencoded; charset=utf-8
# host:iam.amazonaws.com
# x-amz-date:20150830T123600Z
headers_to_sign = self._headers_to_sign()
cr.append('\n'.join([
'%s:%s' % (key, value)
for key, value in sorted(headers_to_sign.items())
]) + '\n')
# 5. Add signed headers into canonical request like
# content-type;host;x-amz-date
cr.append(';'.join(sorted(headers_to_sign)))
# 6. Add payload string at the tail
if 'X-Amz-Credential' in self.params:
# V4 with query parameters only
hashed_payload = 'UNSIGNED-PAYLOAD'
elif 'X-Amz-Content-SHA256' not in self.headers:
msg = 'Missing required header for this request: ' \
'x-amz-content-sha256'
raise InvalidRequest(msg)
else:
hashed_payload = self.headers['X-Amz-Content-SHA256']
cr.append(hashed_payload)
return '\n'.join(cr).encode('utf-8')
def check_copy_source(self, app):
"""
check_copy_source checks the copy source existence and if copying an
object to itself, for illegal request parameters
:returns: the source HEAD response
"""
if 'X-Amz-Copy-Source' not in self.headers:
return None
src_path = unquote(self.headers['X-Amz-Copy-Source'])
src_path = src_path if src_path.startswith('/') else \
('/' + src_path)
src_bucket, src_obj = split_path(src_path, 0, 2, True)
headers = swob.HeaderKeyDict()
headers.update(self._copy_source_headers())
src_resp = self.get_response(app,
'HEAD',
src_bucket,
src_obj,
headers=headers)
if src_resp.status_int == 304: # pylint: disable-msg=E1101
raise PreconditionFailed()
self.headers['X-Amz-Copy-Source'] = \
'/' + self.headers['X-Amz-Copy-Source'].lstrip('/')
source_container, source_obj = \
split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True)
if (self.container_name == source_container
and self.object_name == source_obj and self.headers.get(
'x-amz-metadata-directive', 'COPY') == 'COPY'):
raise InvalidRequest("This copy request is illegal "
"because it is trying to copy an "
"object to itself without "
"changing the object's metadata, "
"storage class, website redirect "
"location or encryption "
"attributes.")
return src_resp
def wrapped(self, req):
if not req.is_object_request:
raise InvalidRequest('A key must be specified')
return func(self, req)
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
objtable = dict((o['name'],
{'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']}) for o in objinfo)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" % (req.object_name, upload_id,
part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
# Following swift commit 7f636a5, zero-byte segments aren't allowed,
# even as the final segment
empty_seg = None
if manifest[-1]['size_bytes'] == 0:
empty_seg = manifest.pop()
# We'll check the sizes of all except the last segment below, but
# since we just popped off a zero-byte segment, we should check
# that last segment, too.
if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app, 'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app, 'PUT', body='',
headers=headers)
except BadSwiftRequest as e:
msg = str(e)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
if empty_seg:
# clean up the zero-byte segment
_, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2)
req.get_response(self.app, 'DELETE',
container=empty_seg_cont, obj=empty_seg_name)
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
pass # We know that this existed long enough for us to HEAD
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = resp.etag
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
log_s3api_command(req, 'complete-multipart-upload')
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
elif _key == 'content-type':
headers['Content-Type'] = val
for key, val in resp.sysmeta_headers.items():
_key = key.lower()
if _key == OBJECT_TAGGING_HEADER.lower():
headers[key] = val
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
# Force the master to be sure to fetch all uploaded parts
req.environ.setdefault('oio.query', {})
req.environ['oio.query']['force_master'] = True
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
# pylint: disable-msg=no-member
objinfo.sort(key=lambda o: int(o['name'].split('/')[-1]))
objtable = dict((o['name'].encode('utf-8'), {
'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']
}) for o in objinfo)
s3_etag_hasher = md5()
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(xml).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" %
(req.object_name, upload_id, part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
s3_etag_hasher.update(binascii.a2b_hex(etag))
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
LOGGER.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
headers[sysmeta_header('object', 'etag')] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag
# Following swift commit 7f636a5, zero-byte segments aren't allowed,
# even as the final segment
empty_seg = None
if manifest[-1]['size_bytes'] == 0:
empty_seg = manifest.pop()
# We'll check the sizes of all except the last segment below, but
# since we just popped off a zero-byte segment, we should check
# that last segment, too.
if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app,
'PUT',
body='',
headers=headers)
except ErrorResponse as e:
msg = str(e._msg)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
if empty_seg:
# clean up the zero-byte segment
_, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2)
req.get_response(self.app,
'DELETE',
container=empty_seg_cont,
obj=empty_seg_name)
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
req.environ['oio.ephemeral_object'] = True
req.get_response(self.app, 'DELETE', container, obj)
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
del resp.headers['ETag']
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp