def PUT(self, app):
if not self.obj:
# Initiate Multipart Uploads (put +segment container)
resp = self._handle_acl(app, 'HEAD')
req_acl = ACL.from_headers(self.req.headers, resp.bucket_acl.owner,
Owner(self.user_id, self.user_id))
acl_headers = encode_acl('object', req_acl)
self.req.headers[sysmeta_header('object', 'tmpacl')] = \
acl_headers[sysmeta_header('object', 'acl')]
def PUT(self, app):
if not self.acl_checked:
resp = self._handle_acl(app, 'HEAD', obj='')
req_acl = ACL.from_headers(self.req.headers, resp.bucket_acl.owner,
Owner(self.user_id, self.user_id))
acl_headers = encode_acl('object', req_acl)
self.req.headers[sysmeta_header('object', 'tmpacl')] = \
acl_headers[sysmeta_header('object', 'acl')]
self.acl_checked = True
def PUT(self, app):
if not self.obj:
# Initiate Multipart Uploads (put +segment container)
resp = self._handle_acl(app, 'HEAD')
req_acl = ACL.from_headers(self.req.headers,
resp.bucket_acl.owner,
Owner(self.user_id, self.user_id))
acl_headers = encode_acl('object', req_acl)
self.req.headers[sysmeta_header('object', 'tmpacl')] = \
acl_headers[sysmeta_header('object', 'acl')]
def PUT(self, app):
container = self.req.container_name + MULTIUPLOAD_SUFFIX
obj = '%s/%s' % (self.obj, self.req.params['uploadId'])
self.req.environ['oio.ephemeral_object'] = True
try:
resp = self.req._get_response(app, 'HEAD', container, obj)
finally:
self.req.environ['oio.ephemeral_object'] = False
self.req.headers[sysmeta_header('object', 'acl')] = \
resp.sysmeta_headers.get(sysmeta_header('object', 'tmpacl'))
def test_fake_swift_sysmeta(self):
swift = FakeSwift()
orig_headers = HeaderKeyDict()
orig_headers.update({sysmeta_header('container', 'acl'): 'test',
'x-container-meta-foo': 'bar'})
swift.register(self.method, self.path, MagicMock(), orig_headers, None)
self._check_headers(swift, self.method, self.path, orig_headers)
new_headers = orig_headers.copy()
del new_headers[sysmeta_header('container', 'acl').title()]
swift.register(self.method, self.path, MagicMock(), new_headers, None)
self._check_headers(swift, self.method, self.path, orig_headers)
def __init__(self, *args, **kwargs):
swob.Response.__init__(self, *args, **kwargs)
sw_sysmeta_headers = swob.HeaderKeyDict()
sw_headers = swob.HeaderKeyDict()
headers = HeaderKeyDict()
self.is_slo = False
for key, val in self.headers.iteritems():
_key = key.lower()
if _key.startswith(sysmeta_prefix('object')) or \
_key.startswith(sysmeta_prefix('container')):
sw_sysmeta_headers[key] = val
else:
sw_headers[key] = val
# Handle swift headers
for key, val in sw_headers.iteritems():
_key = key.lower()
if _key.startswith('x-object-meta-'):
# Note that AWS ignores user-defined headers with '=' in the
# header name. We translated underscores to '=5F' on the way
# in, though.
headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
elif _key in ('content-length', 'content-type', 'content-range',
'content-encoding', 'content-disposition',
'content-language', 'etag', 'last-modified',
'x-robots-tag', 'cache-control', 'expires',
'x-amz-version-id'):
headers[key] = val
elif _key == 'x-object-sysmeta-version-id':
headers['x-amz-version-id'] = val
elif _key == 'x-static-large-object':
# for delete slo
self.is_slo = config_true_value(val)
# Check whether we stored the AWS-style etag on upload
override_etag = sw_sysmeta_headers.get(sysmeta_header(
'object', 'etag'))
if override_etag not in (None, ''):
# Multipart uploads in AWS have ETags like
# <MD5(part_etag1 || ... || part_etagN)>-<number of parts>
headers['etag'] = override_etag
elif self.is_slo and 'etag' in headers:
# Many AWS clients use the presence of a '-' to decide whether
# to attempt client-side download validation, so even if we
# didn't store the AWS-style header, tack on a '-N'. (Use 'N'
# because we don't actually know how many parts there are.)
headers['etag'] += '-N'
self.headers = headers
if self.etag:
# add double quotes to the etag header
self.etag = self.etag
# Used for pure swift header handling at the request layer
self.sw_headers = sw_headers
self.sysmeta_headers = sw_sysmeta_headers
def GETorHEAD(self, req):
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
object_name = req.object_name
version_id = req.params.get('versionId')
if version_id and version_id != 'null':
# get a specific version in the versioning container
req.container_name += VERSIONING_SUFFIX
req.object_name = versioned_object_name(
req.object_name, req.params.pop('versionId'))
cors_rule = None
if req.headers.get('Origin'):
cors_rule = get_cors(self.app, req, req.method,
req.headers.get('Origin'))
try:
resp = req.get_response(self.app)
except NoSuchKey:
resp = None
if version_id and version_id != 'null':
# if the specific version is not in the versioning container,
# it might be the current version
req.container_name = req.container_name[
:-len(VERSIONING_SUFFIX)]
req.object_name = object_name
info = req.get_object_info(self.app, object_name=object_name)
if info.get('sysmeta', {}).get('version-id') == version_id:
resp = req.get_response(self.app)
if resp is None:
raise
if req.method == 'HEAD':
resp.app_iter = None
if 'x-amz-meta-deleted' in resp.headers:
raise NoSuchKey(object_name)
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
if cors_rule is not None:
cors_fill_headers(req, resp, cors_rule)
return resp
def test_encode_acl_object(self):
acl = ACLPrivate(Owner(id='test:tester', name='test:tester'))
acp = encode_acl('object', acl)
header_value = json.loads(acp[sysmeta_header('object', 'acl')])
self.assertTrue('Owner' in header_value)
self.assertTrue('Grant' in header_value)
self.assertEqual('test:tester', header_value['Owner'])
self.assertEqual(len(header_value['Grant']), 1)
def test_encode_acl_object(self):
acl = ACLPrivate(Owner(id='test:tester',
name='test:tester'))
acp = encode_acl('object', acl)
header_value = json.loads(acp[sysmeta_header('object', 'acl')])
self.assertTrue('Owner' in header_value)
self.assertTrue('Grant' in header_value)
self.assertEqual('test:tester', header_value['Owner'])
self.assertEqual(len(header_value['Grant']), 1)
def test_object_multipart_upload_initiate_s3acl(self):
req = Request.blank('/bucket/object?uploads',
environ={'REQUEST_METHOD': 'POST'},
headers={
'Authorization': 'AWS test:tester:hmac',
'x-amz-acl': 'public-read',
'x-amz-meta-foo': 'bar'
})
status, headers, body = self.call_swift3(req)
fromstring(body, 'InitiateMultipartUploadResult')
self.assertEquals(status.split()[0], '200')
_, _, req_headers = self.swift.calls_with_headers[-1]
self.assertEquals(req_headers.get('X-Object-Meta-Foo'), 'bar')
tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl'))
self.assertTrue(tmpacl_header)
acl_header = encode_acl(
'object', ACLPublicRead(Owner('test:tester', 'test:tester')))
self.assertEquals(acl_header.get(sysmeta_header('object', 'acl')),
tmpacl_header)
def POST(self, req):
"""
Handles Initiate Multipart Upload.
"""
# Create a unique S3 upload id from UUID to avoid duplicates.
upload_id = unique_id()
container = req.container_name + MULTIUPLOAD_SUFFIX
content_type = req.headers.get('Content-Type')
if content_type:
req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
req.headers[
sysmeta_header('object', 'content-type')] = content_type
else:
req.headers[sysmeta_header('object', 'has-content-type')] = 'no'
req.headers['Content-Type'] = 'application/directory'
try:
req.get_response(self.app, 'PUT', container, '')
except BucketAlreadyExists:
pass
obj = '%s/%s' % (req.object_name, upload_id)
req.headers.pop('Etag', None)
req.headers.pop('Content-Md5', None)
req.get_response(self.app, 'PUT', container, obj, body='')
result_elem = Element('InitiateMultipartUploadResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def test_decode_acl_object(self):
access_control_policy = \
{'Owner': 'test:tester',
'Grant': [{'Permission': 'FULL_CONTROL',
'Grantee': 'test:tester'}]}
headers = {sysmeta_header('object', 'acl'):
json.dumps(access_control_policy)}
acl = decode_acl('object', headers)
self.assertEqual(type(acl), ACL)
self.assertEqual(acl.owner.id, 'test:tester')
self.assertEqual(len(acl.grants), 1)
self.assertEqual(str(acl.grants[0].grantee), 'test:tester')
self.assertEqual(acl.grants[0].permission, 'FULL_CONTROL')
def test_object_multipart_upload_complete_s3acl(self):
acl_headers = encode_acl(
'object', ACLPublicRead(Owner('test:tester', 'test:tester')))
headers = {}
headers[sysmeta_header('object', 'tmpacl')] = \
acl_headers.get(sysmeta_header('object', 'acl'))
headers['X-Object-Meta-Foo'] = 'bar'
self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object/X',
swob.HTTPOk, headers, None)
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac'},
body=xml)
status, headers, body = self.call_swift3(req)
fromstring(body, 'CompleteMultipartUploadResult')
self.assertEquals(status.split()[0], '200')
_, _, headers = self.swift.calls_with_headers[-2]
self.assertEquals(headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEquals(
tostring(
ACLPublicRead(Owner('test:tester', 'test:tester')).elem()),
tostring(decode_acl('object', headers).elem()))
def decode_acl(resource, headers):
"""
Decode Swift metadata to an ACL instance.
Given a resource type and HTTP headers, this method returns an ACL
instance.
"""
value = ''
key = sysmeta_header(resource, 'acl')
if key in headers:
value = headers[key]
if value == '':
# Fix me: In the case of value is empty or not dict instance,
# I want an instance of Owner as None.
# However, in the above process would occur error in reference
# to an instance variable of Owner.
return ACL(Owner(None, None), [])
try:
encode_value = json.loads(value)
if not isinstance(encode_value, dict):
return ACL(Owner(None, None), [])
id = None
name = None
grants = []
if 'Owner' in encode_value:
id = encode_value['Owner']
name = encode_value['Owner']
if 'Grant' in encode_value:
for grant in encode_value['Grant']:
grantee = None
# pylint: disable-msg=E1101
for group in Group.__subclasses__():
if group.__name__ == grant['Grantee']:
grantee = group()
if not grantee:
grantee = User(grant['Grantee'])
permission = grant['Permission']
grants.append(Grant(grantee, permission))
return ACL(Owner(id, name), grants)
except Exception as e:
LOGGER.debug(e)
pass
raise InvalidSubresource((resource, 'acl', value))
def decode_acl(resource, headers):
"""
Decode Swift metadata to an ACL instance.
Given a resource type and HTTP headers, this method returns an ACL
instance.
"""
value = ''
key = sysmeta_header(resource, 'acl')
if key in headers:
value = headers[key]
if value == '':
# Fix me: In the case of value is empty or not dict instance,
# I want an instance of Owner as None.
# However, in the above process would occur error in reference
# to an instance variable of Owner.
return ACL(Owner(None, None), [])
try:
encode_value = json.loads(value)
if not isinstance(encode_value, dict):
return ACL(Owner(None, None), [])
id = None
name = None
grants = []
if 'Owner' in encode_value:
id = encode_value['Owner']
name = encode_value['Owner']
if 'Grant' in encode_value:
for grant in encode_value['Grant']:
grantee = None
# pylint: disable-msg=E1101
for group in Group.__subclasses__():
if group.__name__ == grant['Grantee']:
grantee = group()
if not grantee:
grantee = User(grant['Grantee'])
permission = grant['Permission']
grants.append(Grant(grantee, permission))
return ACL(Owner(id, name), grants)
except Exception as e:
LOGGER.debug(e)
pass
raise InvalidSubresource((resource, 'acl', value))
def test_encode_acl_many_grant(self):
headers = {}
users = []
for i in range(0, 99):
users.append('id=test:tester%s' % str(i))
users = ','.join(users)
headers['x-amz-grant-read'] = users
acl = ACL.from_headers(headers, Owner('test:tester', 'test:tester'))
acp = encode_acl('container', acl)
header_value = acp[sysmeta_header('container', 'acl')]
header_value = json.loads(header_value)
self.assertTrue('Owner' in header_value)
self.assertTrue('Grant' in header_value)
self.assertEqual('test:tester', header_value['Owner'])
self.assertEqual(len(header_value['Grant']), 99)
def test_encode_acl_many_grant(self):
headers = {}
users = []
for i in range(0, 99):
users.append('id=test:tester%s' % str(i))
users = ','.join(users)
headers['x-amz-grant-read'] = users
acl = ACL.from_headers(headers, Owner('test:tester', 'test:tester'))
acp = encode_acl('container', acl)
header_value = acp[sysmeta_header('container', 'acl')]
header_value = json.loads(header_value)
self.assertTrue('Owner' in header_value)
self.assertTrue('Grant' in header_value)
self.assertEqual('test:tester', header_value['Owner'])
self.assertEqual(len(header_value['Grant']), 99)
def encode_acl(resource, acl):
"""
Encode an ACL instance to Swift metadata.
Given a resource type and an ACL instance, this method returns HTTP
headers, which can be used for Swift metadata.
"""
header_value = {"Owner": acl.owner.id}
grants = []
for grant in acl.grants:
grant = {"Permission": grant.permission, "Grantee": str(grant.grantee)}
grants.append(grant)
header_value.update({"Grant": grants})
headers = {}
key = sysmeta_header(resource, 'acl')
headers[key] = dumps(header_value, separators=(',', ':'))
return headers
def encode_acl(resource, acl):
"""
Encode an ACL instance to Swift metadata.
Given a resource type and an ACL instance, this method returns HTTP
headers, which can be used for Swift metadata.
"""
header_value = {"Owner": acl.owner.id}
grants = []
for grant in acl.grants:
grant = {"Permission": grant.permission,
"Grantee": str(grant.grantee)}
grants.append(grant)
header_value.update({"Grant": grants})
headers = {}
key = sysmeta_header(resource, 'acl')
headers[key] = json.dumps(header_value, separators=(',', ':'))
return headers
def deleter(self):
self.headers[sysmeta_header(resource, 'acl')] = ''
from six.moves.urllib.parse import parse_qs
from swift.common.utils import close_if_possible, public
from swift3.controllers.base import Controller, check_container_existence
from swift3.etree import fromstring, tostring, DocumentInvalid, \
Element, SubElement, XMLSyntaxError
from swift3.iam import check_iam_access
from swift3.response import HTTPNoContent, HTTPOk, MalformedXML, \
NoSuchTagSet, InvalidArgument
from swift3.utils import sysmeta_header
HTTP_HEADER_TAGGING_KEY = "x-amz-tagging"
SYSMETA_TAGGING_KEY = 'swift3-tagging'
BUCKET_TAGGING_HEADER = sysmeta_header('bucket', 'tagging')
OBJECT_TAGGING_HEADER = sysmeta_header('object', 'tagging')
# Not a swift3 header, cannot use sysmeta_header()
VERSION_ID_HEADER = 'X-Object-Sysmeta-Version-Id'
# FIXME(FVE): compute better size estimation according to key/value limits
# 10 tags with 128b key and 256b value should be 3840 + envelope
MAX_TAGGING_BODY_SIZE = 8 * 1024
INVALID_TAGGING = 'An error occurred (InvalidArgument) when calling ' \
'the PutObject operation: The header \'x-amz-tagging\' ' \
'shall be encoded as UTF-8 then URLEncoded URL query ' \
'parameters without tag name duplicates.'
def test_decode_acl_with_invalid_json(self):
headers = {sysmeta_header('container', 'acl'): '['}
self.assertRaises(InvalidSubresource, decode_acl, 'container', headers)
def test_decode_acl_empty_list(self):
headers = {sysmeta_header('container', 'acl'): '[]'}
acl = decode_acl('container', headers)
self.assertEqual(type(acl), ACL)
self.assertEqual(None, acl.owner.id)
self.assertEqual(len(acl.grants), 0)
def test_decode_acl_with_invalid_json(self):
headers = {sysmeta_header('container', 'acl'): '['}
self.assertRaises(InvalidSubresource, decode_acl, 'container', headers)
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
part_number = self.parse_part_number(req)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
method = 'upload-part'
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
method = 'upload-part-copy'
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
if 'X-Amz-Copy-Source' in req.headers:
# Clear some problematic headers that might be on the source
req.headers.update({
sysmeta_header('object', 'etag'):
'',
'X-Object-Sysmeta-Swift3-Etag':
'', # for legacy data
'X-Object-Sysmeta-Slo-Etag':
'',
'X-Object-Sysmeta-Slo-Size':
'',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'',
})
log_s3api_command(req, method)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def GETorHEAD(self, req):
"""
Handled GET or HEAD request on a part of a multipart object.
"""
part_number = self.parse_part_number(req)
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
# Get the list of parts. Must be raw to get all response headers.
slo_resp = req.get_response(self.app,
'GET',
req.container_name,
req.object_name,
query={
'multipart-manifest': 'get',
'format': 'raw'
})
# Check if the object is really a SLO. If not, and user asked
# for the first part, do a regular request.
if 'X-Static-Large-Object' not in slo_resp.sw_headers:
if part_number == 1:
if slo_resp.is_success and req.method == 'HEAD':
# Clear body
slo_resp.body = ''
return slo_resp
else:
close_if_possible(slo_resp.app_iter)
raise InvalidRange()
# Locate the part
slo = json.loads(slo_resp.body)
try:
part = slo[part_number - 1]
except IndexError:
raise InvalidRange()
# Redirect the request on the part
_, req.container_name, req.object_name = part['path'].split('/', 2)
# XXX enforce container_name and object_name to be <str>
# or it will rise issues in swift3/requests when merging both
req.container_name = req.container_name.encode('utf-8')
req.object_name = req.object_name.encode('utf8')
# The etag check was performed with the manifest
if had_match:
for match_header in ('if-match', 'if-none-match'):
req.headers.pop(match_header, None)
resp = req.get_response(self.app)
# Replace status
slo_resp.status = resp.status
# Replace body
slo_resp.app_iter = resp.app_iter
# Update with the size of the part
slo_resp.headers['Content-Length'] = \
resp.headers.get('Content-Length', 0)
slo_resp.sw_headers['Content-Length'] = \
slo_resp.headers['Content-Length']
# Add the number of parts in this object
slo_resp.headers['X-Amz-Mp-Parts-Count'] = len(slo)
return slo_resp
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
log_s3api_command(req, 'complete-multipart-upload')
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
elif _key == 'content-type':
headers['Content-Type'] = val
for key, val in resp.sysmeta_headers.items():
_key = key.lower()
if _key == OBJECT_TAGGING_HEADER.lower():
headers[key] = val
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
# Force the master to be sure to fetch all uploaded parts
req.environ.setdefault('oio.query', {})
req.environ['oio.query']['force_master'] = True
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
# pylint: disable-msg=no-member
objinfo.sort(key=lambda o: int(o['name'].split('/')[-1]))
objtable = dict((o['name'].encode('utf-8'), {
'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']
}) for o in objinfo)
s3_etag_hasher = md5()
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(xml).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" %
(req.object_name, upload_id, part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
s3_etag_hasher.update(binascii.a2b_hex(etag))
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
LOGGER.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
headers[sysmeta_header('object', 'etag')] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag
# Following swift commit 7f636a5, zero-byte segments aren't allowed,
# even as the final segment
empty_seg = None
if manifest[-1]['size_bytes'] == 0:
empty_seg = manifest.pop()
# We'll check the sizes of all except the last segment below, but
# since we just popped off a zero-byte segment, we should check
# that last segment, too.
if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app,
'PUT',
body='',
headers=headers)
except ErrorResponse as e:
msg = str(e._msg)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
if empty_seg:
# clean up the zero-byte segment
_, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2)
req.get_response(self.app,
'DELETE',
container=empty_seg_cont,
obj=empty_seg_name)
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
req.environ['oio.ephemeral_object'] = True
req.get_response(self.app, 'DELETE', container, obj)
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
del resp.headers['ETag']
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def PUT(self, app):
container = self.req.container_name + MULTIUPLOAD_SUFFIX
obj = '%s/%s' % (self.obj, self.req.params['uploadId'])
resp = self.req._get_response(app, 'HEAD', container, obj)
self.req.headers[sysmeta_header('object', 'acl')] = \
resp.sysmeta_headers.get(sysmeta_header('object', 'tmpacl'))
from swift.common.utils import public
from swift3.controllers.base import Controller, bucket_operation, \
check_container_existence
from swift3.etree import fromstring, DocumentInvalid, XMLSyntaxError
from swift3.iam import check_iam_access
from swift3.response import HTTPOk, HTTPNoContent, MalformedXML, \
NoSuchCORSConfiguration, CORSInvalidRequest
from swift3.utils import LOGGER, sysmeta_header, log_s3api_command, \
convert_response
MAX_CORS_BODY_SIZE = 10240
BUCKET_CORS_HEADER = sysmeta_header('bucket', 'cors')
CORS_ALLOWED_HTTP_METHOD = ('GET', 'POST', 'PUT', 'HEAD', 'DELETE')
def match(pattern, value):
'''helper function for wildcard'''
if '*' not in pattern:
return pattern == value
# protect dot as we keep them as is
pattern = pattern.replace('.', '\\.')
pattern = '^' + pattern.replace('*', '.*') + '$'
return re.match(pattern, value) is not None
def get_cors(app, req, method, origin):
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.common.utils import public
from swift3.controllers.base import Controller, bucket_operation
from swift3.etree import fromstring, DocumentInvalid, XMLSyntaxError
from swift3.response import HTTPOk, \
NoSuchLifecycleConfiguration, MalformedXML
from swift3.utils import sysmeta_header
LIFECYCLE_HEADER = sysmeta_header('container', 'lifecycle')
MAX_LIFECYCLE_BODY_SIZE = 64 * 1024 # Arbitrary
class LifecycleController(Controller):
"""
Handles the following APIs:
- GET Bucket lifecycle
- PUT Bucket lifecycle
- DELETE Bucket lifecycle
"""
@public
@bucket_operation(err_resp=NoSuchLifecycleConfiguration)
def GET(self, req):
def test_decode_acl_empty_list(self):
headers = {sysmeta_header('container', 'acl'): '[]'}
acl = decode_acl('container', headers)
self.assertEqual(type(acl), ACL)
self.assertIsNone(acl.owner.id)
self.assertEqual(len(acl.grants), 0)
def deleter(self):
self.headers[sysmeta_header(resource, 'acl')] = ''
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
objtable = dict((o['name'],
{'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']}) for o in objinfo)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" % (req.object_name, upload_id,
part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
# Following swift commit 7f636a5, zero-byte segments aren't allowed,
# even as the final segment
empty_seg = None
if manifest[-1]['size_bytes'] == 0:
empty_seg = manifest.pop()
# We'll check the sizes of all except the last segment below, but
# since we just popped off a zero-byte segment, we should check
# that last segment, too.
if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app, 'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app, 'PUT', body='',
headers=headers)
except BadSwiftRequest as e:
msg = str(e)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
if empty_seg:
# clean up the zero-byte segment
_, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2)
req.get_response(self.app, 'DELETE',
container=empty_seg_cont, obj=empty_seg_name)
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
pass # We know that this existed long enough for us to HEAD
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = resp.etag
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def PUT(self, app):
container = self.req.container_name + MULTIUPLOAD_SUFFIX
obj = '%s/%s' % (self.obj, self.req.params['uploadId'])
resp = self.req._get_response(app, 'HEAD', container, obj)
self.req.headers[sysmeta_header('object', 'acl')] = \
resp.sysmeta_headers.get(sysmeta_header('object', 'tmpacl'))
def deleter(self):
self.headers[sysmeta_header(resource, "acl")] = ""
