def test_bucket_acl_PUT(self):
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = 'id'
acl = SubElement(elem, 'AccessControlList')
grant = SubElement(acl, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
xml = tostring(elem)
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT',
'wsgi.input': StringIO(xml)},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Transfer-Encoding': 'chunked'})
self.assertIsNone(req.content_length)
self.assertIsNone(req.message_length())
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
def _make_xml(grantee):
owner = 'test:tester'
permission = 'READ'
elem = Element('AccessControlPolicy')
elem_owner = SubElement(elem, 'Owner')
SubElement(elem_owner, 'ID').text = owner
SubElement(elem_owner, 'DisplayName').text = owner
acl_list_elem = SubElement(elem, 'AccessControlList')
elem_grant = SubElement(acl_list_elem, 'Grant')
elem_grant.append(grantee)
SubElement(elem_grant, 'Permission').text = permission
return tostring(elem)
def _dict_to_etree(self, parent, d):
for key, value in d.items():
tag = re.sub('\W', '', snake_to_camel(key))
elem = SubElement(parent, tag)
if isinstance(value, (dict, DictMixin)):
self._dict_to_etree(elem, value)
else:
try:
elem.text = str(value)
except ValueError:
# We set an invalid string for XML.
elem.text = '(invalid string)'
def _dict_to_etree(self, parent, d):
for key, value in d.items():
tag = re.sub('\W', '', snake_to_camel(key))
elem = SubElement(parent, tag)
if isinstance(value, (dict, MutableMapping)):
self._dict_to_etree(elem, value)
else:
if isinstance(value, (int, float, bool)):
value = str(value)
try:
elem.text = value
except ValueError:
# We set an invalid string for XML.
elem.text = '(invalid string)'
def test_bucket_fails_with_both_acl_header_and_xml_PUT(self):
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = 'id'
acl = SubElement(elem, 'AccessControlList')
grant = SubElement(acl, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
xml = tostring(elem)
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'X-AMZ-ACL': 'public-read'},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body),
'UnexpectedContent')
def get_acl(account_name, headers):
"""
Attempts to construct an S3 ACL based on what is found in the swift headers
"""
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = account_name
SubElement(owner, 'DisplayName').text = account_name
access_control_list = SubElement(elem, 'AccessControlList')
# grant FULL_CONTROL to myself by default
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'CanonicalUser')
SubElement(grantee, 'ID').text = account_name
SubElement(grantee, 'DisplayName').text = account_name
SubElement(grant, 'Permission').text = 'FULL_CONTROL'
referrers, _ = parse_acl(headers.get('x-container-read'))
if referrer_allowed('unknown', referrers):
# grant public-read access
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
referrers, _ = parse_acl(headers.get('x-container-write'))
if referrer_allowed('unknown', referrers):
# grant public-write access
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'WRITE'
body = tostring(elem)
return HTTPOk(body=body, content_type="text/plain")
def _versioning_PUT_suspended(self, path):
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'Suspended'
xml = tostring(elem)
self.swift.register('POST', '/v1/AUTH_test/bucket', HTTPNoContent,
{'x-container-sysmeta-versions-enabled': 'False'},
None)
req = Request.blank('%s?versioning' % path,
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
calls = self.swift.calls_with_headers
self.assertEqual(calls[-1][0], 'POST')
self.assertIn(('X-Versions-Enabled', 'false'),
list(calls[-1][2].items()))
def GET(self, req):
"""
Handle GET Service request
"""
resp = req.get_response(self.app, query={'format': 'json'})
containers = json.loads(resp.body)
containers = filter(
lambda item: validate_bucket_name(
item['name'], self.conf.dns_compliant_bucket_names),
containers)
# we don't keep the creation time of a bucket (s3cmd doesn't
# work without that) so we use something bogus.
elem = Element('ListAllMyBucketsResult')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
buckets = SubElement(elem, 'Buckets')
for c in containers:
if self.conf.s3_acl and self.conf.check_bucket_owner:
try:
req.get_response(self.app, 'HEAD', c['name'])
except AccessDenied:
continue
except NoSuchBucket:
continue
bucket = SubElement(buckets, 'Bucket')
SubElement(bucket, 'Name').text = c['name']
SubElement(bucket, 'CreationDate').text = \
'2009-02-03T16:45:09.000Z'
body = tostring(elem)
return HTTPOk(content_type='application/xml', body=body)
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
try:
xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.logger.error(e)
raise exc_type, exc_value, exc_traceback
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query)
except NoSuchKey:
pass
except ErrorResponse as e:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg
continue
if not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def _build_versions_result(self, req, objects, encoding_type, tag_max_keys,
is_truncated):
elem = Element('ListVersionsResult')
SubElement(elem, 'Name').text = req.container_name
prefix = swob.wsgi_to_str(req.params.get('prefix'))
if prefix and encoding_type == 'url':
prefix = quote(prefix)
SubElement(elem, 'Prefix').text = prefix
key_marker = swob.wsgi_to_str(req.params.get('key-marker'))
if key_marker and encoding_type == 'url':
key_marker = quote(key_marker)
SubElement(elem, 'KeyMarker').text = key_marker
SubElement(elem, 'VersionIdMarker').text = swob.wsgi_to_str(
req.params.get('version-id-marker'))
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['name']
SubElement(elem, 'NextVersionIdMarker').text = \
objects[-1].get('version') or 'null'
if 'subdir' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['subdir']
SubElement(elem, 'NextVersionIdMarker').text = 'null'
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
if delimiter is not None:
if encoding_type == 'url':
delimiter = quote(delimiter)
SubElement(elem, 'Delimiter').text = delimiter
if encoding_type == 'url':
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
return elem
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try:
xml = req.xml(max_body_size)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
if any(version is not None for _key, version in delete_list):
# TODO: support deleting specific versions of objects
raise S3NotImplemented()
def do_delete(base_req, key, version):
req = copy.copy(base_req)
req.environ = copy.copy(base_req.environ)
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
resp = req.get_response(self.app,
method='DELETE',
query=query,
headers={'Accept': 'application/json'})
# Have to read the response to actually do the SLO delete
if query:
try:
delete_result = json.loads(resp.body)
if delete_result['Errors']:
# NB: bulk includes 404s in "Number Not Found",
# not "Errors"
msg_parts = [delete_result['Response Status']]
msg_parts.extend(
'%s: %s' % (obj, status)
for obj, status in delete_result['Errors'])
return key, {
'code': 'SLODeleteError',
'message': '\n'.join(msg_parts)
}
# else, all good
except (ValueError, TypeError, KeyError):
# Logs get all the gory details
self.logger.exception(
'Could not parse SLO delete response: %r',
resp.body)
# Client gets something more generic
return key, {
'code': 'SLODeleteError',
'message': 'Unexpected swift response'
}
except NoSuchKey:
pass
except ErrorResponse as e:
return key, {'code': e.__class__.__name__, 'message': e._msg}
return key, None
with StreamingPile(self.conf.multi_delete_concurrency) as pile:
for key, err in pile.asyncstarmap(
do_delete,
((req, key, version) for key, version in delete_list)):
if err:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = err['code']
SubElement(error, 'Message').text = err['message']
elif not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def POST(self, req):
"""
Handles Initiate Multipart Upload.
"""
if len(req.object_name) > constraints.MAX_OBJECT_NAME_LENGTH:
# Note that we can still run into trouble where the MPU is just
# within the limit, which means the segment names will go over
raise KeyTooLongError()
# Create a unique S3 upload id from UUID to avoid duplicates.
upload_id = unique_id()
seg_container = req.container_name + MULTIUPLOAD_SUFFIX
content_type = req.headers.get('Content-Type')
if content_type:
req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
req.headers[sysmeta_header('object',
'content-type')] = content_type
else:
req.headers[sysmeta_header('object', 'has-content-type')] = 'no'
req.headers['Content-Type'] = 'application/directory'
try:
seg_req = copy.copy(req)
seg_req.environ = copy.copy(req.environ)
seg_req.container_name = seg_container
seg_req.get_container_info(self.app)
except NoSuchBucket:
try:
# multi-upload bucket doesn't exist, create one with
# same storage policy and acls as the primary bucket
info = req.get_container_info(self.app)
policy_name = POLICIES[info['storage_policy']].name
hdrs = {'X-Storage-Policy': policy_name}
if info.get('read_acl'):
hdrs['X-Container-Read'] = info['read_acl']
if info.get('write_acl'):
hdrs['X-Container-Write'] = info['write_acl']
seg_req.get_response(self.app,
'PUT',
seg_container,
'',
headers=hdrs)
except (BucketAlreadyExists, BucketAlreadyOwnedByYou):
pass
obj = '%s/%s' % (req.object_name, upload_id)
req.headers.pop('Etag', None)
req.headers.pop('Content-Md5', None)
req.get_response(self.app, 'PUT', seg_container, obj, body='')
result_elem = Element('InitiateMultipartUploadResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def test_delete_bucket_multi_upload_object_exisiting(self):
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, _, body) in \
zip(keys, results_generator):
self.assertEqual(status, 200) # sanity
elem = fromstring(body, 'InitiateMultipartUploadResult')
key = elem.find('Key').text
self.assertEqual(expected_key, key) # sanity
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None) # sanity
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
# Upload Part
key, upload_id = uploads[0]
content = b'a' * self.min_segment_size
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)
# Complete Multipart Upload
key, upload_id = uploads[0]
etags = [md5(content, usedforsecurity=False).hexdigest()]
xml = self._gen_comp_xml(etags)
status, headers, body = \
self._complete_multi_upload(bucket, key, upload_id, xml)
self.assertEqual(status, 200) # sanity
# GET multipart object
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# DELETE bucket while the object existing
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 409) # sanity
# The object must still be there.
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# Can delete it with DeleteMultipleObjects request
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
obj_elem = SubElement(elem, 'Object')
SubElement(obj_elem, 'Key').text = key
body = tostring(elem, use_s3ns=False)
status, headers, body = self.conn.make_request(
'POST',
bucket,
body=body,
query='delete',
headers={'Content-MD5': calculate_md5(body)})
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 404) # sanity
# Now we can delete
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 204) # sanity
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
objtable = dict((o['name'], {
'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']
}) for o in objinfo)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
complete_elem = fromstring(xml, 'CompleteMultipartUpload',
self.logger)
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" %
(req.object_name, upload_id, part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < self.conf.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app,
'PUT',
body='',
headers=headers)
except BadSwiftRequest as e:
msg = str(e)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
pass # We know that this existed long enough for us to HEAD
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = resp.etag
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def _build_base_listing_element(self, req):
elem = Element('ListBucketResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
return elem
def elem(self):
elem = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
elem.set('{%s}type' % XMLNS_XSI, self.type)
SubElement(elem, 'ID').text = self.id
SubElement(elem, 'DisplayName').text = self.display_name
return elem
def get_acl(account_name, headers):
"""
Attempts to construct an S3 ACL based on what is found in the swift headers
"""
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = account_name
SubElement(owner, 'DisplayName').text = account_name
access_control_list = SubElement(elem, 'AccessControlList')
# grant FULL_CONTROL to myself by default
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'CanonicalUser')
SubElement(grantee, 'ID').text = account_name
SubElement(grantee, 'DisplayName').text = account_name
SubElement(grant, 'Permission').text = 'FULL_CONTROL'
referrers, _ = parse_acl(headers.get('x-container-read'))
if referrer_allowed('unknown', referrers):
# grant public-read access
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
referrers, _ = parse_acl(headers.get('x-container-write'))
if referrer_allowed('unknown', referrers):
# grant public-write access
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'WRITE'
body = tostring(elem)
return HTTPOk(body=body, content_type="text/plain")
def GET(self, req):
"""
Handle GET Bucket (List Objects) request
"""
max_keys = req.get_validated_param('max-keys',
self.conf.max_bucket_listing)
# TODO: Separate max_bucket_listing and default_bucket_listing
tag_max_keys = max_keys
max_keys = min(max_keys, self.conf.max_bucket_listing)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
query = {
'format': 'json',
'limit': max_keys + 1,
}
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
if 'delimiter' in req.params:
query.update({'delimiter': req.params['delimiter']})
fetch_owner = False
if 'versions' in req.params:
listing_type = 'object-versions'
if 'key-marker' in req.params:
query.update({'marker': req.params['key-marker']})
elif 'version-id-marker' in req.params:
err_msg = ('A version-id marker cannot be specified without '
'a key marker.')
raise InvalidArgument('version-id-marker',
req.params['version-id-marker'], err_msg)
elif int(req.params.get('list-type', '1')) == 2:
listing_type = 'version-2'
if 'start-after' in req.params:
query.update({'marker': req.params['start-after']})
# continuation-token overrides start-after
if 'continuation-token' in req.params:
decoded = b64decode(req.params['continuation-token'])
query.update({'marker': decoded})
if 'fetch-owner' in req.params:
fetch_owner = config_true_value(req.params['fetch-owner'])
else:
listing_type = 'version-1'
if 'marker' in req.params:
query.update({'marker': req.params['marker']})
resp = req.get_response(self.app, query=query)
objects = json.loads(resp.body)
# in order to judge that truncated is valid, check whether
# max_keys + 1 th element exists in swift.
is_truncated = max_keys > 0 and len(objects) > max_keys
objects = objects[:max_keys]
if listing_type == 'object-versions':
elem = Element('ListVersionsResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
SubElement(elem, 'KeyMarker').text = req.params.get('key-marker')
SubElement(
elem,
'VersionIdMarker').text = req.params.get('version-id-marker')
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['name']
if 'subdir' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['subdir']
SubElement(elem, 'NextVersionIdMarker').text = 'null'
else:
elem = Element('ListBucketResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
if listing_type == 'version-1':
SubElement(elem, 'Marker').text = req.params.get('marker')
if is_truncated and 'delimiter' in req.params:
if 'name' in objects[-1]:
name = objects[-1]['name']
else:
name = objects[-1]['subdir']
if encoding_type == 'url':
name = quote(name)
SubElement(elem, 'NextMarker').text = name
elif listing_type == 'version-2':
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['name'].encode('utf8'))
if 'subdir' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['subdir'].encode('utf8'))
if 'continuation-token' in req.params:
SubElement(elem, 'ContinuationToken').text = \
req.params['continuation-token']
if 'start-after' in req.params:
SubElement(elem, 'StartAfter').text = \
req.params['start-after']
SubElement(elem, 'KeyCount').text = str(len(objects))
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
if 'delimiter' in req.params:
SubElement(elem, 'Delimiter').text = req.params['delimiter']
if encoding_type == 'url':
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
for o in objects:
if 'subdir' not in o:
name = o['name']
if encoding_type == 'url':
name = quote(name.encode('utf-8'))
if listing_type == 'object-versions':
contents = SubElement(elem, 'Version')
SubElement(contents, 'Key').text = name
SubElement(contents, 'VersionId').text = 'null'
SubElement(contents, 'IsLatest').text = 'true'
else:
contents = SubElement(elem, 'Contents')
SubElement(contents, 'Key').text = name
SubElement(contents, 'LastModified').text = \
o['last_modified'][:-3] + 'Z'
if 's3_etag' in o:
# New-enough MUs are already in the right format
etag = o['s3_etag']
elif 'slo_etag' in o:
# SLOs may be in something *close* to the MU format
etag = '"%s-N"' % o['slo_etag'].strip('"')
else:
# Normal objects just use the MD5
etag = '"%s"' % o['hash']
# This also catches sufficiently-old SLOs, but we have
# no way to identify those from container listings
SubElement(contents, 'ETag').text = etag
SubElement(contents, 'Size').text = str(o['bytes'])
if fetch_owner or listing_type != 'version-2':
owner = SubElement(contents, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
SubElement(contents, 'StorageClass').text = 'STANDARD'
for o in objects:
if 'subdir' in o:
common_prefixes = SubElement(elem, 'CommonPrefixes')
name = o['subdir']
if encoding_type == 'url':
name = quote(name.encode('utf-8'))
SubElement(common_prefixes, 'Prefix').text = name
body = tostring(elem)
return HTTPOk(body=body, content_type='application/xml')
def _add_subdir(self, elem, o, encoding_type):
common_prefixes = SubElement(elem, 'CommonPrefixes')
name = o['subdir']
if encoding_type == 'url':
name = quote(name.encode('utf-8'))
SubElement(common_prefixes, 'Prefix').text = name
def elem(self):
elem = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
elem.set('{%s}type' % XMLNS_XSI, self.type)
SubElement(elem, 'URI').text = self.uri
return elem
def _add_object(self, req, elem, o, encoding_type, listing_type,
fetch_owner):
name = o['name']
if encoding_type == 'url':
name = quote(name.encode('utf-8'))
if listing_type == 'object-versions':
if o['content_type'] == DELETE_MARKER_CONTENT_TYPE:
contents = SubElement(elem, 'DeleteMarker')
else:
contents = SubElement(elem, 'Version')
SubElement(contents, 'Key').text = name
SubElement(contents,
'VersionId').text = o.get('version_id') or 'null'
if 'object_versioning' in get_swift_info():
SubElement(
contents,
'IsLatest').text = ('true' if o['is_latest'] else 'false')
else:
SubElement(contents, 'IsLatest').text = 'true'
else:
contents = SubElement(elem, 'Contents')
SubElement(contents, 'Key').text = name
SubElement(contents, 'LastModified').text = \
o['last_modified'][:-3] + 'Z'
if contents.tag != 'DeleteMarker':
if 's3_etag' in o:
# New-enough MUs are already in the right format
etag = o['s3_etag']
elif 'slo_etag' in o:
# SLOs may be in something *close* to the MU format
etag = '"%s-N"' % o['slo_etag'].strip('"')
else:
# Normal objects just use the MD5
etag = o['hash']
if len(etag) < 2 or etag[::len(etag) - 1] != '""':
# Normal objects just use the MD5
etag = '"%s"' % o['hash']
# This also catches sufficiently-old SLOs, but we have
# no way to identify those from container listings
# Otherwise, somebody somewhere (proxyfs, maybe?) made this
# look like an RFC-compliant ETag; we don't need to
# quote-wrap.
SubElement(contents, 'ETag').text = etag
SubElement(contents, 'Size').text = str(o['bytes'])
if fetch_owner or listing_type != 'version-2':
owner = SubElement(contents, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
if contents.tag != 'DeleteMarker':
SubElement(contents, 'StorageClass').text = 'STANDARD'
def _gen_location_xml(self, location):
elem = Element('CreateBucketConfiguration')
SubElement(elem, 'LocationConstraint').text = location
return tostring(elem)
def test_object_multi_DELETE_with_error(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPForbidden, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
slo_delete_resp = {
'Number Not Found': 0,
'Response Status': '400 Bad Request',
'Errors': [
["/bucket+segments/obj1", "403 Forbidden"],
["/bucket+segments/obj2", "403 Forbidden"]
],
'Response Body': '',
'Number Deleted': 8
}
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk, {}, json.dumps(slo_delete_resp))
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3', 'Key4']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 2)
self.assertEqual(len(elem.findall('Error')), 2)
self.assertEqual(
[(el.find('Code').text, el.find('Message').text)
for el in elem.findall('Error')],
[('AccessDenied', 'Access Denied.'),
('SLODeleteError', '\n'.join([
'400 Bad Request',
'/bucket+segments/obj1: 403 Forbidden',
'/bucket+segments/obj2: 403 Forbidden']))]
)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key1'),
('HEAD', '/v1/AUTH_test/bucket/Key2?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key2'),
('HEAD', '/v1/AUTH_test/bucket/Key3?symlink=get'),
('HEAD', '/v1/AUTH_test/bucket/Key4?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key4?multipart-manifest=delete'),
])
def response_iter():
# NB: XML requires that the XML declaration, if present, be at the
# very start of the document. Clients *will* call us out on not
# being valid XML if we pass through whitespace before it.
# Track whether we've sent anything yet so we can yield out that
# declaration *first*
yielded_anything = False
try:
try:
# TODO: add support for versioning
put_resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={
'multipart-manifest':
'put',
'heartbeat': 'on'
},
headers=headers)
if put_resp.status_int == 202:
body = []
put_resp.fix_conditional_response()
for chunk in put_resp.response_iter:
if not chunk.strip():
if time.time() - start_time < 10:
# Include some grace period to keep
# ceph-s3tests happy
continue
if not yielded_anything:
yield (b'<?xml version="1.0" '
b'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
continue
body.append(chunk)
body = json.loads(b''.join(body))
if body['Response Status'] != '201 Created':
for seg, err in body['Errors']:
if err == too_small_message:
raise EntityTooSmall()
elif err in ('Etag Mismatch', '404 Not Found'):
raise InvalidPart(upload_id=upload_id)
raise InvalidRequest(
status=body['Response Status'],
msg='\n'.join(': '.join(err)
for err in body['Errors']))
except BadSwiftRequest as e:
msg = str(e)
if too_small_message in msg:
raise EntityTooSmall(msg)
elif ', Etag Mismatch' in msg:
raise InvalidPart(upload_id=upload_id)
elif ', 404 Not Found' in msg:
raise InvalidPart(upload_id=upload_id)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
# We know that this existed long enough for us to HEAD
pass
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at
# the request header when the port is non default value and it
# makes req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
# Why are we doing our own port parsing? Because py3 decided
# to start raising ValueErrors on access after parsing such
# an invalid port
netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
if ':' in netloc:
port = netloc.split(':', 2)[1]
host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None)
if yielded_anything:
yield b'\n'
yield tostring(result_elem,
xml_declaration=not yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
for chunk in err_resp({}, lambda *a: None):
yield chunk
def test_object_multi_DELETE_versioned_enabled(self):
self.swift.register(
'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, {
'X-Container-Sysmeta-Versions-Enabled': 'True',
}, None)
t1 = next(self.ts)
key1 = '/v1/AUTH_test/bucket/Key1' \
'?symlink=get&version-id=%s' % t1.normal
self.swift.register('HEAD', key1, swob.HTTPOk, {}, None)
self.swift.register('DELETE', key1, swob.HTTPNoContent, {}, None)
t2 = next(self.ts)
key2 = '/v1/AUTH_test/bucket/Key2' \
'?symlink=get&version-id=%s' % t2.normal
# this 404 could just mean it's a delete marker
self.swift.register('HEAD', key2, swob.HTTPNotFound, {}, None)
self.swift.register('DELETE', key2, swob.HTTPNoContent, {}, None)
key3 = '/v1/AUTH_test/bucket/Key3'
self.swift.register('HEAD', key3 + '?symlink=get',
swob.HTTPOk, {}, None)
self.swift.register('DELETE', key3, swob.HTTPNoContent, {}, None)
key4 = '/v1/AUTH_test/bucket/Key4?symlink=get&version-id=null'
self.swift.register('HEAD', key4, swob.HTTPOk, {}, None)
self.swift.register('DELETE', key4, swob.HTTPNoContent, {}, None)
elem = Element('Delete')
items = (
('Key1', t1.normal),
('Key2', t2.normal),
('Key3', None),
('Key4', 'null'),
)
for key, version in items:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
if version:
SubElement(obj, 'VersionId').text = version
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', key1),
('DELETE', key1),
('HEAD', key2),
('DELETE', key2),
('HEAD', key3 + '?symlink=get'),
('DELETE', key3),
('HEAD', key4),
('DELETE', key4),
])
elem = fromstring(body)
self.assertEqual({'Key1', 'Key2', 'Key3', 'Key4'}, set(
e.findtext('Key') for e in elem.findall('Deleted')))
def test_object_multi_upload_part_copy_version(self):
if 'object_versioning' not in tf.cluster_info:
self.skipTest('Object Versioning not enabled')
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
zip(keys, results_generator):
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(elem.find('Bucket').text, bucket)
key = elem.find('Key').text
self.assertEqual(expected_key, key)
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None)
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
key, upload_id = uploads[0]
src_bucket = 'bucket2'
src_obj = 'obj4'
src_content = b'y' * (self.min_segment_size // 2) + b'z' * \
self.min_segment_size
etags = [md5(src_content, usedforsecurity=False).hexdigest()]
# prepare null-version src obj
self.conn.make_request('PUT', src_bucket)
self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
self.assertCommonResponseHeaders(headers)
# Turn on versioning
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
src_bucket,
body=xml,
query='versioning')
self.assertEqual(status, 200)
src_obj2 = 'obj5'
src_content2 = b'stub'
etags.append(md5(src_content2, usedforsecurity=False).hexdigest())
# prepare src obj w/ real version
self.conn.make_request('PUT', src_bucket, src_obj2, body=src_content2)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj2)
self.assertCommonResponseHeaders(headers)
version_id2 = headers['x-amz-version-id']
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj, bucket,
key, upload_id, 1,
src_version_id='null')
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modifieds = [elem.find('LastModified').text]
self.assertTrue(last_modifieds[0] is not None)
self.assertEqual(resp_etag, etags[0])
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj2, bucket,
key, upload_id, 2,
src_version_id=version_id2)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modifieds.append(elem.find('LastModified').text)
self.assertTrue(last_modifieds[1] is not None)
self.assertEqual(resp_etag, etags[1])
# Check last-modified timestamp
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('GET', bucket, key, query=query)
elem = fromstring(body, 'ListPartsResult')
# FIXME: COPY result drops milli/microseconds but GET doesn't
last_modified_gets = [
p.find('LastModified').text for p in elem.iterfind('Part')
]
self.assertEqual([lm.rsplit('.', 1)[0] for lm in last_modified_gets],
[lm.rsplit('.', 1)[0] for lm in last_modifieds])
# There should be *exactly* two parts in the result
self.assertEqual(2, len(last_modified_gets))
# Abort Multipart Upload
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('DELETE', bucket, key, query=query)
# sanity checks
self.assertEqual(status, 204)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], '0')
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try:
xml = req.xml(max_body_size)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query)
except NoSuchKey:
pass
except ErrorResponse as e:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg
continue
if not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def GET(self, req):
"""
Handles List Multipart Uploads
"""
def separate_uploads(uploads, prefix, delimiter):
"""
separate_uploads will separate uploads into non_delimited_uploads
(a subset of uploads) and common_prefixes according to the
specified delimiter. non_delimited_uploads is a list of uploads
which exclude the delimiter. common_prefixes is a set of prefixes
prior to the specified delimiter. Note that the prefix in the
common_prefixes includes the delimiter itself.
i.e. if '/' delimiter specified and then the uploads is consists of
['foo', 'foo/bar'], this function will return (['foo'], ['foo/']).
:param uploads: A list of uploads dictionary
:param prefix: A string of prefix reserved on the upload path.
(i.e. the delimiter must be searched behind the
prefix)
:param delimiter: A string of delimiter to split the path in each
upload
:return (non_delimited_uploads, common_prefixes)
"""
if six.PY2:
(prefix, delimiter) = utf8encode(prefix, delimiter)
non_delimited_uploads = []
common_prefixes = set()
for upload in uploads:
key = upload['key']
end = key.find(delimiter, len(prefix))
if end >= 0:
common_prefix = key[:end + len(delimiter)]
common_prefixes.add(common_prefix)
else:
non_delimited_uploads.append(upload)
return non_delimited_uploads, sorted(common_prefixes)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
keymarker = req.params.get('key-marker', '')
uploadid = req.params.get('upload-id-marker', '')
maxuploads = req.get_validated_param('max-uploads',
DEFAULT_MAX_UPLOADS,
DEFAULT_MAX_UPLOADS)
query = {
'format': 'json',
'limit': maxuploads + 1,
}
if uploadid and keymarker:
query.update({'marker': '%s/%s' % (keymarker, uploadid)})
elif keymarker:
query.update({'marker': '%s/~' % (keymarker)})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
container = req.container_name + MULTIUPLOAD_SUFFIX
try:
resp = req.get_response(self.app, container=container, query=query)
objects = json.loads(resp.body)
except NoSuchBucket:
# Assume NoSuchBucket as no uploads
objects = []
def object_to_upload(object_info):
obj, upid = object_info['name'].rsplit('/', 1)
obj_dict = {
'key': obj,
'upload_id': upid,
'last_modified': object_info['last_modified']
}
return obj_dict
# uploads is a list consists of dict, {key, upload_id, last_modified}
# Note that pattern matcher will drop whole segments objects like as
# object_name/upload_id/1.
pattern = re.compile('/[0-9]+$')
uploads = [
object_to_upload(obj) for obj in objects
if pattern.search(obj.get('name', '')) is None
]
prefixes = []
if 'delimiter' in req.params:
prefix = req.params.get('prefix', '')
delimiter = req.params['delimiter']
uploads, prefixes = separate_uploads(uploads, prefix, delimiter)
if len(uploads) > maxuploads:
uploads = uploads[:maxuploads]
truncated = True
else:
truncated = False
nextkeymarker = ''
nextuploadmarker = ''
if len(uploads) > 1:
nextuploadmarker = uploads[-1]['upload_id']
nextkeymarker = uploads[-1]['key']
result_elem = Element('ListMultipartUploadsResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'KeyMarker').text = keymarker
SubElement(result_elem, 'UploadIdMarker').text = uploadid
SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker
SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker
if 'delimiter' in req.params:
SubElement(result_elem, 'Delimiter').text = req.params['delimiter']
if 'prefix' in req.params:
SubElement(result_elem, 'Prefix').text = req.params['prefix']
SubElement(result_elem, 'MaxUploads').text = str(maxuploads)
if encoding_type is not None:
SubElement(result_elem, 'EncodingType').text = encoding_type
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
# TODO: don't show uploads which are initiated before this bucket is
# created.
for u in uploads:
upload_elem = SubElement(result_elem, 'Upload')
name = u['key']
if encoding_type == 'url':
name = quote(name)
SubElement(upload_elem, 'Key').text = name
SubElement(upload_elem, 'UploadId').text = u['upload_id']
initiator_elem = SubElement(upload_elem, 'Initiator')
SubElement(initiator_elem, 'ID').text = req.user_id
SubElement(initiator_elem, 'DisplayName').text = req.user_id
owner_elem = SubElement(upload_elem, 'Owner')
SubElement(owner_elem, 'ID').text = req.user_id
SubElement(owner_elem, 'DisplayName').text = req.user_id
SubElement(upload_elem, 'StorageClass').text = 'STANDARD'
SubElement(upload_elem, 'Initiated').text = \
u['last_modified'][:-3] + 'Z'
for p in prefixes:
elem = SubElement(result_elem, 'CommonPrefixes')
SubElement(elem, 'Prefix').text = p
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def response_iter():
# NB: XML requires that the XML declaration, if present, be at the
# very start of the document. Clients *will* call us out on not
# being valid XML if we pass through whitespace before it.
# Track whether we've sent anything yet so we can yield out that
# declaration *first*
yielded_anything = False
try:
try:
# TODO: add support for versioning
put_resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={
'multipart-manifest':
'put',
'heartbeat': 'on'
},
headers=headers)
if put_resp.status_int == 202:
body = []
put_resp.fix_conditional_response()
for chunk in put_resp.response_iter:
if not chunk.strip():
if not yielded_anything:
yield ('<?xml version="1.0" '
'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
body.append(chunk)
body = json.loads(''.join(body))
if body['Response Status'] != '201 Created':
raise InvalidRequest(
status=body['Response Status'],
msg='\n'.join(': '.join(err)
for err in body['Errors']))
except BadSwiftRequest as e:
msg = str(e)
expected_msg = ('too small; each segment must be '
'at least 1 byte')
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if
# there is only one part. Use a COPY request to copy
# the part object from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
# We know that this existed long enough for us to HEAD
pass
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at
# the request header when the port is non default value and it
# makes req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None)
if yielded_anything:
yield '\n'
yield tostring(result_elem,
xml_declaration=not yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield '\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
for chunk in err_resp({}, lambda *a: None):
yield chunk
def GET(self, req):
"""
Handles List Parts.
"""
def filter_part_num_marker(o):
try:
num = int(os.path.basename(o['name']))
return num > part_num_marker
except ValueError:
return False
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
upload_id = req.params['uploadId']
_get_upload_info(req, self.app, upload_id)
maxparts = req.get_validated_param('max-parts',
DEFAULT_MAX_PARTS_LISTING,
self.conf.max_parts_listing)
part_num_marker = req.get_validated_param('part-number-marker', 0)
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/',
'marker': '',
}
container = req.container_name + MULTIUPLOAD_SUFFIX
# Because the parts are out of order in Swift, we list up to the
# maximum number of parts and then apply the marker and limit options.
objects = []
while True:
resp = req.get_response(self.app,
container=container,
obj='',
query=query)
new_objects = json.loads(resp.body)
if not new_objects:
break
objects.extend(new_objects)
query['marker'] = new_objects[-1]['name']
last_part = 0
# If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list.
objList = [obj for obj in objects if filter_part_num_marker(obj)]
# pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
if len(objList) > maxparts:
objList = objList[:maxparts]
truncated = True
else:
truncated = False
# TODO: We have to retrieve object list again when truncated is True
# and some objects filtered by invalid name because there could be no
# enough objects for limit defined by maxparts.
if objList:
o = objList[-1]
last_part = os.path.basename(o['name'])
result_elem = Element('ListPartsResult')
SubElement(result_elem, 'Bucket').text = req.container_name
name = req.object_name
if encoding_type == 'url':
name = quote(name)
SubElement(result_elem, 'Key').text = name
SubElement(result_elem, 'UploadId').text = upload_id
initiator_elem = SubElement(result_elem, 'Initiator')
SubElement(initiator_elem, 'ID').text = req.user_id
SubElement(initiator_elem, 'DisplayName').text = req.user_id
owner_elem = SubElement(result_elem, 'Owner')
SubElement(owner_elem, 'ID').text = req.user_id
SubElement(owner_elem, 'DisplayName').text = req.user_id
SubElement(result_elem, 'StorageClass').text = 'STANDARD'
SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker)
SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part)
SubElement(result_elem, 'MaxParts').text = str(maxparts)
if 'encoding-type' in req.params:
SubElement(result_elem, 'EncodingType').text = \
req.params['encoding-type']
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
for i in objList:
part_elem = SubElement(result_elem, 'Part')
SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1]
SubElement(part_elem, 'LastModified').text = \
i['last_modified'][:-3] + 'Z'
SubElement(part_elem, 'ETag').text = '"%s"' % i['hash']
SubElement(part_elem, 'Size').text = str(i['bytes'])
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def GET(self, req):
"""
Handle GET Bucket (List Objects) request
"""
max_keys = req.get_validated_param('max-keys',
self.conf.max_bucket_listing)
# TODO: Separate max_bucket_listing and default_bucket_listing
tag_max_keys = max_keys
max_keys = min(max_keys, self.conf.max_bucket_listing)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
query = {
'format': 'json',
'limit': max_keys + 1,
}
if 'marker' in req.params:
query.update({'marker': req.params['marker']})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
if 'delimiter' in req.params:
query.update({'delimiter': req.params['delimiter']})
# GET Bucket (List Objects) Version 2 parameters
is_v2 = int(req.params.get('list-type', '1')) == 2
fetch_owner = False
if is_v2:
if 'start-after' in req.params:
query.update({'marker': req.params['start-after']})
# continuation-token overrides start-after
if 'continuation-token' in req.params:
decoded = b64decode(req.params['continuation-token'])
query.update({'marker': decoded})
if 'fetch-owner' in req.params:
fetch_owner = config_true_value(req.params['fetch-owner'])
resp = req.get_response(self.app, query=query)
objects = json.loads(resp.body)
elem = Element('ListBucketResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
# in order to judge that truncated is valid, check whether
# max_keys + 1 th element exists in swift.
is_truncated = max_keys > 0 and len(objects) > max_keys
objects = objects[:max_keys]
if not is_v2:
SubElement(elem, 'Marker').text = req.params.get('marker')
if is_truncated and 'delimiter' in req.params:
if 'name' in objects[-1]:
SubElement(elem, 'NextMarker').text = \
objects[-1]['name']
if 'subdir' in objects[-1]:
SubElement(elem, 'NextMarker').text = \
objects[-1]['subdir']
else:
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['name'])
if 'subdir' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['subdir'])
if 'continuation-token' in req.params:
SubElement(elem, 'ContinuationToken').text = \
req.params['continuation-token']
if 'start-after' in req.params:
SubElement(elem, 'StartAfter').text = \
req.params['start-after']
SubElement(elem, 'KeyCount').text = str(len(objects))
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
if 'delimiter' in req.params:
SubElement(elem, 'Delimiter').text = req.params['delimiter']
if encoding_type is not None:
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
for o in objects:
if 'subdir' not in o:
contents = SubElement(elem, 'Contents')
SubElement(contents, 'Key').text = o['name']
SubElement(contents, 'LastModified').text = \
o['last_modified'][:-3] + 'Z'
SubElement(contents, 'ETag').text = '"%s"' % o['hash']
SubElement(contents, 'Size').text = str(o['bytes'])
if fetch_owner or not is_v2:
owner = SubElement(contents, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
SubElement(contents, 'StorageClass').text = 'STANDARD'
for o in objects:
if 'subdir' in o:
common_prefixes = SubElement(elem, 'CommonPrefixes')
SubElement(common_prefixes, 'Prefix').text = o['subdir']
body = tostring(elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')
