def test_object_multi_DELETE_quiet(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 0)
def test_object_multi_DELETE_lots_of_keys(self):
elem = Element('Delete')
for i in range(self.s3api.conf.max_multi_delete_objects):
status = swob.HTTPOk if i % 2 else swob.HTTPNotFound
name = 'x' * 1000 + str(i)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name,
status, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/%s' % name,
swob.HTTPNoContent, {}, None)
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = name
body = tostring(elem, use_s3ns=False)
content_md5 = (base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip())
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual('200 OK', status)
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')),
self.s3api.conf.max_multi_delete_objects)
def test_versioning_put_error(self):
# Root tag is not VersioningConfiguration
elem = Element('foo')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 400)
self.assertEqual(get_error_code(body), 'MalformedXML')
# Status is not "Enabled" or "Suspended"
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = '...'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 400)
self.assertEqual(get_error_code(body), 'MalformedXML')
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = ''
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 400)
self.assertEqual(get_error_code(body), 'MalformedXML')
def _versioning_PUT_error(self, path):
# Root tag is not VersioningConfiguration
elem = Element('foo')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
req = Request.blank('%s?versioning' % path,
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
# Status is not "Enabled" or "Suspended"
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'enabled'
xml = tostring(elem)
req = Request.blank('%s?versioning' % path,
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
def test_object_multi_DELETE_with_error(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPForbidden, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 2)
self.assertEqual(len(elem.findall('Error')), 1)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1'),
('DELETE', '/v1/AUTH_test/bucket/Key1'),
('HEAD', '/v1/AUTH_test/bucket/Key2'),
('HEAD', '/v1/AUTH_test/bucket/Key3'),
])
def test_object_multi_DELETE_quiet(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 0)
def test_object_multi_DELETE(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
slo_delete_resp = {
'Number Not Found': 0,
'Response Status': '200 OK',
'Errors': [],
'Response Body': '',
'Number Deleted': 8
}
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk, {}, json.dumps(slo_delete_resp))
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk,
{'x-static-large-object': 'True',
'x-object-sysmeta-s3api-etag': 'some-etag'},
None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key4',
swob.HTTPNoContent, {}, None)
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3', 'Key4']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 4)
self.assertEqual(len(elem.findall('Error')), 0)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key1'),
('HEAD', '/v1/AUTH_test/bucket/Key2?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key2'),
('HEAD', '/v1/AUTH_test/bucket/Key3?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key3?multipart-manifest=delete'),
('HEAD', '/v1/AUTH_test/bucket/Key4?symlink=get'),
('DELETE',
'/v1/AUTH_test/bucket/Key4?async=on&multipart-manifest=delete'),
])
def test_object_multi_DELETE_versioned_suspended(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent,
{}, None)
t1 = next(self.ts)
key1 = '/v1/AUTH_test/bucket/Key1' + \
'?symlink=get&version-id=%s' % t1.normal
self.swift.register('HEAD', key1, swob.HTTPOk, {}, None)
self.swift.register('DELETE', key1, swob.HTTPNoContent, {}, None)
t2 = next(self.ts)
key2 = '/v1/AUTH_test/bucket/Key2' + \
'?symlink=get&version-id=%s' % t2.normal
self.swift.register('HEAD', key2, swob.HTTPNotFound, {}, None)
self.swift.register('DELETE', key2, swob.HTTPNotFound, {}, None)
key3 = '/v1/AUTH_test/bucket/Key3'
self.swift.register('HEAD', key3, swob.HTTPOk, {}, None)
self.swift.register('DELETE', key3, swob.HTTPNoContent, {}, None)
elem = Element('Delete')
items = (
('Key1', t1),
('Key2', t2),
('Key3', None),
)
for key, ts in items:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
if ts:
SubElement(obj, 'VersionId').text = ts.normal
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5
},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 3)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1'
'?symlink=get&version-id=%s' % t1.normal),
('DELETE', '/v1/AUTH_test/bucket/Key1'
'?symlink=get&version-id=%s' % t1.normal),
('HEAD', '/v1/AUTH_test/bucket/Key2'
'?symlink=get&version-id=%s' % t2.normal),
('DELETE', '/v1/AUTH_test/bucket/Key2'
'?symlink=get&version-id=%s' % t2.normal),
('HEAD', '/v1/AUTH_test/bucket/Key3?symlink=get'),
('DELETE', '/v1/AUTH_test/bucket/Key3'),
])
def elem(self):
"""
Create an etree element.
"""
elem = Element('Grant')
elem.append(self.grantee.elem())
SubElement(elem, 'Permission').text = self.permission
return elem
def elem(self):
"""
Create an etree element.
"""
elem = Element('Grant')
elem.append(self.grantee.elem())
SubElement(elem, 'Permission').text = self.permission
return elem
def test_versioning_put(self):
# Versioning not configured
status, headers, body = self.conn.make_request('GET',
'bucket',
query='versioning')
self.assertEqual(status, 200)
elem = fromstring(body)
self.assertEqual(list(elem), [])
# Enable versioning
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 200)
status, headers, body = self.conn.make_request('GET',
'bucket',
query='versioning')
self.assertEqual(status, 200)
elem = fromstring(body)
self.assertEqual(elem.find('./Status').text, 'Enabled')
# Suspend versioning
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'Suspended'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 200)
status, headers, body = self.conn.make_request('GET',
'bucket',
query='versioning')
self.assertEqual(status, 200)
elem = fromstring(body)
self.assertEqual(elem.find('./Status').text, 'Suspended')
# Resume versioning
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 200)
status, headers, body = self.conn.make_request('GET',
'bucket',
query='versioning')
self.assertEqual(status, 200)
elem = fromstring(body)
self.assertEqual(elem.find('./Status').text, 'Enabled')
def test_grant_invalid_group_xml(self):
grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Invalid')
xml = _make_xml(grantee=grantee)
req = Request.blank('/bucket/object?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'MalformedACLError')
def GET(self, req):
"""
Handles GET Bucket location.
"""
req.get_response(self.app, method='HEAD')
elem = Element('LocationConstraint')
if self.conf.location != 'US':
elem.text = self.conf.location
body = tostring(elem)
return HTTPOk(body=body, content_type='application/xml')
def test_grant_email_xml(self):
grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'AmazonCustomerByEmail')
SubElement(grantee, 'EmailAddress').text = '*****@*****.**'
xml = _make_xml(grantee=grantee)
req = Request.blank('/bucket/object?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'NotImplemented')
def GET(self, req):
"""
Handles GET Bucket location.
"""
self.set_s3api_command(req, 'get-bucket-location')
req.get_response(self.app, method='HEAD')
elem = Element('LocationConstraint')
if self.conf.location != 'us-east-1':
elem.text = self.conf.location
body = tostring(elem)
return HTTPOk(body=body, content_type='application/xml')
def _test_object_multi_DELETE(self, account):
self.keys = ['Key1', 'Key2']
self.swift.register(
'DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[0],
swob.HTTPNoContent, {}, None)
self.swift.register(
'DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[1],
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
for key in self.keys:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = (
base64.b64encode(md5(body, usedforsecurity=False).digest())
.strip())
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS %s:hmac' % account,
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
req.date = datetime.now()
req.content_type = 'text/plain'
return self.call_s3api(req)
def test_object_multi_DELETE_versioned(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
SubElement(obj, 'VersionId').text = 'not-supported'
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5
},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(self._get_error_code(body), 'NotImplemented')
def _build_versions_result(self, req, objects, encoding_type, tag_max_keys,
is_truncated):
elem = Element('ListVersionsResult')
SubElement(elem, 'Name').text = req.container_name
prefix = swob.wsgi_to_str(req.params.get('prefix'))
if prefix and encoding_type == 'url':
prefix = quote(prefix)
SubElement(elem, 'Prefix').text = prefix
key_marker = swob.wsgi_to_str(req.params.get('key-marker'))
if key_marker and encoding_type == 'url':
key_marker = quote(key_marker)
SubElement(elem, 'KeyMarker').text = key_marker
SubElement(elem, 'VersionIdMarker').text = swob.wsgi_to_str(
req.params.get('version-id-marker'))
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['name']
SubElement(elem, 'NextVersionIdMarker').text = \
objects[-1].get('version') or 'null'
if 'subdir' in objects[-1]:
SubElement(elem, 'NextKeyMarker').text = \
objects[-1]['subdir']
SubElement(elem, 'NextVersionIdMarker').text = 'null'
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
delimiter = swob.wsgi_to_str(req.params.get('delimiter'))
if delimiter is not None:
if encoding_type == 'url':
delimiter = quote(delimiter)
SubElement(elem, 'Delimiter').text = delimiter
if encoding_type == 'url':
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
return elem
def _make_complete_body(req, s3_etag, yielded_anything):
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at
# the request header when the port is non default value and it
# makes req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
# Why are we doing our own port parsing? Because py3 decided
# to start raising ValueErrors on access after parsing such
# an invalid port
netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
if ':' in netloc:
port = netloc.split(':', 2)[1]
host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
body = tostring(result_elem, xml_declaration=not yielded_anything)
if yielded_anything:
return b'\n' + body
return body
def GET(self, req): # pylint: disable=invalid-name
"""
Handles GET Bucket and Object tagging.
"""
if req.is_object_request:
self.set_s3api_command(req, 'get-object-tagging')
else:
self.set_s3api_command(req, 'get-bucket-tagging')
resp = req._get_response(self.app, 'HEAD', req.container_name,
req.object_name)
headers = dict()
if req.is_object_request:
body = resp.sysmeta_headers.get(OBJECT_TAGGING_HEADER)
# It seems that S3 returns x-amz-version-id,
# even if it is not documented.
headers['x-amz-version-id'] = resp.sw_headers[VERSION_ID_HEADER]
else:
body = resp.sysmeta_headers.get(BUCKET_TAGGING_HEADER)
close_if_possible(resp.app_iter)
if not body:
if not req.is_object_request:
raise NoSuchTagSet(headers=headers)
else:
elem = Element('Tagging')
SubElement(elem, 'TagSet')
body = tostring(elem)
return HTTPOk(body=body,
content_type='application/xml',
headers=headers)
def test_bucket_acl_PUT(self):
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = 'id'
acl = SubElement(elem, 'AccessControlList')
grant = SubElement(acl, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
xml = tostring(elem)
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT',
'wsgi.input': BytesIO(xml)},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Transfer-Encoding': 'chunked'})
self.assertIsNone(req.content_length)
self.assertIsNone(req.message_length())
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
def append_copy_resp_body(self, controller_name, last_modified):
elem = Element('Copy%sResult' % controller_name)
SubElement(elem, 'LastModified').text = last_modified
SubElement(elem, 'ETag').text = '"%s"' % self.etag
self.headers['Content-Type'] = 'application/xml'
self.body = tostring(elem)
self.etag = None
def _gen_comp_xml(self, etags):
elem = Element('CompleteMultipartUpload')
for i, etag in enumerate(etags):
elem_part = SubElement(elem, 'Part')
SubElement(elem_part, 'PartNumber').text = str(i + 1)
SubElement(elem_part, 'ETag').text = etag
return tostring(elem)
def _gen_invalid_multi_delete_xml(self, hasObjectTag=False):
elem = Element('Delete')
if hasObjectTag:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = ''
return tostring(elem, use_s3ns=False)
def test_object_multi_DELETE(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
slo_delete_resp = {
'Number Not Found': 0,
'Response Status': '200 OK',
'Errors': [],
'Response Body': '',
'Number Deleted': 8
}
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk, {}, json.dumps(slo_delete_resp))
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 3)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1'),
('DELETE', '/v1/AUTH_test/bucket/Key1'),
('HEAD', '/v1/AUTH_test/bucket/Key2'),
('HEAD', '/v1/AUTH_test/bucket/Key3'),
('DELETE', '/v1/AUTH_test/bucket/Key3?multipart-manifest=delete'),
])
def _build_tagging_body(self, n_tags=1):
elem = Element('Tagging')
sub = SubElement(elem, 'TagSet')
for num in range(n_tags):
tag = SubElement(sub, 'Tag')
SubElement(tag, 'Key').text = 'key' * 41 + '%05d' % num
SubElement(tag, 'Value').text = 'value' * 50 + '%06d' % num
return tostring(elem)
def _build_base_listing_element(self, req, encoding_type):
elem = Element('ListBucketResult')
SubElement(elem, 'Name').text = req.container_name
prefix = swob.wsgi_to_str(req.params.get('prefix'))
if prefix and encoding_type == 'url':
prefix = quote(prefix)
SubElement(elem, 'Prefix').text = prefix
return elem
def _gen_multi_delete_xml(self, objects, quiet=None):
elem = Element('Delete')
if quiet:
SubElement(elem, 'Quiet').text = quiet
for key in objects:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
return tostring(elem, use_s3ns=False)
def GET(self, req):
"""
Handles GET Bucket and Object tagging.
"""
elem = Element('Tagging')
SubElement(elem, 'TagSet')
body = tostring(elem)
return HTTPOk(body=body, content_type=None)
def test_object_multi_DELETE_with_non_json(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPForbidden, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk, {}, b'asdf')
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3', 'Key4']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(
md5(body, usedforsecurity=False).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 2)
self.assertEqual(len(elem.findall('Error')), 2)
self.assertEqual(
[tuple(el.find(x).text for x in ('Key', 'Code', 'Message'))
for el in elem.findall('Error')],
[('Key3', 'AccessDenied', 'Access Denied.'),
('Key4', 'SLODeleteError', 'Unexpected swift response')])
self.assertEqual(self.s3api.logger.get_lines_for_level('error'), [
'Could not parse SLO delete response (200 OK): %s: ' % b'asdf'])
self.s3api.logger.clear()
def POST(self, req):
"""
Handles Initiate Multipart Upload.
"""
# Create a unique S3 upload id from UUID to avoid duplicates.
upload_id = unique_id()
seg_container = req.container_name + MULTIUPLOAD_SUFFIX
content_type = req.headers.get('Content-Type')
if content_type:
req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
req.headers[sysmeta_header('object',
'content-type')] = content_type
else:
req.headers[sysmeta_header('object', 'has-content-type')] = 'no'
req.headers['Content-Type'] = 'application/directory'
try:
seg_req = copy.copy(req)
seg_req.environ = copy.copy(req.environ)
seg_req.container_name = seg_container
seg_req.get_container_info(self.app)
except NoSuchBucket:
try:
# multi-upload bucket doesn't exist, create one with
# same storage policy and acls as the primary bucket
info = req.get_container_info(self.app)
policy_name = POLICIES[info['storage_policy']].name
hdrs = {'X-Storage-Policy': policy_name}
if info.get('read_acl'):
hdrs['X-Container-Read'] = info['read_acl']
if info.get('write_acl'):
hdrs['X-Container-Write'] = info['write_acl']
seg_req.get_response(self.app,
'PUT',
seg_container,
'',
headers=hdrs)
except (BucketAlreadyExists, BucketAlreadyOwnedByYou):
pass
obj = '%s/%s' % (req.object_name, upload_id)
req.headers.pop('Etag', None)
req.headers.pop('Content-Md5', None)
req.get_response(self.app, 'PUT', seg_container, obj, body='')
result_elem = Element('InitiateMultipartUploadResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def _body_iter(self):
error_elem = Element('Error')
SubElement(error_elem, 'Code').text = self._code
SubElement(error_elem, 'Message').text = self._msg
if 'swift.trans_id' in self.environ:
request_id = self.environ['swift.trans_id']
SubElement(error_elem, 'RequestId').text = request_id
self._dict_to_etree(error_elem, self.info)
yield tostring(error_elem, use_s3ns=False)
def GET(self, req):
"""
Handles GET Bucket versioning.
"""
req.get_response(self.app, method='HEAD')
# Just report there is no versioning configured here.
elem = Element('VersioningConfiguration')
body = tostring(elem)
return HTTPOk(body=body, content_type="text/plain")
def test_object_multi_DELETE_lots_of_keys(self):
elem = Element('Delete')
for i in range(self.conf.max_multi_delete_objects):
name = 'x' * 1000 + str(i)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name,
swob.HTTPNotFound, {}, None)
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = name
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual('200 OK', status)
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')),
self.conf.max_multi_delete_objects)
def test_object_multi_DELETE(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk, {}, None)
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 3)
_, path, _ = self.swift.calls_with_headers[-1]
path, query_string = path.split('?', 1)
self.assertEqual(path, '/v1/AUTH_test/bucket/Key3')
query = dict(urllib.parse.parse_qsl(query_string))
self.assertEqual(query['multipart-manifest'], 'delete')
def elem(self):
elem = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
elem.set('{%s}type' % XMLNS_XSI, self.type)
SubElement(elem, 'ID').text = self.id
SubElement(elem, 'DisplayName').text = self.display_name
return elem
def elem(self):
elem = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
elem.set('{%s}type' % XMLNS_XSI, self.type)
SubElement(elem, 'URI').text = self.uri
return elem
def test_object_multi_DELETE_with_error(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPForbidden, {}, None)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
slo_delete_resp = {
'Number Not Found': 0,
'Response Status': '400 Bad Request',
'Errors': [
["/bucket+segments/obj1", "403 Forbidden"],
["/bucket+segments/obj2", "403 Forbidden"]
],
'Response Body': '',
'Number Deleted': 8
}
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key4',
swob.HTTPOk, {}, json.dumps(slo_delete_resp))
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3', 'Key4']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 2)
self.assertEqual(len(elem.findall('Error')), 2)
self.assertEqual(
[(el.find('Code').text, el.find('Message').text)
for el in elem.findall('Error')],
[('AccessDenied', 'Access Denied.'),
('SLODeleteError', '\n'.join([
'400 Bad Request',
'/bucket+segments/obj1: 403 Forbidden',
'/bucket+segments/obj2: 403 Forbidden']))]
)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1'),
('DELETE', '/v1/AUTH_test/bucket/Key1'),
('HEAD', '/v1/AUTH_test/bucket/Key2'),
('HEAD', '/v1/AUTH_test/bucket/Key3'),
('HEAD', '/v1/AUTH_test/bucket/Key4'),
('DELETE', '/v1/AUTH_test/bucket/Key4?multipart-manifest=delete'),
])
