def GET(self, req):
"""
Handles GET Bucket versioning.
"""
req.get_response(self.app, method='HEAD')
# Just report there is no versioning configured here.
elem = Element('VersioningConfiguration')
body = tostring(elem)
return HTTPOk(body=body, content_type="text/plain")
def GET(self, req):
"""
Handles GET Bucket logging.
"""
req.get_response(self.app, method='HEAD')
# logging disabled
elem = Element('BucketLoggingStatus')
body = tostring(elem)
return HTTPOk(body=body, content_type='application/xml')
def test_grant_email_xml(self):
grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'AmazonCustomerByEmail')
SubElement(grantee, 'EmailAddress').text = '*****@*****.**'
xml = _make_xml(grantee=grantee)
req = Request.blank('/bucket/object?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac'},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'NotImplemented')
def test_bucket_PUT_with_location(self):
elem = Element('CreateBucketConfiguration')
SubElement(elem, 'LocationConstraint').text = 'US'
xml = tostring(elem)
req = Request.blank('/bucket',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac'},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEquals(status.split()[0], '200')
def test_grant_invalid_group_xml(self):
grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Invalid')
xml = _make_xml(grantee=grantee)
req = Request.blank('/bucket/object?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'MalformedACLError')
def _body_iter(self):
error_elem = Element('Error')
SubElement(error_elem, 'Code').text = self._code
SubElement(error_elem, 'Message').text = self._msg
if 'swift.trans_id' in self.environ:
request_id = self.environ['swift.trans_id']
SubElement(error_elem, 'RequestId').text = request_id
self._dict_to_etree(error_elem, self.info)
yield tostring(error_elem, use_s3ns=False)
def _test_bucket_PUT_with_location(self, root_element):
elem = Element(root_element)
SubElement(elem, 'LocationConstraint').text = 'US'
xml = tostring(elem)
req = Request.blank('/bucket',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '200')
def GET(self, req):
"""
Handles GET Bucket location.
"""
req.get_response(self.app, method='HEAD')
elem = Element('LocationConstraint')
if CONF.location != 'US':
elem.text = CONF.location
body = tostring(elem)
return HTTPOk(body=body, content_type='application/xml')
def _make_xml(grantee):
owner = 'test:tester'
permission = 'READ'
elem = Element('AccessControlPolicy')
elem_owner = SubElement(elem, 'Owner')
SubElement(elem_owner, 'ID').text = owner
SubElement(elem_owner, 'DisplayName').text = owner
acl_list_elem = SubElement(elem, 'AccessControlList')
elem_grant = SubElement(acl_list_elem, 'Grant')
elem_grant.append(grantee)
SubElement(elem_grant, 'Permission').text = permission
return tostring(elem)
def test_grant_invalid_uri_xml(self):
grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = 'invalid'
xml = _make_xml(grantee)
req = Request.blank('/bucket/object?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac'},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_versioning_put_error(self):
# Root tag is not VersioningConfiguration
elem = Element('foo')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 400)
self.assertEqual(get_error_code(body), 'MalformedXML')
# Status is not "Enabled" or "Suspended"
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = '...'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
'bucket',
body=xml,
query='versioning')
self.assertEqual(status, 400)
self.assertEqual(get_error_code(body), 'MalformedXML')
def test_object_multi_DELETE_without_md5(self):
elem = Element('Delete')
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac'},
body=body)
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidRequest')
def test_bucket_PUT_with_location_error(self):
elem = Element('CreateBucketConfiguration')
SubElement(elem, 'LocationConstraint').text = 'XXX'
xml = tostring(elem)
req = Request.blank('/bucket',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEqual(self._get_error_code(body),
'InvalidLocationConstraint')
def elem(self):
"""
Decode the value to an ACL instance.
"""
elem = Element(self.root_tag)
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = self.owner.id
SubElement(owner, 'DisplayName').text = self.owner.name
SubElement(elem,
'AccessControlList').extend(g.elem() for g in self.grants)
return elem
def convert_urlquery_to_xml(val):
"""Convert x-amz-tagging to a Tagging XML."""
root = Element('Tagging')
elem = SubElement(root, 'TagSet')
# AWS support key1=&key2=
items = parse_qs(val, keep_blank_values=True)
for key, val in items.items():
if len(val) != 1:
raise InvalidArgument(HTTP_HEADER_TAGGING_KEY,
value=val,
msg=INVALID_TAGGING)
tag = SubElement(elem, 'Tag')
SubElement(tag, 'Key').text = key
SubElement(tag, 'Value').text = val[0]
return tostring(root)
def _handle_create_multipart_upload(self, s3_key, req):
req.headers.setdefault('content-type', 'application/octet-stream')
boto3_resp = self.local_to_me_provider._create_multipart_upload(
req.headers, s3_key)
# NOTE: we forced swift3 to fully delegate to us, so we are
# responsible for returning a valid S3 API response.
result_elem = Element('InitiateMultipartUploadResult')
SubElement(result_elem, 'Bucket').text = self.container_name
SubElement(result_elem, 'Key').text = self.object_name
SubElement(result_elem, 'UploadId').text = boto3_resp['UploadId']
body = tostring(result_elem)
# Note: swift3 mw requires obj POST to return 202
return swob.HTTPAccepted(body=body, content_type='application/xml')
def test_object_multi_DELETE_to_object(self):
elem = Element('Delete')
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'object'
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket/object?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '200')
def test_object_multi_DELETE_too_many_keys(self):
elem = Element('Delete')
for i in range(CONF.max_multi_delete_objects + 1):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = str(i)
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_swift3(req)
self.assertEqual(self._get_error_code(body), 'MalformedXML')
def _handle_complete_multipart_upload(self, s3_key, req):
upload_id = req.params['uploadId']
parts = []
xml = req.environ[CC_SWIFT_REQ_KEY].xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
etag = part_elem.find('./ETag').text
parts.append({'ETag': etag, 'PartNumber': part_number})
boto3_resp = self.local_to_me_provider._complete_multipart_upload(
s3_key, upload_id, parts)
# NOTE: we forced swift3 to fully delegate to us, so we are
# responsible for returning a valid S3 API response.
# NOTE (for the note): this workaround for a client library (boto) was
# copied verbatim from the "swift3" codebase. It may or may not be
# relevant (I think it is for any client talking to cloud-connector
# using boto and a non-default port number), but this comment and the
# workaround both come from "swift3" and should be safe for us as well.
# (sic) vvvvvvvvvvvvvvvvvvvvvvvvvv
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it
# makes req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
# (sic) ^^^^^^^^^^^^^^^^^^^^^^^^^^
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
result_elem = Element('CompleteMultipartUploadResult')
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = self.container_name
SubElement(result_elem, 'Key').text = self.object_name
SubElement(result_elem, 'ETag').text = boto3_resp['ETag']
body = tostring(result_elem)
# Note: swift3 mw requires obj POST to return 202
return swob.HTTPAccepted(body=body, content_type='application/xml')
def GET(self, req):
"""
Handle GET Service request
"""
log_s3api_command(req, 'list-buckets')
resp = req.get_response(self.app, query={'format': 'json'})
containers = json.loads(resp.body)
containers = filter(
lambda item: validate_bucket_name(item['name']), containers)
# we don't keep the creation time of a bucket (s3cmd doesn't
# work without that) so we use something bogus.
elem = Element('ListAllMyBucketsResult')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
buckets = SubElement(elem, 'Buckets')
for c in containers:
if 'last_modified' in c:
ts = last_modified_date_to_timestamp(c['last_modified'])
creation_date = S3Timestamp(ts).s3xmlformat
else:
creation_date = '2009-02-03T16:45:09.000Z'
if CONF.s3_acl and CONF.check_bucket_owner:
try:
cname = c['name'].encode('utf-8')
c_resp = req.get_response(self.app, 'HEAD', cname)
if 'X-Timestamp' in c_resp.sw_headers:
creation_date = S3Timestamp(
c_resp.sw_headers['X-Timestamp']).s3xmlformat
except AccessDenied:
continue
except NoSuchBucket:
continue
bucket = SubElement(buckets, 'Bucket')
SubElement(bucket, 'Name').text = c['name']
SubElement(bucket, 'CreationDate').text = creation_date
body = tostring(elem)
return HTTPOk(content_type='application/xml', body=body)
def test_bucket_acl_PUT(self):
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = 'id'
acl = SubElement(elem, 'AccessControlList')
grant = SubElement(acl, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
xml = tostring(elem)
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac'},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEquals(status.split()[0], '200')
def get_acl(account_name, headers):
"""
Attempts to construct an S3 ACL based on what is found in the swift headers
"""
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = account_name
SubElement(owner, 'DisplayName').text = account_name
access_control_list = SubElement(elem, 'AccessControlList')
# grant FULL_CONTROL to myself by default
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'CanonicalUser')
SubElement(grantee, 'ID').text = account_name
SubElement(grantee, 'DisplayName').text = account_name
SubElement(grant, 'Permission').text = 'FULL_CONTROL'
referrers, _ = parse_acl(headers.get('x-container-read'))
if referrer_allowed('unknown', referrers):
# grant public-read access
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
referrers, _ = parse_acl(headers.get('x-container-write'))
if referrer_allowed('unknown', referrers):
# grant public-write access
grant = SubElement(access_control_list, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'WRITE'
body = tostring(elem)
return HTTPOk(body=body, content_type="text/plain")
def PUT(self, req):
"""
Handle PUT Object and PUT Object (Copy) request
"""
if CONF.s3_acl:
if 'X-Amz-Copy-Source' in req.headers:
src_path = req.headers['X-Amz-Copy-Source']
src_path = src_path if src_path.startswith('/') else \
('/' + src_path)
src_bucket, src_obj = split_path(src_path, 0, 2, True)
req.get_response(self.app,
'HEAD',
src_bucket,
src_obj,
permission='READ')
b_resp = req.get_response(self.app, 'HEAD', obj='')
# To avoid overwriting the existing object by unauthorized user,
# we send HEAD request first before writing the object to make
# sure that the target object does not exist or the user that sent
# the PUT request have write permission.
try:
req.get_response(self.app, 'HEAD')
except NoSuchKey:
pass
req_acl = ACL.from_headers(req.headers, b_resp.bucket_acl.owner,
Owner(req.user_id, req.user_id))
req.object_acl = req_acl
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
elem = Element('CopyObjectResult')
SubElement(elem, 'ETag').text = '"%s"' % resp.etag
body = tostring(elem, use_s3ns=False)
return HTTPOk(body=body, headers=resp.headers)
resp.status = HTTP_OK
return resp
def test_bucket_fails_with_both_acl_header_and_xml_PUT(self):
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = 'id'
acl = SubElement(elem, 'AccessControlList')
grant = SubElement(acl, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
xml = tostring(elem)
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'X-AMZ-ACL': 'public-read'},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEqual(self._get_error_code(body),
'UnexpectedContent')
def test_object_multi_DELETE_no_key(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key')
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_swift3(req)
self.assertEqual(self._get_error_code(body), 'UserKeyMustBeSpecified')
def POST(self, req):
"""
Handles Initiate Multipart Upload.
"""
# Create a unique S3 upload id from UUID to avoid duplicates.
upload_id = unique_id()
container = req.container_name + MULTIUPLOAD_SUFFIX
content_type = req.headers.get('Content-Type')
if content_type:
req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
req.headers[
sysmeta_header('object', 'content-type')] = content_type
else:
req.headers[sysmeta_header('object', 'has-content-type')] = 'no'
req.headers['Content-Type'] = 'application/directory'
try:
req.get_response(self.app, 'PUT', container, '')
except BucketAlreadyExists:
pass
obj = '%s/%s' % (req.object_name, upload_id)
req.headers.pop('Etag', None)
req.headers.pop('Content-Md5', None)
req.get_response(self.app, 'PUT', container, obj, body='')
result_elem = Element('InitiateMultipartUploadResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def test_bucket_acl_PUT(self):
elem = Element('AccessControlPolicy')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = 'id'
acl = SubElement(elem, 'AccessControlList')
grant = SubElement(acl, 'Grant')
grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI})
grantee.set('{%s}type' % XMLNS_XSI, 'Group')
SubElement(grantee, 'URI').text = \
'http://acs.amazonaws.com/groups/global/AllUsers'
SubElement(grant, 'Permission').text = 'READ'
xml = tostring(elem)
req = Request.blank('/bucket?acl',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
},
body=xml)
status, headers, body = self.call_swift3(req)
self.assertEquals(status.split()[0], '200')
req = Request.blank('/bucket?acl',
environ={
'REQUEST_METHOD': 'PUT',
'wsgi.input': StringIO(xml)
},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Transfer-Encoding': 'chunked'
})
self.assertIsNone(req.content_length)
self.assertIsNone(req.message_length())
status, headers, body = self.call_swift3(req)
self.assertEquals(status.split()[0], '200')
def test_object_multi_DELETE(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk,
{'x-static-large-object': 'True'}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk,
{}, None)
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5
},
body=body)
req.date = datetime.now()
req.content_type = 'text/plain'
status, headers, body = self.call_swift3(req)
self.assertEquals(status.split()[0], '200')
elem = fromstring(body)
self.assertEquals(len(elem.findall('Deleted')), 3)
_, path, _ = self.swift.calls_with_headers[-1]
path, query_string = path.split('?', 1)
self.assertEquals(path, '/v1/AUTH_test/bucket/Key3')
query = dict(urllib.parse.parse_qsl(query_string))
self.assertEquals(query['multipart-manifest'], 'delete')
def GET(self, req):
"""
Handle GET Service request
"""
resp = req.get_response(self.app, query={'format': 'json'})
containers = json.loads(resp.body)
containers = filter(lambda item: validate_bucket_name(item['name']),
containers)
# we don't keep the creation time of a backet (s3cmd doesn't
# work without that) so we use something bogus.
elem = Element('ListAllMyBucketsResult')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
buckets = SubElement(elem, 'Buckets')
for c in containers:
if CONF.s3_acl and CONF.check_bucket_owner:
try:
req.get_response(self.app, 'HEAD', c['name'])
except AccessDenied:
continue
except NoSuchBucket:
continue
bucket = SubElement(buckets, 'Bucket')
SubElement(bucket, 'Name').text = c['name']
SubElement(bucket, 'CreationDate').text = \
'2009-02-03T16:45:09.000Z'
body = tostring(elem)
return HTTPOk(content_type='application/xml', body=body)
def _test_object_multi_DELETE(self, account):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={
'Authorization': 'AWS %s:hmac' % account,
'Content-MD5': content_md5
},
body=body)
req.date = datetime.now()
req.content_type = 'text/plain'
return self.call_swift3(req)
