def test_bucket_GET_is_truncated_url_encoded(self):
bucket_name = 'junk'
req = Request.blank(
'/%s?encoding-type=url&max-keys=%d' % (
bucket_name, len(self.objects)),
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./IsTruncated').text, 'false')
req = Request.blank(
'/%s?encoding-type=url&max-keys=%d' % (
bucket_name, len(self.objects) - 1),
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
req = Request.blank('/subdirs?encoding-type=url&delimiter=/&'
'max-keys=2',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
self.assertEqual(elem.find('./NextMarker').text,
quote(u'but-\u062a/'.encode('utf-8')))
def test_service(self):
# GET Service(without bucket)
status, headers, body = self.conn.make_request('GET')
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue(headers['content-type'] is not None)
# TODO; requires consideration
# self.assertEqual(headers['transfer-encoding'], 'chunked')
elem = fromstring(body, 'ListAllMyBucketsResult')
buckets = elem.findall('./Buckets/Bucket')
self.assertEqual(list(buckets), [])
owner = elem.find('Owner')
self.assertEqual(self.conn.user_id, owner.find('ID').text)
self.assertEqual(self.conn.user_id, owner.find('DisplayName').text)
# GET Service(with Bucket)
req_buckets = ('bucket', 'bucket2')
for bucket in req_buckets:
self.conn.make_request('PUT', bucket)
status, headers, body = self.conn.make_request('GET')
self.assertEqual(status, 200)
elem = fromstring(body, 'ListAllMyBucketsResult')
resp_buckets = elem.findall('./Buckets/Bucket')
self.assertEqual(len(list(resp_buckets)), 2)
for b in resp_buckets:
self.assertTrue(b.find('Name').text in req_buckets)
self.assertTrue(b.find('CreationDate') is not None)
def test_bucket_versioning_GET(self):
req = Request.blank('/bucket?versioning',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
fromstring(body, 'VersioningConfiguration')
def test_bucket_GET_v2_with_delimiter_max_keys(self):
bucket_name = 'junk'
req = Request.blank(
'/%s?list-type=2&delimiter=a&max-keys=2' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
next_token = elem.find('./NextContinuationToken')
self.assertIsNotNone(next_token)
self.assertEqual(elem.find('./MaxKeys').text, '2')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
req = Request.blank(
'/%s?list-type=2&delimiter=a&max-keys=2&continuation-token=%s' %
(bucket_name, next_token.text),
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
names = [o.find('./Key').text for o in elem.iterchildren('Contents')]
self.assertEqual(names[0], 'lily')
def test_bucket_GET_max_keys(self):
bucket_name = 'junk'
req = Request.blank('/%s?max-keys=5' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./MaxKeys').text, '5')
_, path = self.swift.calls[-1]
_, query_string = path.split('?')
args = dict(cgi.parse_qsl(query_string))
self.assertEqual(args['limit'], '6')
req = Request.blank('/%s?max-keys=5000' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./MaxKeys').text, '5000')
_, path = self.swift.calls[-1]
_, query_string = path.split('?')
args = dict(cgi.parse_qsl(query_string))
self.assertEqual(args['limit'], '1001')
def test_bucket_GET_v2_fetch_owner(self):
bucket_name = 'junk'
req = Request.blank('/%s?list-type=2' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
name = elem.find('./Name').text
self.assertEqual(name, bucket_name)
objects = elem.iterchildren('Contents')
for o in objects:
self.assertIsNone(o.find('./Owner'))
req = Request.blank('/%s?list-type=2&fetch-owner=true' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
name = elem.find('./Name').text
self.assertEqual(name, bucket_name)
objects = elem.iterchildren('Contents')
for o in objects:
self.assertIsNotNone(o.find('./Owner'))
def test_delete_multi_objects_with_quiet(self):
bucket = 'bucket'
put_objects = ['obj']
query = 'delete'
# with Quiet true
quiet = 'true'
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects, quiet)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), 0)
# with Quiet false
quiet = 'false'
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects, quiet)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), 1)
def test_bucket_GET_is_truncated(self):
bucket_name = 'junk'
req = Request.blank('/%s?max-keys=5' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./IsTruncated').text, 'false')
req = Request.blank('/%s?max-keys=4' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
req = Request.blank('/subdirs?delimiter=/&max-keys=2',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
self.assertEqual(elem.find('./NextMarker').text, 'but/')
def test_xml_namespace(self):
def test_xml(ns, prefix):
return '<A %(ns)s><%(prefix)sB>C</%(prefix)sB></A>' % \
({'ns': ns, 'prefix': prefix})
# No namespace is same as having the S3 namespace.
xml = test_xml('', '')
elem = etree.fromstring(xml)
self.assertEqual(elem.find('./B').text, 'C')
# The S3 namespace is handled as no namespace.
xml = test_xml('xmlns="%s"' % etree.XMLNS_S3, '')
elem = etree.fromstring(xml)
self.assertEqual(elem.find('./B').text, 'C')
xml = test_xml('xmlns:s3="%s"' % etree.XMLNS_S3, 's3:')
elem = etree.fromstring(xml)
self.assertEqual(elem.find('./B').text, 'C')
# Any namespaces without a prefix work as no namespace.
xml = test_xml('xmlns="http://example.com/"', '')
elem = etree.fromstring(xml)
self.assertEqual(elem.find('./B').text, 'C')
xml = test_xml('xmlns:s3="http://example.com/"', 's3:')
elem = etree.fromstring(xml)
self.assertIsNone(elem.find('./B'))
def test_bucket_GET_v2_is_truncated(self):
bucket_name = 'junk'
req = Request.blank(
'/%s?list-type=2&max-keys=%d' % (bucket_name, len(self.objects)),
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./KeyCount').text, str(len(self.objects)))
self.assertEqual(elem.find('./IsTruncated').text, 'false')
req = Request.blank(
'/%s?list-type=2&max-keys=%d' % (bucket_name,
len(self.objects) - 1),
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertIsNotNone(elem.find('./NextContinuationToken'))
self.assertEqual(elem.find('./KeyCount').text,
str(len(self.objects) - 1))
self.assertEqual(elem.find('./IsTruncated').text, 'true')
req = Request.blank('/subdirs?list-type=2&delimiter=/&max-keys=2',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertIsNotNone(elem.find('./NextContinuationToken'))
self.assertEqual(elem.find('./KeyCount').text, '2')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
def test_get_bucket_v2_with_continuation_token(self):
bucket = 'bucket'
put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
'dir/subdir/object')
self._prepare_test_get_bucket(bucket, put_objects)
query = 'list-type=2&max-keys=3'
expect_objects = ('dir/subdir/object', 'object', 'object2')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('MaxKeys').text, '3')
self.assertEqual(elem.find('KeyCount').text, '3')
self.assertEqual(elem.find('IsTruncated').text, 'true')
next_cont_token_elem = elem.find('NextContinuationToken')
self.assertIsNotNone(next_cont_token_elem)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
query = 'list-type=2&max-keys=3&continuation-token=%s' % \
next_cont_token_elem.text
expect_objects = ('subdir/object', 'subdir2/object')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('MaxKeys').text, '3')
self.assertEqual(elem.find('KeyCount').text, '2')
self.assertEqual(elem.find('IsTruncated').text, 'false')
self.assertIsNone(elem.find('NextContinuationToken'))
cont_token_elem = elem.find('ContinuationToken')
self.assertEqual(cont_token_elem.text, next_cont_token_elem.text)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
def test_abort_multi_upload_error(self):
bucket = 'bucket'
self.conn.make_request('PUT', bucket)
key = 'obj'
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
self._upload_part(bucket, key, upload_id)
query = 'uploadId=%s' % upload_id
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('DELETE', bucket, key, query=query)
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('DELETE', 'nothing', key, query=query)
self.assertEqual(get_error_code(body), 'NoSuchBucket')
status, headers, body = \
self.conn.make_request('DELETE', bucket, 'nothing', query=query)
self.assertEqual(get_error_code(body), 'NoSuchUpload')
query = 'uploadId=%s' % 'nothing'
status, headers, body = \
self.conn.make_request('DELETE', bucket, key, query=query)
self.assertEqual(get_error_code(body), 'NoSuchUpload')
def test_get_bucket_with_delimiter(self):
bucket = 'bucket'
put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
'dir/subdir/object')
self._prepare_test_get_bucket(bucket, put_objects)
delimiter = '/'
query = 'delimiter=%s' % delimiter
expect_objects = ('object', 'object2')
expect_prefixes = ('dir/', 'subdir/', 'subdir2/')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Delimiter').text, delimiter)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
self.assertTrue(o.find('Owner/DisplayName').text,
self.conn.user_id)
resp_prefixes = elem.findall('CommonPrefixes')
self.assertEqual(len(resp_prefixes), len(expect_prefixes))
for i, p in enumerate(resp_prefixes):
self.assertEqual(p.find('./Prefix').text, expect_prefixes[i])
def test_upload_part_error(self):
bucket = 'bucket'
self.conn.make_request('PUT', bucket)
query = 'uploads'
key = 'obj'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
query = 'partNumber=%s&uploadId=%s' % (1, upload_id)
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('PUT', bucket, key, query=query)
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('PUT', 'nothing', key, query=query)
self.assertEqual(get_error_code(body), 'NoSuchBucket')
query = 'partNumber=%s&uploadId=%s' % (1, 'nothing')
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query)
self.assertEqual(get_error_code(body), 'NoSuchUpload')
query = 'partNumber=%s&uploadId=%s' % (0, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query)
self.assertEqual(get_error_code(body), 'InvalidArgument')
err_msg = 'Part number must be an integer between 1 and'
self.assertTrue(err_msg in get_error_msg(body))
def test_bucket_GET(self):
bucket_name = 'junk'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
name = elem.find('./Name').text
self.assertEqual(name, bucket_name)
objects = elem.iterchildren('Contents')
names = []
for o in objects:
names.append(o.find('./Key').text)
self.assertEqual('2011-01-05T02:19:14.275Z',
o.find('./LastModified').text)
self.assertEqual('"0"', o.find('./ETag').text)
self.assertEqual(len(names), len(self.objects))
for i in self.objects:
self.assertTrue(i[0] in names)
def test_fromstring_with_nonascii_text(self):
input_str = '<?xml version="1.0" encoding="UTF-8"?>\n' \
'<Test><FOO>\xef\xbc\xa1</FOO></Test>'
elem = etree.fromstring(input_str)
text = elem.find('FOO').text
self.assertEqual(text, '\xef\xbc\xa1')
self.assertTrue(isinstance(text, str))
def test_get_bucket_with_prefix(self):
bucket = 'bucket'
req_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
'dir/subdir/object')
self._prepare_test_get_bucket(bucket, req_objects)
prefix = 'object'
query = 'prefix=%s' % prefix
expect_objects = ('object', 'object2')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('Prefix').text, prefix)
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertIsNotNone(o.find('ETag').text)
self.assertIsNotNone(o.find('Size').text)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
def test_service_GET_with_blind_resource(self):
buckets = (('apple', 1, 200), ('orange', 3, 430),
('apple+segment', 1, 200))
expected = buckets[:-1]
bucket_list = create_bucket_list_json(buckets)
self.swift.register('GET', '/v1/AUTH_test', swob.HTTPOk, {},
bucket_list)
req = Request.blank('/',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListAllMyBucketsResult')
all_buckets = elem.find('./Buckets')
buckets = all_buckets.iterchildren('Bucket')
listing = list(list(buckets)[0])
self.assertEqual(len(listing), 2)
names = []
for b in all_buckets.iterchildren('Bucket'):
names.append(b.find('./Name').text)
self.assertEqual(len(names), len(expected))
for i in expected:
self.assertTrue(i[0] in names)
def test_get_bucket_v2_with_fetch_owner(self):
bucket = 'bucket'
put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object',
'dir/subdir/object')
self._prepare_test_get_bucket(bucket, put_objects)
query = 'list-type=2&fetch-owner=true'
expect_objects = ('dir/subdir/object', 'object', 'object2',
'subdir/object', 'subdir2/object')
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('KeyCount').text, '5')
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), len(expect_objects))
for i, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text, expect_objects[i])
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertTrue(o.find('Owner/ID').text, self.conn.user_id)
self.assertTrue(o.find('Owner/DisplayName').text,
self.conn.user_id)
def PUT(self, req):
"""
Handle PUT Bucket request
"""
xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE)
if xml:
# check location
try:
elem = fromstring(
xml, 'CreateBucketConfiguration', self.logger)
location = elem.find('./LocationConstraint').text
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
self.logger.error(e)
raise
if location != self.conf.location:
# s3api cannot support multiple regions currently.
raise InvalidLocationConstraint()
resp = req.get_response(self.app)
resp.status = HTTP_OK
resp.location = '/' + req.container_name
return resp
def get_acl(self, headers, body, bucket_owner, object_owner=None):
"""
Get ACL instance from S3 (e.g. x-amz-grant) headers or S3 acl xml body.
"""
acl = ACL.from_headers(headers, bucket_owner, object_owner,
as_private=False)
if acl is None:
# Get acl from request body if possible.
if not body:
raise MissingSecurityHeader(missing_header_name='x-amz-acl')
try:
elem = fromstring(body, ACL.root_tag)
acl = ACL.from_elem(
elem, True, self.req.allow_no_owner)
except(XMLSyntaxError, DocumentInvalid):
raise MalformedACLError()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.logger.error(e)
raise exc_type, exc_value, exc_traceback
else:
if body:
# Specifying grant with both header and xml is not allowed.
raise UnexpectedContent()
return acl
def test_object_multi_DELETE_quiet(self):
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
for key in ['Key1', 'Key2']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 0)
def test_acl(self):
self.conn.make_request('PUT', self.bucket, self.obj)
query = 'acl'
# PUT Bucket ACL
headers = {'x-amz-acl': 'public-read'}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, headers=headers,
query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertEqual(headers['content-length'], '0')
# GET Bucket ACL
status, headers, body = \
self.conn.make_request('GET', self.bucket, query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
# TODO: Fix the response that last-modified must be in the response.
# self.assertTrue(headers['last-modified'] is not None)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue(headers['content-type'] is not None)
elem = fromstring(body, 'AccessControlPolicy')
owner = elem.find('Owner')
self.assertEqual(owner.find('ID').text, self.conn.user_id)
self.assertEqual(owner.find('DisplayName').text, self.conn.user_id)
acl = elem.find('AccessControlList')
self.assertTrue(acl.find('Grant') is not None)
# GET Object ACL
status, headers, body = \
self.conn.make_request('GET', self.bucket, self.obj, query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
# TODO: Fix the response that last-modified must be in the response.
# self.assertTrue(headers['last-modified'] is not None)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue(headers['content-type'] is not None)
elem = fromstring(body, 'AccessControlPolicy')
owner = elem.find('Owner')
self.assertEqual(owner.find('ID').text, self.conn.user_id)
self.assertEqual(owner.find('DisplayName').text, self.conn.user_id)
acl = elem.find('AccessControlList')
self.assertTrue(acl.find('Grant') is not None)
def test_object_multi_DELETE_without_permission(self):
status, headers, body = self._test_object_multi_DELETE('test:other')
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
errors = elem.findall('Error')
self.assertEqual(len(errors), len(self.keys))
for e in errors:
self.assertTrue(e.find('Key').text in self.keys)
self.assertEqual(e.find('Code').text, 'AccessDenied')
self.assertEqual(e.find('Message').text, 'Access Denied.')
def test_fromstring_with_nonascii_text(self):
input_str = b'<?xml version="1.0" encoding="UTF-8"?>\n' \
b'<Test><FOO>\xef\xbc\xa1</FOO></Test>'
elem = etree.fromstring(input_str)
text = elem.find('FOO').text
if six.PY2:
self.assertEqual(text, b'\xef\xbc\xa1')
else:
self.assertEqual(text, b'\xef\xbc\xa1'.decode('utf8'))
self.assertIsInstance(text, str)
def test_object_location(self):
req = Request.blank('/bucket?location',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'LocationConstraint')
location = elem.text
self.assertIsNone(location)
def test_get_bucket_v2_with_continuation_token_and_delimiter(self):
bucket = 'bucket'
put_objects = ('object', u'object2-\u062a', 'subdir/object',
u'subdir2-\u062a/object', 'dir/subdir/object',
'x', 'y', 'z')
self._prepare_test_get_bucket(bucket, put_objects)
expected = [{'objects': ['object', u'object2-\u062a'],
'subdirs': ['dir/']},
{'objects': ['x'],
'subdirs': ['subdir/', u'subdir2-\u062a/']},
{'objects': ['y', 'z'],
'subdirs': []}]
continuation_token = ''
query = 'list-type=2&max-keys=3&delimiter=/&continuation-token=%s'
for i in range(len(expected)):
status, headers, body = self.conn.make_request(
'GET', bucket, query=query % continuation_token)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('MaxKeys').text, '3')
self.assertEqual(
elem.find('KeyCount').text,
str(len(expected[i]['objects']) + len(expected[i]['subdirs'])))
expect_truncated = 'true' if i < len(expected) - 1 else 'false'
self.assertEqual(elem.find('IsTruncated').text, expect_truncated)
next_cont_token_elem = elem.find('NextContinuationToken')
if expect_truncated == 'true':
self.assertIsNotNone(next_cont_token_elem)
continuation_token = next_cont_token_elem.text
resp_objects = elem.findall('./Contents')
self.assertEqual(
len(list(resp_objects)), len(expected[i]['objects']))
for j, o in enumerate(resp_objects):
self.assertEqual(o.find('Key').text,
expected[i]['objects'][j].encode('utf-8'))
self.assertTrue(o.find('LastModified').text is not None)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertTrue(o.find('ETag').text is not None)
self.assertTrue(o.find('Size').text is not None)
self.assertEqual(o.find('StorageClass').text, 'STANDARD')
self.assertIsNone(o.find('Owner/ID'))
self.assertIsNone(o.find('Owner/DisplayName'))
resp_subdirs = elem.findall('./CommonPrefixes')
self.assertEqual(
len(list(resp_subdirs)), len(expected[i]['subdirs']))
for j, o in enumerate(resp_subdirs):
self.assertEqual(
o.find('Prefix').text,
expected[i]['subdirs'][j].encode('utf-8'))
def test_object_PUT_copy_self(self):
status, headers, body = \
self._test_object_PUT_copy_self(swob.HTTPOk)
self.assertEqual(status.split()[0], '400')
elem = fromstring(body, 'Error')
err_msg = ("This copy request is illegal because it is trying to copy "
"an object to itself without changing the object's "
"metadata, storage class, website redirect location or "
"encryption attributes.")
self.assertEqual(elem.find('Code').text, 'InvalidRequest')
self.assertEqual(elem.find('Message').text, err_msg)
def test_bucket_GET_subdir_with_delimiter_max_keys(self):
bucket_name = 'junk-subdir'
req = Request.blank('/%s?delimiter=a&max-keys=1' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./NextMarker').text, 'rose')
self.assertEqual(elem.find('./MaxKeys').text, '1')
self.assertEqual(elem.find('./IsTruncated').text, 'true')
def test_delete_bucket_multi_upload_object_exisiting(self):
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, _, body) in \
izip(keys, results_generator):
self.assertEqual(status, 200) # sanity
elem = fromstring(body, 'InitiateMultipartUploadResult')
key = elem.find('Key').text
self.assertEqual(expected_key, key) # sanity
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None) # sanity
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
# Upload Part
key, upload_id = uploads[0]
content = 'a' * self.min_segment_size
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)
# Complete Multipart Upload
key, upload_id = uploads[0]
etags = [md5(content).hexdigest()]
xml = self._gen_comp_xml(etags)
status, headers, body = \
self._complete_multi_upload(bucket, key, upload_id, xml)
self.assertEqual(status, 200) # sanity
# GET multipart object
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# DELETE bucket while the object existing
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 409) # sanity
# The object must still be there.
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
def test_bucket_GET_passthroughs(self):
bucket_name = 'junk'
req = Request.blank('/%s?delimiter=a&marker=b&prefix=c' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./Prefix').text, 'c')
self.assertEqual(elem.find('./Marker').text, 'b')
self.assertEqual(elem.find('./Delimiter').text, 'a')
_, path = self.swift.calls[-1]
_, query_string = path.split('?')
args = dict(cgi.parse_qsl(query_string))
self.assertEqual(args['delimiter'], 'a')
self.assertEqual(args['marker'], 'b')
self.assertEqual(args['prefix'], 'c')
def test_bucket_GET_subdir(self):
bucket_name = 'junk-subdir'
req = Request.blank('/%s' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
name = elem.find('./Name').text
self.assertEqual(name, bucket_name)
prefixes = elem.findall('CommonPrefixes')
self.assertEqual(len(prefixes), len(self.prefixes))
for p in prefixes:
self.assertTrue(p.find('./Prefix').text in self.prefixes)
def test_bucket_GET_with_versions_versioning_not_configured(self):
req = Request.blank('/junk?versions',
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListVersionsResult')
self.assertEqual(elem.find('./Name').text, 'junk')
self.assertIsNone(elem.find('./Prefix').text)
self.assertIsNone(elem.find('./KeyMarker').text)
self.assertIsNone(elem.find('./VersionIdMarker').text)
self.assertEqual(elem.find('./MaxKeys').text, '1000')
self.assertEqual(elem.find('./IsTruncated').text, 'false')
self.assertEqual(elem.findall('./DeleteMarker'), [])
versions = elem.findall('./Version')
objects = list(self.objects)
self.assertEqual([v.find('./Key').text for v in versions],
[v[0].encode('utf-8') for v in objects])
self.assertEqual([v.find('./IsLatest').text for v in versions],
['true' for v in objects])
self.assertEqual([v.find('./VersionId').text for v in versions],
['null' for v in objects])
# Last modified in self.objects is 2011-01-05T02:19:14.275290 but
# the returned value is 2011-01-05T02:19:14.275Z
self.assertEqual([v.find('./LastModified').text for v in versions],
[v[1][:-3] + 'Z' for v in objects])
self.assertEqual(
[v.find('./ETag').text for v in versions],
['"0-N"' if v[0] == 'slo' else '"0"' for v in objects])
self.assertEqual([v.find('./Size').text for v in versions],
[str(v[3]) for v in objects])
self.assertEqual([v.find('./Owner/ID').text for v in versions],
['test:tester' for v in objects])
self.assertEqual(
[v.find('./Owner/DisplayName').text for v in versions],
['test:tester' for v in objects])
self.assertEqual([v.find('./StorageClass').text for v in versions],
['STANDARD' for v in objects])
def PUT(self, req): # pylint: disable=invalid-name
"""
Handles PUT Bucket CORS.
"""
xml = req.xml(MAX_CORS_BODY_SIZE)
try:
data = fromstring(xml, "CorsConfiguration")
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as exc:
self.logger.error(exc)
raise
# forbid wildcard for ExposeHeader
check_cors_rule(data)
req.headers[BUCKET_CORS_HEADER] = xml
resp = req._get_response(self.app, 'POST',
req.container_name, None)
return self.convert_response(req, resp, 204, HTTPOk)
def test_service_GET_without_owner_bucket(self):
bucket_list = []
for var in range(0, 10):
user_id = 'test:other'
bucket = 'bucket%s' % var
owner = Owner(user_id, user_id)
headers = encode_acl('container', ACL(owner, []))
self.swift.register('HEAD', '/v1/AUTH_test/%s' % bucket,
swob.HTTPNoContent, headers, None)
bucket_list.append((bucket, var, 300 + var))
status, headers, body = \
self._test_service_GET_for_check_bucket_owner(bucket_list)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListAllMyBucketsResult')
resp_buckets = elem.find('./Buckets')
buckets = resp_buckets.iterchildren('Bucket')
self.assertEqual(len(list(buckets)), 0)
def _test_method_error(self, method, path, response_class, headers={},
env={}, expected_xml_tags=None):
if not path.startswith('/'):
path = '/' + path # add a missing slash before the path
uri = '/v1/AUTH_test'
if path != '/':
uri += path
self.swift.register(method, uri, response_class, headers, None)
headers.update({'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
env.update({'REQUEST_METHOD': method})
req = swob.Request.blank(path, environ=env, headers=headers)
status, headers, body = self.call_s3api(req)
if expected_xml_tags is not None:
elem = fromstring(body, 'Error')
self.assertEqual(set(expected_xml_tags),
{x.tag for x in elem})
return self._get_error_code(body)
def test_service_GET_bucket_list(self):
bucket_list = []
for var in range(0, 10):
if var % 3 == 0:
user_id = 'test:tester'
else:
user_id = 'test:other'
bucket = 'bucket%s' % var
owner = Owner(user_id, user_id)
headers = encode_acl('container', ACL(owner, []))
# set register to get owner of buckets
if var % 3 == 2:
self.swift.register('HEAD', '/v1/AUTH_test/%s' % bucket,
swob.HTTPNotFound, {}, None)
else:
self.swift.register('HEAD', '/v1/AUTH_test/%s' % bucket,
swob.HTTPNoContent, headers, None)
bucket_list.append((bucket, var, 300 + var))
status, headers, body = \
self._test_service_GET_for_check_bucket_owner(bucket_list)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListAllMyBucketsResult')
resp_buckets = elem.find('./Buckets')
buckets = resp_buckets.iterchildren('Bucket')
listing = list(list(buckets)[0])
self.assertEqual(len(listing), 2)
names = []
for b in resp_buckets.iterchildren('Bucket'):
names.append(b.find('./Name').text)
# Check whether getting bucket only locate in multiples of 3 in
# bucket_list which mean requested user is owner.
expected_buckets = [b for i, b in enumerate(bucket_list)
if i % 3 == 0]
self.assertEqual(len(names), len(expected_buckets))
for i in expected_buckets:
self.assertTrue(i[0] in names)
self.assertEqual(len(self.swift.calls_with_headers), 11)
def _upload_part_copy(self,
src_bucket,
src_obj,
dst_bucket,
dst_key,
upload_id,
part_num=1,
src_range=None):
src_path = '%s/%s' % (src_bucket, src_obj)
query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id)
req_headers = {'X-Amz-Copy-Source': src_path}
if src_range:
req_headers['X-Amz-Copy-Source-Range'] = src_range
status, headers, body = \
self.conn.make_request('PUT', dst_bucket, dst_key,
headers=req_headers,
query=query)
elem = fromstring(body, 'CopyPartResult')
etag = elem.find('ETag').text.strip('"')
return status, headers, body, etag
def test_bucket_GET_v2_with_nonascii_queries(self):
bucket_name = 'junk'
req = Request.blank(
'/%s?list-type=2&delimiter=\xef\xbc\xa1&start-after=\xef\xbc\xa2&'
'prefix=\xef\xbc\xa3' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={
'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()
})
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'ListBucketResult')
self.assertEqual(elem.find('./Prefix').text, '\xef\xbc\xa3')
self.assertEqual(elem.find('./StartAfter').text, '\xef\xbc\xa2')
self.assertEqual(elem.find('./Delimiter').text, '\xef\xbc\xa1')
_, path = self.swift.calls[-1]
_, query_string = path.split('?')
args = dict(cgi.parse_qsl(query_string))
self.assertEqual(args['delimiter'], '\xef\xbc\xa1')
self.assertEqual(args['marker'], '\xef\xbc\xa2')
self.assertEqual(args['prefix'], '\xef\xbc\xa3')
def do_test(src_path=None):
date_header = self.get_date_header()
timestamp = mktime(date_header)
last_modified = S3Timestamp(timestamp).s3xmlformat
status, headers, body = self._test_object_PUT_copy(
swob.HTTPOk,
put_header={'Date': date_header},
timestamp=timestamp,
src_path=src_path)
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Content-Type'], 'application/xml')
self.assertTrue(headers.get('etag') is None)
self.assertTrue(headers.get('x-amz-meta-something') is None)
elem = fromstring(body, 'CopyObjectResult')
self.assertEqual(elem.find('LastModified').text, last_modified)
self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag)
_, _, headers = self.swift.calls_with_headers[-1]
self.assertEqual(headers['X-Copy-From'], '/some/source')
self.assertEqual(headers['Content-Length'], '0')
def test_object_multi_DELETE(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk,
{'x-static-large-object': 'True'},
None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2',
swob.HTTPNotFound, {}, None)
self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3',
swob.HTTPOk, {}, None)
elem = Element('Delete')
for key in ['Key1', 'Key2', 'Key3']:
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Content-Type': 'multipart/form-data',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')), 3)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket/Key1'),
('DELETE', '/v1/AUTH_test/bucket/Key1'),
('HEAD', '/v1/AUTH_test/bucket/Key2'),
('HEAD', '/v1/AUTH_test/bucket/Key3'),
('DELETE', '/v1/AUTH_test/bucket/Key3?multipart-manifest=delete'),
])
def test_service_GET_subresource(self):
req = Request.blank('/?acl',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListAllMyBucketsResult')
all_buckets = elem.find('./Buckets')
buckets = all_buckets.iterchildren('Bucket')
listing = list(list(buckets)[0])
self.assertEqual(len(listing), 2)
names = []
for b in all_buckets.iterchildren('Bucket'):
names.append(b.find('./Name').text)
self.assertEqual(len(names), len(self.buckets))
for i in self.buckets:
self.assertTrue(i[0] in names)
def test_bucket_GET_other_account(self):
self.assertIsNone(self.db.get_owner('bucket'))
status, _, _ = self._bucket_put()
self.assertEqual(status.split()[0], '200')
self.assertEqual(self.db.get_owner('bucket'), 'AUTH_test')
# Register request with account 'test'.
expected_body = json.dumps([{
"name": "expected",
"last_modified": "2017-04-21T16:30:34.133034",
"hash": "0000",
"bytes": 0
}]).encode('utf-8')
self.swift.register('GET',
'/v1/AUTH_test/bucket?format=json&limit=1001',
swob.HTTPOk, {}, expected_body)
# Then do a call with 'test2' account, that should be changed
# to 'test' by the middleware (because the bucket 'bucket' belongs
# to account 'test').
status, _, body = self._bucket_get('bucket', account='test2')
elem = fromstring(body, "ListBucketResult")
self.assertEqual(status.split()[0], '200')
self.assertEqual(elem.find('Contents').find('Key').text, "expected")
def test_complete_upload_with_fewer_etags(self):
bucket = 'bucket'
key = 'obj'
self.conn.make_request('PUT', bucket)
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
etags = []
for i in xrange(1, 4):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key,
body='A' * 1024 * 1024 * 5, query=query)
etags.append(headers['etag'])
query = 'uploadId=%s' % upload_id
xml = self._gen_comp_xml(etags[:-1])
status, headers, body = \
self.conn.make_request('POST', bucket, key, body=xml,
query=query)
self.assertEqual(status, 200)
def test_object_multi_DELETE_lots_of_keys(self):
elem = Element('Delete')
for i in range(self.conf.max_multi_delete_objects):
name = 'x' * 1000 + str(i)
self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name,
swob.HTTPNotFound, {}, None)
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = name
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(),
'Content-MD5': content_md5},
body=body)
status, headers, body = self.call_s3api(req)
self.assertEqual('200 OK', status)
elem = fromstring(body)
self.assertEqual(len(elem.findall('Deleted')),
self.conf.max_multi_delete_objects)
def test_bucket_GET_url_encoded(self):
bucket_name = 'junk'
req = Request.blank('/%s?encoding-type=url' % bucket_name,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'ListBucketResult')
name = elem.find('./Name').text
self.assertEqual(name, bucket_name)
objects = elem.iterchildren('Contents')
items = []
for o in objects:
items.append((o.find('./Key').text, o.find('./ETag').text))
self.assertEqual('2011-01-05T02:19:14.275Z',
o.find('./LastModified').text)
self.assertEqual(items, [
(quote(i[0].encode('utf-8')), '"0-N"' if i[0] == 'slo' else '"0"')
for i in self.objects])
def test_object_multi_upload_part_copy_range(self):
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
izip(keys, results_generator):
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(elem.find('Bucket').text, bucket)
key = elem.find('Key').text
self.assertEqual(expected_key, key)
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None)
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
# Upload Part Copy Range
key, upload_id = uploads[0]
src_bucket = 'bucket2'
src_obj = 'obj4'
src_content = 'y' * (self.min_segment_size / 2) + 'z' * \
self.min_segment_size
src_range = 'bytes=0-%d' % (self.min_segment_size - 1)
etag = md5(src_content[:self.min_segment_size]).hexdigest()
# prepare src obj
self.conn.make_request('PUT', src_bucket)
self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
self.assertCommonResponseHeaders(headers)
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj, bucket,
key, upload_id, 1, src_range)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modified = elem.find('LastModified').text
self.assertTrue(last_modified is not None)
self.assertEqual(resp_etag, etag)
# Check last-modified timestamp
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('GET', bucket, key, query=query)
elem = fromstring(body, 'ListPartsResult')
# FIXME: COPY result drops milli/microseconds but GET doesn't
last_modified_gets = [p.find('LastModified').text
for p in elem.iterfind('Part')]
self.assertEqual(
last_modified_gets[0].rsplit('.', 1)[0],
last_modified.rsplit('.', 1)[0],
'%r != %r' % (last_modified_gets[0], last_modified))
# There should be *exactly* one parts in the result
self.assertEqual(1, len(last_modified_gets))
# Abort Multipart Upload
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('DELETE', bucket, key, query=query)
# sanity checks
self.assertEqual(status, 204)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], '0')
def test_complete_upload_min_segment_size(self):
bucket = 'bucket'
key = 'obj'
self.conn.make_request('PUT', bucket)
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
# multi parts with no body
etags = []
for i in xrange(1, 3):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query)
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('POST', bucket, key, body=xml,
query=query)
self.assertEqual(get_error_code(body), 'EntityTooSmall')
# multi parts with all parts less than min segment size
etags = []
for i in xrange(1, 3):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='AA')
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('POST', bucket, key, body=xml,
query=query)
self.assertEqual(get_error_code(body), 'EntityTooSmall')
# one part and less than min segment size
etags = []
query = 'partNumber=1&uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='AA')
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('POST', bucket, key, body=xml,
query=query)
self.assertEqual(status, 200)
# multi parts with all parts except the first part less than min
# segment size
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
etags = []
body_size = [self.min_segment_size, self.min_segment_size - 1, 2]
for i in xrange(1, 3):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='A' * body_size[i])
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('POST', bucket, key, body=xml,
query=query)
self.assertEqual(get_error_code(body), 'EntityTooSmall')
# multi parts with all parts except last part more than min segment
# size
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
etags = []
body_size = [self.min_segment_size, self.min_segment_size, 2]
for i in xrange(1, 3):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query,
body='A' * body_size[i])
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('POST', bucket, key, body=xml,
query=query)
self.assertEqual(status, 200)
def test_complete_multi_upload_error(self):
bucket = 'bucket'
keys = ['obj', 'obj2']
self.conn.make_request('PUT', bucket)
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, keys[0], query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
etags = []
for i in xrange(1, 3):
query = 'partNumber=%s&uploadId=%s' % (i, upload_id)
status, headers, body = \
self.conn.make_request('PUT', bucket, keys[0], query=query)
etags.append(headers['etag'])
xml = self._gen_comp_xml(etags)
# part 1 too small
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('POST', bucket, keys[0], body=xml,
query=query)
self.assertEqual(get_error_code(body), 'EntityTooSmall')
# invalid credentials
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('POST', bucket, keys[0], body=xml,
query=query)
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
# wrong/missing bucket
status, headers, body = \
self.conn.make_request('POST', 'nothing', keys[0], query=query)
self.assertEqual(get_error_code(body), 'NoSuchBucket')
# wrong upload ID
query = 'uploadId=%s' % 'nothing'
status, headers, body = \
self.conn.make_request('POST', bucket, keys[0], body=xml,
query=query)
self.assertEqual(get_error_code(body), 'NoSuchUpload')
# without Part tag in xml
query = 'uploadId=%s' % upload_id
xml = self._gen_comp_xml([])
status, headers, body = \
self.conn.make_request('POST', bucket, keys[0], body=xml,
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# with invalid etag in xml
invalid_etag = 'invalid'
xml = self._gen_comp_xml([invalid_etag])
status, headers, body = \
self.conn.make_request('POST', bucket, keys[0], body=xml,
query=query)
self.assertEqual(get_error_code(body), 'InvalidPart')
# without part in Swift
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, keys[1], query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
query = 'uploadId=%s' % upload_id
xml = self._gen_comp_xml([etags[0]])
status, headers, body = \
self.conn.make_request('POST', bucket, keys[1], body=xml,
query=query)
self.assertEqual(get_error_code(body), 'InvalidPart')
def test_object_multi_upload(self):
bucket = 'bucket'
keys = ['obj1', 'obj2', 'obj3']
headers = [None,
{'Content-MD5': base64.b64encode('a' * 16).strip()},
{'Etag': 'nonsense'}]
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys, headers=headers)
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
izip(keys, results_generator):
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(elem.find('Bucket').text, bucket)
key = elem.find('Key').text
self.assertEqual(expected_key, key)
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None)
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
# List Multipart Uploads
query = 'uploads'
status, headers, body = \
self.conn.make_request('GET', bucket, query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'ListMultipartUploadsResult')
self.assertEqual(elem.find('Bucket').text, bucket)
self.assertIsNone(elem.find('KeyMarker').text)
self.assertEqual(elem.find('NextKeyMarker').text, uploads[-1][0])
self.assertIsNone(elem.find('UploadIdMarker').text)
self.assertEqual(elem.find('NextUploadIdMarker').text, uploads[-1][1])
self.assertEqual(elem.find('MaxUploads').text, '1000')
self.assertTrue(elem.find('EncodingType') is None)
self.assertEqual(elem.find('IsTruncated').text, 'false')
self.assertEqual(len(elem.findall('Upload')), 3)
for (expected_key, expected_upload_id), u in \
izip(uploads, elem.findall('Upload')):
key = u.find('Key').text
upload_id = u.find('UploadId').text
self.assertEqual(expected_key, key)
self.assertEqual(expected_upload_id, upload_id)
self.assertEqual(u.find('Initiator/ID').text,
self.conn.user_id)
self.assertEqual(u.find('Initiator/DisplayName').text,
self.conn.user_id)
self.assertEqual(u.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(u.find('Owner/DisplayName').text,
self.conn.user_id)
self.assertEqual(u.find('StorageClass').text, 'STANDARD')
self.assertTrue(u.find('Initiated').text is not None)
# Upload Part
key, upload_id = uploads[0]
content = 'a' * self.min_segment_size
etag = md5(content).hexdigest()
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers, etag)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], '0')
expected_parts_list = [(headers['etag'], mktime(headers['date']))]
# Upload Part Copy
key, upload_id = uploads[1]
src_bucket = 'bucket2'
src_obj = 'obj3'
src_content = 'b' * self.min_segment_size
etag = md5(src_content).hexdigest()
# prepare src obj
self.conn.make_request('PUT', src_bucket)
self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
self.assertCommonResponseHeaders(headers)
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj, bucket,
key, upload_id)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modified = elem.find('LastModified').text
self.assertTrue(last_modified is not None)
self.assertEqual(resp_etag, etag)
# Check last-modified timestamp
key, upload_id = uploads[1]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('GET', bucket, key, query=query)
self.assertEqual(200, status)
elem = fromstring(body, 'ListPartsResult')
# FIXME: COPY result drops milli/microseconds but GET doesn't
last_modified_gets = [p.find('LastModified').text
for p in elem.iterfind('Part')]
self.assertEqual(
last_modified_gets[0].rsplit('.', 1)[0],
last_modified.rsplit('.', 1)[0],
'%r != %r' % (last_modified_gets[0], last_modified))
# There should be *exactly* two parts in the result
self.assertEqual(1, len(last_modified_gets))
# List Parts
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('GET', bucket, key, query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'ListPartsResult')
self.assertEqual(elem.find('Bucket').text, bucket)
self.assertEqual(elem.find('Key').text, key)
self.assertEqual(elem.find('UploadId').text, upload_id)
self.assertEqual(elem.find('Initiator/ID').text, self.conn.user_id)
self.assertEqual(elem.find('Initiator/DisplayName').text,
self.conn.user_id)
self.assertEqual(elem.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(elem.find('Owner/DisplayName').text,
self.conn.user_id)
self.assertEqual(elem.find('StorageClass').text, 'STANDARD')
self.assertEqual(elem.find('PartNumberMarker').text, '0')
self.assertEqual(elem.find('NextPartNumberMarker').text, '1')
self.assertEqual(elem.find('MaxParts').text, '1000')
self.assertEqual(elem.find('IsTruncated').text, 'false')
self.assertEqual(len(elem.findall('Part')), 1)
# etags will be used to generate xml for Complete Multipart Upload
etags = []
for (expected_etag, expected_date), p in \
izip(expected_parts_list, elem.findall('Part')):
last_modified = p.find('LastModified').text
self.assertTrue(last_modified is not None)
# TODO: sanity check
# (kota_) How do we check the sanity?
# the last-modified header drops milli-seconds info
# by the constraint of the format.
# For now, we can do either the format check or round check
# last_modified_from_xml = mktime(last_modified)
# self.assertEqual(expected_date,
# last_modified_from_xml)
self.assertEqual(expected_etag, p.find('ETag').text)
self.assertEqual(self.min_segment_size, int(p.find('Size').text))
etags.append(p.find('ETag').text)
# Abort Multipart Uploads
# note that uploads[1] has part data while uploads[2] does not
for key, upload_id in uploads[1:]:
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('DELETE', bucket, key, query=query)
self.assertEqual(status, 204)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'],
'text/html; charset=UTF-8')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], '0')
# Complete Multipart Upload
key, upload_id = uploads[0]
xml = self._gen_comp_xml(etags)
status, headers, body = \
self._complete_multi_upload(bucket, key, upload_id, xml)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'CompleteMultipartUploadResult')
# TODO: use tf.config value
self.assertEqual(
'http://%s:%s/bucket/obj1' % (self.conn.host, self.conn.port),
elem.find('Location').text)
self.assertEqual(elem.find('Bucket').text, bucket)
self.assertEqual(elem.find('Key').text, key)
concatted_etags = ''.join(etag.strip('"') for etag in etags)
exp_etag = '"%s-%s"' % (
md5(concatted_etags.decode('hex')).hexdigest(), len(etags))
etag = elem.find('ETag').text
self.assertEqual(etag, exp_etag)
exp_size = self.min_segment_size * len(etags)
swift_etag = '"%s"' % md5(concatted_etags).hexdigest()
# TODO: GET via swift api, check against swift_etag
# Check object
def check_obj(req_headers, exp_status):
status, headers, body = \
self.conn.make_request('HEAD', bucket, key, req_headers)
self.assertEqual(status, exp_status)
self.assertCommonResponseHeaders(headers)
self.assertIn('content-length', headers)
if exp_status == 412:
self.assertNotIn('etag', headers)
self.assertEqual(headers['content-length'], '0')
else:
self.assertIn('etag', headers)
self.assertEqual(headers['etag'], exp_etag)
if exp_status == 304:
self.assertEqual(headers['content-length'], '0')
else:
self.assertEqual(headers['content-length'], str(exp_size))
check_obj({}, 200)
# Sanity check conditionals
check_obj({'If-Match': 'some other thing'}, 412)
check_obj({'If-None-Match': 'some other thing'}, 200)
# More interesting conditional cases
check_obj({'If-Match': exp_etag}, 200)
check_obj({'If-Match': swift_etag}, 412)
check_obj({'If-None-Match': swift_etag}, 200)
check_obj({'If-None-Match': exp_etag}, 304)
# Check listings
status, headers, body = self.conn.make_request('GET', bucket)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
resp_objects = elem.findall('./Contents')
self.assertEqual(len(list(resp_objects)), 1)
for o in resp_objects:
self.assertEqual(o.find('Key').text, key)
self.assertIsNotNone(o.find('LastModified').text)
self.assertRegexpMatches(
o.find('LastModified').text,
r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
self.assertEqual(o.find('ETag').text, exp_etag)
self.assertEqual(o.find('Size').text, str(exp_size))
self.assertIsNotNone(o.find('StorageClass').text is not None)
self.assertEqual(o.find('Owner/ID').text, self.conn.user_id)
self.assertEqual(o.find('Owner/DisplayName').text,
self.conn.user_id)
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try:
xml = req.xml(max_body_size)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
if any(version is not None for _key, version in delete_list):
# TODO: support deleting specific versions of objects
raise S3NotImplemented()
def do_delete(base_req, key, version):
req = copy.copy(base_req)
req.environ = copy.copy(base_req.environ)
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
resp = req.get_response(self.app,
method='DELETE',
query=query,
headers={'Accept': 'application/json'})
# Have to read the response to actually do the SLO delete
if query:
try:
delete_result = json.loads(resp.body)
if delete_result['Errors']:
# NB: bulk includes 404s in "Number Not Found",
# not "Errors"
msg_parts = [delete_result['Response Status']]
msg_parts.extend(
'%s: %s' % (obj, status)
for obj, status in delete_result['Errors'])
return key, {
'code': 'SLODeleteError',
'message': '\n'.join(msg_parts)
}
# else, all good
except (ValueError, TypeError, KeyError):
# Logs get all the gory details
self.logger.exception(
'Could not parse SLO delete response: %r',
resp.body)
# Client gets something more generic
return key, {
'code': 'SLODeleteError',
'message': 'Unexpected swift response'
}
except NoSuchKey:
pass
except ErrorResponse as e:
return key, {'code': e.__class__.__name__, 'message': e._msg}
return key, None
with StreamingPile(self.conf.multi_delete_concurrency) as pile:
for key, err in pile.asyncstarmap(
do_delete,
((req, key, version) for key, version in delete_list)):
if err:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = err['code']
SubElement(error, 'Message').text = err['message']
elif not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def _get_error_message(self, body):
elem = fromstring(body, 'Error')
return elem.find('./Message').text
def test_object_multi_upload_part_copy_version(self):
if 'object_versioning' not in tf.cluster_info:
self.skipTest('Object Versioning not enabled')
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, headers, body) in \
zip(keys, results_generator):
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(elem.find('Bucket').text, bucket)
key = elem.find('Key').text
self.assertEqual(expected_key, key)
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None)
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
key, upload_id = uploads[0]
src_bucket = 'bucket2'
src_obj = 'obj4'
src_content = b'y' * (self.min_segment_size // 2) + b'z' * \
self.min_segment_size
etags = [md5(src_content, usedforsecurity=False).hexdigest()]
# prepare null-version src obj
self.conn.make_request('PUT', src_bucket)
self.conn.make_request('PUT', src_bucket, src_obj, body=src_content)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj)
self.assertCommonResponseHeaders(headers)
# Turn on versioning
elem = Element('VersioningConfiguration')
SubElement(elem, 'Status').text = 'Enabled'
xml = tostring(elem)
status, headers, body = self.conn.make_request('PUT',
src_bucket,
body=xml,
query='versioning')
self.assertEqual(status, 200)
src_obj2 = 'obj5'
src_content2 = b'stub'
etags.append(md5(src_content2, usedforsecurity=False).hexdigest())
# prepare src obj w/ real version
self.conn.make_request('PUT', src_bucket, src_obj2, body=src_content2)
_, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj2)
self.assertCommonResponseHeaders(headers)
version_id2 = headers['x-amz-version-id']
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj, bucket,
key, upload_id, 1,
src_version_id='null')
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modifieds = [elem.find('LastModified').text]
self.assertTrue(last_modifieds[0] is not None)
self.assertEqual(resp_etag, etags[0])
status, headers, body, resp_etag = \
self._upload_part_copy(src_bucket, src_obj2, bucket,
key, upload_id, 2,
src_version_id=version_id2)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'application/xml')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], str(len(body)))
self.assertTrue('etag' not in headers)
elem = fromstring(body, 'CopyPartResult')
last_modifieds.append(elem.find('LastModified').text)
self.assertTrue(last_modifieds[1] is not None)
self.assertEqual(resp_etag, etags[1])
# Check last-modified timestamp
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('GET', bucket, key, query=query)
elem = fromstring(body, 'ListPartsResult')
# FIXME: COPY result drops milli/microseconds but GET doesn't
last_modified_gets = [
p.find('LastModified').text for p in elem.iterfind('Part')
]
self.assertEqual([lm.rsplit('.', 1)[0] for lm in last_modified_gets],
[lm.rsplit('.', 1)[0] for lm in last_modifieds])
# There should be *exactly* two parts in the result
self.assertEqual(2, len(last_modified_gets))
# Abort Multipart Upload
key, upload_id = uploads[0]
query = 'uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('DELETE', bucket, key, query=query)
# sanity checks
self.assertEqual(status, 204)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
self.assertTrue('content-length' in headers)
self.assertEqual(headers['content-length'], '0')
def test_delete_bucket_multi_upload_object_exisiting(self):
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, _, body) in \
zip(keys, results_generator):
self.assertEqual(status, 200) # sanity
elem = fromstring(body, 'InitiateMultipartUploadResult')
key = elem.find('Key').text
self.assertEqual(expected_key, key) # sanity
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None) # sanity
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
# Upload Part
key, upload_id = uploads[0]
content = b'a' * self.min_segment_size
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)
# Complete Multipart Upload
key, upload_id = uploads[0]
etags = [md5(content, usedforsecurity=False).hexdigest()]
xml = self._gen_comp_xml(etags)
status, headers, body = \
self._complete_multi_upload(bucket, key, upload_id, xml)
self.assertEqual(status, 200) # sanity
# GET multipart object
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# DELETE bucket while the object existing
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 409) # sanity
# The object must still be there.
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# Can delete it with DeleteMultipleObjects request
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
obj_elem = SubElement(elem, 'Object')
SubElement(obj_elem, 'Key').text = key
body = tostring(elem, use_s3ns=False)
status, headers, body = self.conn.make_request(
'POST',
bucket,
body=body,
query='delete',
headers={'Content-MD5': calculate_md5(body)})
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 404) # sanity
# Now we can delete
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 204) # sanity
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {'Accept': 'application/json'}
for key, val in resp.headers.items():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
container = req.container_name + MULTIUPLOAD_SUFFIX
s3_etag_hasher = md5()
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(xml).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload',
self.logger)
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = normalize_etag(part_elem.find('./ETag').text)
if len(etag) != 32 or any(c not in '0123456789abcdef'
for c in etag):
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
manifest.append({
'path':
'/%s/%s/%s/%d' %
(container, req.object_name, upload_id, part_number),
'etag':
etag
})
s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
headers[sysmeta_header('object', 'etag')] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers[get_container_update_override_key('etag')] = c_etag
too_small_message = ('s3api requires that each segment be at least '
'%d bytes' % self.conf.min_segment_size)
def size_checker(manifest):
# Check the size of each segment except the last and make sure
# they are all more than the minimum upload chunk size.
# Note that we need to use the *internal* keys, since we're
# looking at the manifest that's about to be written.
return [(item['name'], too_small_message) for item in manifest[:-1]
if item and item['bytes'] < self.conf.min_segment_size]
req.environ['swift.callback.slo_manifest_hook'] = size_checker
start_time = time.time()
def response_iter():
# NB: XML requires that the XML declaration, if present, be at the
# very start of the document. Clients *will* call us out on not
# being valid XML if we pass through whitespace before it.
# Track whether we've sent anything yet so we can yield out that
# declaration *first*
yielded_anything = False
try:
try:
# TODO: add support for versioning
put_resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={
'multipart-manifest':
'put',
'heartbeat': 'on'
},
headers=headers)
if put_resp.status_int == 202:
body = []
put_resp.fix_conditional_response()
for chunk in put_resp.response_iter:
if not chunk.strip():
if time.time() - start_time < 10:
# Include some grace period to keep
# ceph-s3tests happy
continue
if not yielded_anything:
yield (b'<?xml version="1.0" '
b'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
continue
body.append(chunk)
body = json.loads(b''.join(body))
if body['Response Status'] != '201 Created':
for seg, err in body['Errors']:
if err == too_small_message:
raise EntityTooSmall()
elif err in ('Etag Mismatch', '404 Not Found'):
raise InvalidPart(upload_id=upload_id)
raise InvalidRequest(
status=body['Response Status'],
msg='\n'.join(': '.join(err)
for err in body['Errors']))
except BadSwiftRequest as e:
msg = str(e)
if too_small_message in msg:
raise EntityTooSmall(msg)
elif ', Etag Mismatch' in msg:
raise InvalidPart(upload_id=upload_id)
elif ', 404 Not Found' in msg:
raise InvalidPart(upload_id=upload_id)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
# We know that this existed long enough for us to HEAD
pass
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at
# the request header when the port is non default value and it
# makes req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
# Why are we doing our own port parsing? Because py3 decided
# to start raising ValueErrors on access after parsing such
# an invalid port
netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
if ':' in netloc:
port = netloc.split(':', 2)[1]
host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None)
if yielded_anything:
yield b'\n'
yield tostring(result_elem,
xml_declaration=not yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
for chunk in err_resp({}, lambda *a: None):
yield chunk
resp = HTTPOk() # assume we're good for now... but see above!
resp.app_iter = reiterate(response_iter())
resp.content_type = "application/xml"
return resp
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
objtable = dict((o['name'], {
'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']
}) for o in objinfo)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
complete_elem = fromstring(xml, 'CompleteMultipartUpload',
self.logger)
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" %
(req.object_name, upload_id, part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < self.conf.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app,
'PUT',
body='',
headers=headers)
except BadSwiftRequest as e:
msg = str(e)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
pass # We know that this existed long enough for us to HEAD
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = resp.etag
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def test_delete_multi_objects(self):
bucket = 'bucket'
put_objects = ['obj%s' % var for var in xrange(4)]
self._prepare_test_delete_multi_objects(bucket, put_objects)
query = 'delete'
# Delete an object via MultiDelete API
req_objects = ['obj0']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue(headers['content-type'] is not None)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body)
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API
req_objects = ['obj1', 'obj2']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API but one (obj4) doesn't exist.
req_objects = ['obj3', 'obj4']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
# S3 assumes a NoSuchKey object as deleted.
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API but no objects exist
req_objects = ['obj4', 'obj5']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
def _check_acl(self, owner, body):
elem = fromstring(body, 'AccessControlPolicy')
permission = elem.find('./AccessControlList/Grant/Permission').text
self.assertEqual(permission, 'FULL_CONTROL')
name = elem.find('./AccessControlList/Grant/Grantee/ID').text
self.assertEqual(name, owner)
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {
'Accept': 'application/json',
sysmeta_header('object', 'upload-id'): upload_id
}
for key, val in resp.headers.items():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
container = req.container_name + MULTIUPLOAD_SUFFIX
s3_etag_hasher = md5(usedforsecurity=False)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(
xml, usedforsecurity=False).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload',
self.logger)
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = normalize_etag(part_elem.find('./ETag').text)
if len(etag) != 32 or any(c not in '0123456789abcdef'
for c in etag):
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
manifest.append({
'path':
'/%s/%s/%s/%d' %
(wsgi_to_str(container), wsgi_to_str(
req.object_name), upload_id, part_number),
'etag':
etag
})
s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
s3_etag_header = sysmeta_header('object', 'etag')
if resp.sysmeta_headers.get(s3_etag_header) == s3_etag:
# This header should only already be present if the upload marker
# has been cleaned up and the current target uses the same
# upload-id; assuming the segments to use haven't changed, the work
# is already done
return HTTPOk(body=_make_complete_body(req, s3_etag, False),
content_type='application/xml')
headers[s3_etag_header] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers[get_container_update_override_key('etag')] = c_etag
too_small_message = ('s3api requires that each segment be at least '
'%d bytes' % self.conf.min_segment_size)
def size_checker(manifest):
# Check the size of each segment except the last and make sure
# they are all more than the minimum upload chunk size.
# Note that we need to use the *internal* keys, since we're
# looking at the manifest that's about to be written.
return [(item['name'], too_small_message) for item in manifest[:-1]
if item and item['bytes'] < self.conf.min_segment_size]
req.environ['swift.callback.slo_manifest_hook'] = size_checker
start_time = time.time()
def response_iter():
# NB: XML requires that the XML declaration, if present, be at the
# very start of the document. Clients *will* call us out on not
# being valid XML if we pass through whitespace before it.
# Track whether we've sent anything yet so we can yield out that
# declaration *first*
yielded_anything = False
try:
try:
# TODO: add support for versioning
put_resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={
'multipart-manifest':
'put',
'heartbeat': 'on'
},
headers=headers)
if put_resp.status_int == 202:
body = []
put_resp.fix_conditional_response()
for chunk in put_resp.response_iter:
if not chunk.strip():
if time.time() - start_time < 10:
# Include some grace period to keep
# ceph-s3tests happy
continue
if not yielded_anything:
yield (b'<?xml version="1.0" '
b'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
continue
body.append(chunk)
body = json.loads(b''.join(body))
if body['Response Status'] != '201 Created':
for seg, err in body['Errors']:
if err == too_small_message:
raise EntityTooSmall()
elif err in ('Etag Mismatch', '404 Not Found'):
raise InvalidPart(upload_id=upload_id)
raise InvalidRequest(
status=body['Response Status'],
msg='\n'.join(': '.join(err)
for err in body['Errors']))
except BadSwiftRequest as e:
msg = str(e)
if too_small_message in msg:
raise EntityTooSmall(msg)
elif ', Etag Mismatch' in msg:
raise InvalidPart(upload_id=upload_id)
elif ', 404 Not Found' in msg:
raise InvalidPart(upload_id=upload_id)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
# The important thing is that we wrote out a tombstone to
# make sure the marker got cleaned up. If it's already
# gone (e.g., because of concurrent completes or a retried
# complete), so much the better.
pass
yield _make_complete_body(req, s3_etag, yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
for chunk in err_resp({}, lambda *a: None):
yield chunk
resp = HTTPOk() # assume we're good for now... but see above!
resp.app_iter = reiterate(response_iter())
resp.content_type = "application/xml"
return resp
def test_object(self):
obj = 'object name with %-sign'
content = b'abc123'
etag = md5(content).hexdigest()
# PUT Object
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, body=content)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue('content-length' in headers) # sanity
self.assertEqual(headers['content-length'], '0')
self._assertObjectEtag(self.bucket, obj, etag)
# PUT Object Copy
dst_bucket = 'dst-bucket'
dst_obj = 'dst_obj'
self.conn.make_request('PUT', dst_bucket)
headers = {'x-amz-copy-source': '/%s/%s' % (self.bucket, obj)}
status, headers, body = \
self.conn.make_request('PUT', dst_bucket, dst_obj,
headers=headers)
self.assertEqual(status, 200)
# PUT Object Copy with URL-encoded Source
dst_bucket = 'dst-bucket'
dst_obj = 'dst_obj'
self.conn.make_request('PUT', dst_bucket)
headers = {'x-amz-copy-source': quote('/%s/%s' % (self.bucket, obj))}
status, headers, body = \
self.conn.make_request('PUT', dst_bucket, dst_obj,
headers=headers)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body, 'CopyObjectResult')
self.assertTrue(elem.find('LastModified').text is not None)
last_modified_xml = elem.find('LastModified').text
self.assertTrue(elem.find('ETag').text is not None)
self.assertEqual(etag, elem.find('ETag').text.strip('"'))
self._assertObjectEtag(dst_bucket, dst_obj, etag)
# Check timestamp on Copy:
status, headers, body = \
self.conn.make_request('GET', dst_bucket)
self.assertEqual(status, 200)
elem = fromstring(body, 'ListBucketResult')
# FIXME: COPY result drops milli/microseconds but GET doesn't
self.assertEqual(
elem.find('Contents').find("LastModified").text.rsplit('.', 1)[0],
last_modified_xml.rsplit('.', 1)[0])
# GET Object
status, headers, body = \
self.conn.make_request('GET', self.bucket, obj)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers, etag)
self.assertTrue(headers['last-modified'] is not None)
self.assertTrue(headers['content-type'] is not None)
self.assertEqual(headers['content-length'], str(len(content)))
# HEAD Object
status, headers, body = \
self.conn.make_request('HEAD', self.bucket, obj)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers, etag)
self.assertTrue(headers['last-modified'] is not None)
self.assertTrue('content-type' in headers)
self.assertEqual(headers['content-length'], str(len(content)))
# DELETE Object
status, headers, body = \
self.conn.make_request('DELETE', self.bucket, obj)
self.assertEqual(status, 204)
self.assertCommonResponseHeaders(headers)
# DELETE Non-Existent Object
status, headers, body = \
self.conn.make_request('DELETE', self.bucket, 'does-not-exist')
self.assertEqual(status, 204)
self.assertCommonResponseHeaders(headers)
