def test_bucket_GET_is_truncated_url_encoded(self): bucket_name = 'junk' req = Request.blank( '/%s?encoding-type=url&max-keys=%d' % ( bucket_name, len(self.objects)), environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'false') req = Request.blank( '/%s?encoding-type=url&max-keys=%d' % ( bucket_name, len(self.objects) - 1), environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'true') req = Request.blank('/subdirs?encoding-type=url&delimiter=/&' 'max-keys=2', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'true') self.assertEqual(elem.find('./NextMarker').text, quote(u'but-\u062a/'.encode('utf-8')))
def test_service(self): # GET Service(without bucket) status, headers, body = self.conn.make_request('GET') self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue(headers['content-type'] is not None) # TODO; requires consideration # self.assertEqual(headers['transfer-encoding'], 'chunked') elem = fromstring(body, 'ListAllMyBucketsResult') buckets = elem.findall('./Buckets/Bucket') self.assertEqual(list(buckets), []) owner = elem.find('Owner') self.assertEqual(self.conn.user_id, owner.find('ID').text) self.assertEqual(self.conn.user_id, owner.find('DisplayName').text) # GET Service(with Bucket) req_buckets = ('bucket', 'bucket2') for bucket in req_buckets: self.conn.make_request('PUT', bucket) status, headers, body = self.conn.make_request('GET') self.assertEqual(status, 200) elem = fromstring(body, 'ListAllMyBucketsResult') resp_buckets = elem.findall('./Buckets/Bucket') self.assertEqual(len(list(resp_buckets)), 2) for b in resp_buckets: self.assertTrue(b.find('Name').text in req_buckets) self.assertTrue(b.find('CreationDate') is not None)
def test_bucket_versioning_GET(self): req = Request.blank('/bucket?versioning', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) fromstring(body, 'VersioningConfiguration')
def test_bucket_GET_v2_with_delimiter_max_keys(self): bucket_name = 'junk' req = Request.blank( '/%s?list-type=2&delimiter=a&max-keys=2' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') next_token = elem.find('./NextContinuationToken') self.assertIsNotNone(next_token) self.assertEqual(elem.find('./MaxKeys').text, '2') self.assertEqual(elem.find('./IsTruncated').text, 'true') req = Request.blank( '/%s?list-type=2&delimiter=a&max-keys=2&continuation-token=%s' % (bucket_name, next_token.text), environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') names = [o.find('./Key').text for o in elem.iterchildren('Contents')] self.assertEqual(names[0], 'lily')
def test_bucket_GET_max_keys(self): bucket_name = 'junk' req = Request.blank('/%s?max-keys=5' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./MaxKeys').text, '5') _, path = self.swift.calls[-1] _, query_string = path.split('?') args = dict(cgi.parse_qsl(query_string)) self.assertEqual(args['limit'], '6') req = Request.blank('/%s?max-keys=5000' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./MaxKeys').text, '5000') _, path = self.swift.calls[-1] _, query_string = path.split('?') args = dict(cgi.parse_qsl(query_string)) self.assertEqual(args['limit'], '1001')
def test_bucket_GET_v2_fetch_owner(self): bucket_name = 'junk' req = Request.blank('/%s?list-type=2' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) objects = elem.iterchildren('Contents') for o in objects: self.assertIsNone(o.find('./Owner')) req = Request.blank('/%s?list-type=2&fetch-owner=true' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) objects = elem.iterchildren('Contents') for o in objects: self.assertIsNotNone(o.find('./Owner'))
def test_delete_multi_objects_with_quiet(self): bucket = 'bucket' put_objects = ['obj'] query = 'delete' # with Quiet true quiet = 'true' self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects, quiet) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), 0) # with Quiet false quiet = 'false' self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects, quiet) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), 1)
def test_bucket_GET_is_truncated(self): bucket_name = 'junk' req = Request.blank('/%s?max-keys=5' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'false') req = Request.blank('/%s?max-keys=4' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'true') req = Request.blank('/subdirs?delimiter=/&max-keys=2', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./IsTruncated').text, 'true') self.assertEqual(elem.find('./NextMarker').text, 'but/')
def test_xml_namespace(self): def test_xml(ns, prefix): return '<A %(ns)s><%(prefix)sB>C</%(prefix)sB></A>' % \ ({'ns': ns, 'prefix': prefix}) # No namespace is same as having the S3 namespace. xml = test_xml('', '') elem = etree.fromstring(xml) self.assertEqual(elem.find('./B').text, 'C') # The S3 namespace is handled as no namespace. xml = test_xml('xmlns="%s"' % etree.XMLNS_S3, '') elem = etree.fromstring(xml) self.assertEqual(elem.find('./B').text, 'C') xml = test_xml('xmlns:s3="%s"' % etree.XMLNS_S3, 's3:') elem = etree.fromstring(xml) self.assertEqual(elem.find('./B').text, 'C') # Any namespaces without a prefix work as no namespace. xml = test_xml('xmlns="http://example.com/"', '') elem = etree.fromstring(xml) self.assertEqual(elem.find('./B').text, 'C') xml = test_xml('xmlns:s3="http://example.com/"', 's3:') elem = etree.fromstring(xml) self.assertIsNone(elem.find('./B'))
def test_bucket_GET_v2_is_truncated(self): bucket_name = 'junk' req = Request.blank( '/%s?list-type=2&max-keys=%d' % (bucket_name, len(self.objects)), environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./KeyCount').text, str(len(self.objects))) self.assertEqual(elem.find('./IsTruncated').text, 'false') req = Request.blank( '/%s?list-type=2&max-keys=%d' % (bucket_name, len(self.objects) - 1), environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertIsNotNone(elem.find('./NextContinuationToken')) self.assertEqual(elem.find('./KeyCount').text, str(len(self.objects) - 1)) self.assertEqual(elem.find('./IsTruncated').text, 'true') req = Request.blank('/subdirs?list-type=2&delimiter=/&max-keys=2', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertIsNotNone(elem.find('./NextContinuationToken')) self.assertEqual(elem.find('./KeyCount').text, '2') self.assertEqual(elem.find('./IsTruncated').text, 'true')
def test_get_bucket_v2_with_continuation_token(self): bucket = 'bucket' put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object', 'dir/subdir/object') self._prepare_test_get_bucket(bucket, put_objects) query = 'list-type=2&max-keys=3' expect_objects = ('dir/subdir/object', 'object', 'object2') status, headers, body = \ self.conn.make_request('GET', bucket, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('MaxKeys').text, '3') self.assertEqual(elem.find('KeyCount').text, '3') self.assertEqual(elem.find('IsTruncated').text, 'true') next_cont_token_elem = elem.find('NextContinuationToken') self.assertIsNotNone(next_cont_token_elem) resp_objects = elem.findall('./Contents') self.assertEqual(len(list(resp_objects)), len(expect_objects)) for i, o in enumerate(resp_objects): self.assertEqual(o.find('Key').text, expect_objects[i]) self.assertTrue(o.find('LastModified').text is not None) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertTrue(o.find('ETag').text is not None) self.assertTrue(o.find('Size').text is not None) self.assertEqual(o.find('StorageClass').text, 'STANDARD') self.assertIsNone(o.find('Owner/ID')) self.assertIsNone(o.find('Owner/DisplayName')) query = 'list-type=2&max-keys=3&continuation-token=%s' % \ next_cont_token_elem.text expect_objects = ('subdir/object', 'subdir2/object') status, headers, body = \ self.conn.make_request('GET', bucket, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('MaxKeys').text, '3') self.assertEqual(elem.find('KeyCount').text, '2') self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertIsNone(elem.find('NextContinuationToken')) cont_token_elem = elem.find('ContinuationToken') self.assertEqual(cont_token_elem.text, next_cont_token_elem.text) resp_objects = elem.findall('./Contents') self.assertEqual(len(list(resp_objects)), len(expect_objects)) for i, o in enumerate(resp_objects): self.assertEqual(o.find('Key').text, expect_objects[i]) self.assertTrue(o.find('LastModified').text is not None) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertTrue(o.find('ETag').text is not None) self.assertTrue(o.find('Size').text is not None) self.assertEqual(o.find('StorageClass').text, 'STANDARD') self.assertIsNone(o.find('Owner/ID')) self.assertIsNone(o.find('Owner/DisplayName'))
def test_abort_multi_upload_error(self): bucket = 'bucket' self.conn.make_request('PUT', bucket) key = 'obj' query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, key, query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text self._upload_part(bucket, key, upload_id) query = 'uploadId=%s' % upload_id auth_error_conn = Connection(aws_secret_key='invalid') status, headers, body = \ auth_error_conn.make_request('DELETE', bucket, key, query=query) self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') status, headers, body = \ self.conn.make_request('DELETE', 'nothing', key, query=query) self.assertEqual(get_error_code(body), 'NoSuchBucket') status, headers, body = \ self.conn.make_request('DELETE', bucket, 'nothing', query=query) self.assertEqual(get_error_code(body), 'NoSuchUpload') query = 'uploadId=%s' % 'nothing' status, headers, body = \ self.conn.make_request('DELETE', bucket, key, query=query) self.assertEqual(get_error_code(body), 'NoSuchUpload')
def test_get_bucket_with_delimiter(self): bucket = 'bucket' put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object', 'dir/subdir/object') self._prepare_test_get_bucket(bucket, put_objects) delimiter = '/' query = 'delimiter=%s' % delimiter expect_objects = ('object', 'object2') expect_prefixes = ('dir/', 'subdir/', 'subdir2/') status, headers, body = \ self.conn.make_request('GET', bucket, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('Delimiter').text, delimiter) resp_objects = elem.findall('./Contents') self.assertEqual(len(list(resp_objects)), len(expect_objects)) for i, o in enumerate(resp_objects): self.assertEqual(o.find('Key').text, expect_objects[i]) self.assertTrue(o.find('LastModified').text is not None) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertTrue(o.find('ETag').text is not None) self.assertTrue(o.find('Size').text is not None) self.assertEqual(o.find('StorageClass').text, 'STANDARD') self.assertTrue(o.find('Owner/ID').text, self.conn.user_id) self.assertTrue(o.find('Owner/DisplayName').text, self.conn.user_id) resp_prefixes = elem.findall('CommonPrefixes') self.assertEqual(len(resp_prefixes), len(expect_prefixes)) for i, p in enumerate(resp_prefixes): self.assertEqual(p.find('./Prefix').text, expect_prefixes[i])
def test_upload_part_error(self): bucket = 'bucket' self.conn.make_request('PUT', bucket) query = 'uploads' key = 'obj' status, headers, body = \ self.conn.make_request('POST', bucket, key, query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text query = 'partNumber=%s&uploadId=%s' % (1, upload_id) auth_error_conn = Connection(aws_secret_key='invalid') status, headers, body = \ auth_error_conn.make_request('PUT', bucket, key, query=query) self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') status, headers, body = \ self.conn.make_request('PUT', 'nothing', key, query=query) self.assertEqual(get_error_code(body), 'NoSuchBucket') query = 'partNumber=%s&uploadId=%s' % (1, 'nothing') status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query) self.assertEqual(get_error_code(body), 'NoSuchUpload') query = 'partNumber=%s&uploadId=%s' % (0, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query) self.assertEqual(get_error_code(body), 'InvalidArgument') err_msg = 'Part number must be an integer between 1 and' self.assertTrue(err_msg in get_error_msg(body))
def test_bucket_GET(self): bucket_name = 'junk' req = Request.blank('/%s' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) objects = elem.iterchildren('Contents') names = [] for o in objects: names.append(o.find('./Key').text) self.assertEqual('2011-01-05T02:19:14.275Z', o.find('./LastModified').text) self.assertEqual('"0"', o.find('./ETag').text) self.assertEqual(len(names), len(self.objects)) for i in self.objects: self.assertTrue(i[0] in names)
def test_fromstring_with_nonascii_text(self): input_str = '<?xml version="1.0" encoding="UTF-8"?>\n' \ '<Test><FOO>\xef\xbc\xa1</FOO></Test>' elem = etree.fromstring(input_str) text = elem.find('FOO').text self.assertEqual(text, '\xef\xbc\xa1') self.assertTrue(isinstance(text, str))
def test_get_bucket_with_prefix(self): bucket = 'bucket' req_objects = ('object', 'object2', 'subdir/object', 'subdir2/object', 'dir/subdir/object') self._prepare_test_get_bucket(bucket, req_objects) prefix = 'object' query = 'prefix=%s' % prefix expect_objects = ('object', 'object2') status, headers, body = \ self.conn.make_request('GET', bucket, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('Prefix').text, prefix) resp_objects = elem.findall('./Contents') self.assertEqual(len(list(resp_objects)), len(expect_objects)) for i, o in enumerate(resp_objects): self.assertEqual(o.find('Key').text, expect_objects[i]) self.assertIsNotNone(o.find('LastModified').text) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertIsNotNone(o.find('ETag').text) self.assertIsNotNone(o.find('Size').text) self.assertEqual(o.find('StorageClass').text, 'STANDARD') self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) self.assertEqual(o.find('Owner/DisplayName').text, self.conn.user_id)
def test_service_GET_with_blind_resource(self): buckets = (('apple', 1, 200), ('orange', 3, 430), ('apple+segment', 1, 200)) expected = buckets[:-1] bucket_list = create_bucket_list_json(buckets) self.swift.register('GET', '/v1/AUTH_test', swob.HTTPOk, {}, bucket_list) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListAllMyBucketsResult') all_buckets = elem.find('./Buckets') buckets = all_buckets.iterchildren('Bucket') listing = list(list(buckets)[0]) self.assertEqual(len(listing), 2) names = [] for b in all_buckets.iterchildren('Bucket'): names.append(b.find('./Name').text) self.assertEqual(len(names), len(expected)) for i in expected: self.assertTrue(i[0] in names)
def test_get_bucket_v2_with_fetch_owner(self): bucket = 'bucket' put_objects = ('object', 'object2', 'subdir/object', 'subdir2/object', 'dir/subdir/object') self._prepare_test_get_bucket(bucket, put_objects) query = 'list-type=2&fetch-owner=true' expect_objects = ('dir/subdir/object', 'object', 'object2', 'subdir/object', 'subdir2/object') status, headers, body = \ self.conn.make_request('GET', bucket, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('KeyCount').text, '5') resp_objects = elem.findall('./Contents') self.assertEqual(len(list(resp_objects)), len(expect_objects)) for i, o in enumerate(resp_objects): self.assertEqual(o.find('Key').text, expect_objects[i]) self.assertTrue(o.find('LastModified').text is not None) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertTrue(o.find('ETag').text is not None) self.assertTrue(o.find('Size').text is not None) self.assertEqual(o.find('StorageClass').text, 'STANDARD') self.assertTrue(o.find('Owner/ID').text, self.conn.user_id) self.assertTrue(o.find('Owner/DisplayName').text, self.conn.user_id)
def PUT(self, req): """ Handle PUT Bucket request """ xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE) if xml: # check location try: elem = fromstring( xml, 'CreateBucketConfiguration', self.logger) location = elem.find('./LocationConstraint').text except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except Exception as e: self.logger.error(e) raise if location != self.conf.location: # s3api cannot support multiple regions currently. raise InvalidLocationConstraint() resp = req.get_response(self.app) resp.status = HTTP_OK resp.location = '/' + req.container_name return resp
def get_acl(self, headers, body, bucket_owner, object_owner=None): """ Get ACL instance from S3 (e.g. x-amz-grant) headers or S3 acl xml body. """ acl = ACL.from_headers(headers, bucket_owner, object_owner, as_private=False) if acl is None: # Get acl from request body if possible. if not body: raise MissingSecurityHeader(missing_header_name='x-amz-acl') try: elem = fromstring(body, ACL.root_tag) acl = ACL.from_elem( elem, True, self.req.allow_no_owner) except(XMLSyntaxError, DocumentInvalid): raise MalformedACLError() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.logger.error(e) raise exc_type, exc_value, exc_traceback else: if body: # Specifying grant with both header and xml is not allowed. raise UnexpectedContent() return acl
def test_object_multi_DELETE_quiet(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) elem = Element('Delete') SubElement(elem, 'Quiet').text = 'true' for key in ['Key1', 'Key2']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 0)
def test_acl(self): self.conn.make_request('PUT', self.bucket, self.obj) query = 'acl' # PUT Bucket ACL headers = {'x-amz-acl': 'public-read'} status, headers, body = \ self.conn.make_request('PUT', self.bucket, headers=headers, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertEqual(headers['content-length'], '0') # GET Bucket ACL status, headers, body = \ self.conn.make_request('GET', self.bucket, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) # TODO: Fix the response that last-modified must be in the response. # self.assertTrue(headers['last-modified'] is not None) self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue(headers['content-type'] is not None) elem = fromstring(body, 'AccessControlPolicy') owner = elem.find('Owner') self.assertEqual(owner.find('ID').text, self.conn.user_id) self.assertEqual(owner.find('DisplayName').text, self.conn.user_id) acl = elem.find('AccessControlList') self.assertTrue(acl.find('Grant') is not None) # GET Object ACL status, headers, body = \ self.conn.make_request('GET', self.bucket, self.obj, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) # TODO: Fix the response that last-modified must be in the response. # self.assertTrue(headers['last-modified'] is not None) self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue(headers['content-type'] is not None) elem = fromstring(body, 'AccessControlPolicy') owner = elem.find('Owner') self.assertEqual(owner.find('ID').text, self.conn.user_id) self.assertEqual(owner.find('DisplayName').text, self.conn.user_id) acl = elem.find('AccessControlList') self.assertTrue(acl.find('Grant') is not None)
def test_object_multi_DELETE_without_permission(self): status, headers, body = self._test_object_multi_DELETE('test:other') self.assertEqual(status.split()[0], '200') elem = fromstring(body) errors = elem.findall('Error') self.assertEqual(len(errors), len(self.keys)) for e in errors: self.assertTrue(e.find('Key').text in self.keys) self.assertEqual(e.find('Code').text, 'AccessDenied') self.assertEqual(e.find('Message').text, 'Access Denied.')
def test_fromstring_with_nonascii_text(self): input_str = b'<?xml version="1.0" encoding="UTF-8"?>\n' \ b'<Test><FOO>\xef\xbc\xa1</FOO></Test>' elem = etree.fromstring(input_str) text = elem.find('FOO').text if six.PY2: self.assertEqual(text, b'\xef\xbc\xa1') else: self.assertEqual(text, b'\xef\xbc\xa1'.decode('utf8')) self.assertIsInstance(text, str)
def test_object_location(self): req = Request.blank('/bucket?location', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'LocationConstraint') location = elem.text self.assertIsNone(location)
def test_get_bucket_v2_with_continuation_token_and_delimiter(self): bucket = 'bucket' put_objects = ('object', u'object2-\u062a', 'subdir/object', u'subdir2-\u062a/object', 'dir/subdir/object', 'x', 'y', 'z') self._prepare_test_get_bucket(bucket, put_objects) expected = [{'objects': ['object', u'object2-\u062a'], 'subdirs': ['dir/']}, {'objects': ['x'], 'subdirs': ['subdir/', u'subdir2-\u062a/']}, {'objects': ['y', 'z'], 'subdirs': []}] continuation_token = '' query = 'list-type=2&max-keys=3&delimiter=/&continuation-token=%s' for i in range(len(expected)): status, headers, body = self.conn.make_request( 'GET', bucket, query=query % continuation_token) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('MaxKeys').text, '3') self.assertEqual( elem.find('KeyCount').text, str(len(expected[i]['objects']) + len(expected[i]['subdirs']))) expect_truncated = 'true' if i < len(expected) - 1 else 'false' self.assertEqual(elem.find('IsTruncated').text, expect_truncated) next_cont_token_elem = elem.find('NextContinuationToken') if expect_truncated == 'true': self.assertIsNotNone(next_cont_token_elem) continuation_token = next_cont_token_elem.text resp_objects = elem.findall('./Contents') self.assertEqual( len(list(resp_objects)), len(expected[i]['objects'])) for j, o in enumerate(resp_objects): self.assertEqual(o.find('Key').text, expected[i]['objects'][j].encode('utf-8')) self.assertTrue(o.find('LastModified').text is not None) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertTrue(o.find('ETag').text is not None) self.assertTrue(o.find('Size').text is not None) self.assertEqual(o.find('StorageClass').text, 'STANDARD') self.assertIsNone(o.find('Owner/ID')) self.assertIsNone(o.find('Owner/DisplayName')) resp_subdirs = elem.findall('./CommonPrefixes') self.assertEqual( len(list(resp_subdirs)), len(expected[i]['subdirs'])) for j, o in enumerate(resp_subdirs): self.assertEqual( o.find('Prefix').text, expected[i]['subdirs'][j].encode('utf-8'))
def test_object_PUT_copy_self(self): status, headers, body = \ self._test_object_PUT_copy_self(swob.HTTPOk) self.assertEqual(status.split()[0], '400') elem = fromstring(body, 'Error') err_msg = ("This copy request is illegal because it is trying to copy " "an object to itself without changing the object's " "metadata, storage class, website redirect location or " "encryption attributes.") self.assertEqual(elem.find('Code').text, 'InvalidRequest') self.assertEqual(elem.find('Message').text, err_msg)
def test_bucket_GET_subdir_with_delimiter_max_keys(self): bucket_name = 'junk-subdir' req = Request.blank('/%s?delimiter=a&max-keys=1' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./NextMarker').text, 'rose') self.assertEqual(elem.find('./MaxKeys').text, '1') self.assertEqual(elem.find('./IsTruncated').text, 'true')
def test_delete_bucket_multi_upload_object_exisiting(self): bucket = 'bucket' keys = ['obj1'] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( bucket, keys) # Initiate Multipart Upload for expected_key, (status, _, body) in \ izip(keys, results_generator): self.assertEqual(status, 200) # sanity elem = fromstring(body, 'InitiateMultipartUploadResult') key = elem.find('Key').text self.assertEqual(expected_key, key) # sanity upload_id = elem.find('UploadId').text self.assertTrue(upload_id is not None) # sanity self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity # Upload Part key, upload_id = uploads[0] content = 'a' * self.min_segment_size status, headers, body = \ self._upload_part(bucket, key, upload_id, content) self.assertEqual(status, 200) # Complete Multipart Upload key, upload_id = uploads[0] etags = [md5(content).hexdigest()] xml = self._gen_comp_xml(etags) status, headers, body = \ self._complete_multi_upload(bucket, key, upload_id, xml) self.assertEqual(status, 200) # sanity # GET multipart object status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 200) # sanity self.assertEqual(content, body) # sanity # DELETE bucket while the object existing status, headers, body = \ self.conn.make_request('DELETE', bucket) self.assertEqual(status, 409) # sanity # The object must still be there. status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 200) # sanity self.assertEqual(content, body) # sanity
def test_bucket_GET_passthroughs(self): bucket_name = 'junk' req = Request.blank('/%s?delimiter=a&marker=b&prefix=c' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./Prefix').text, 'c') self.assertEqual(elem.find('./Marker').text, 'b') self.assertEqual(elem.find('./Delimiter').text, 'a') _, path = self.swift.calls[-1] _, query_string = path.split('?') args = dict(cgi.parse_qsl(query_string)) self.assertEqual(args['delimiter'], 'a') self.assertEqual(args['marker'], 'b') self.assertEqual(args['prefix'], 'c')
def test_bucket_GET_subdir(self): bucket_name = 'junk-subdir' req = Request.blank('/%s' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) prefixes = elem.findall('CommonPrefixes') self.assertEqual(len(prefixes), len(self.prefixes)) for p in prefixes: self.assertTrue(p.find('./Prefix').text in self.prefixes)
def test_bucket_GET_with_versions_versioning_not_configured(self): req = Request.blank('/junk?versions', environ={'REQUEST_METHOD': 'GET'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListVersionsResult') self.assertEqual(elem.find('./Name').text, 'junk') self.assertIsNone(elem.find('./Prefix').text) self.assertIsNone(elem.find('./KeyMarker').text) self.assertIsNone(elem.find('./VersionIdMarker').text) self.assertEqual(elem.find('./MaxKeys').text, '1000') self.assertEqual(elem.find('./IsTruncated').text, 'false') self.assertEqual(elem.findall('./DeleteMarker'), []) versions = elem.findall('./Version') objects = list(self.objects) self.assertEqual([v.find('./Key').text for v in versions], [v[0].encode('utf-8') for v in objects]) self.assertEqual([v.find('./IsLatest').text for v in versions], ['true' for v in objects]) self.assertEqual([v.find('./VersionId').text for v in versions], ['null' for v in objects]) # Last modified in self.objects is 2011-01-05T02:19:14.275290 but # the returned value is 2011-01-05T02:19:14.275Z self.assertEqual([v.find('./LastModified').text for v in versions], [v[1][:-3] + 'Z' for v in objects]) self.assertEqual( [v.find('./ETag').text for v in versions], ['"0-N"' if v[0] == 'slo' else '"0"' for v in objects]) self.assertEqual([v.find('./Size').text for v in versions], [str(v[3]) for v in objects]) self.assertEqual([v.find('./Owner/ID').text for v in versions], ['test:tester' for v in objects]) self.assertEqual( [v.find('./Owner/DisplayName').text for v in versions], ['test:tester' for v in objects]) self.assertEqual([v.find('./StorageClass').text for v in versions], ['STANDARD' for v in objects])
def PUT(self, req): # pylint: disable=invalid-name """ Handles PUT Bucket CORS. """ xml = req.xml(MAX_CORS_BODY_SIZE) try: data = fromstring(xml, "CorsConfiguration") except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except Exception as exc: self.logger.error(exc) raise # forbid wildcard for ExposeHeader check_cors_rule(data) req.headers[BUCKET_CORS_HEADER] = xml resp = req._get_response(self.app, 'POST', req.container_name, None) return self.convert_response(req, resp, 204, HTTPOk)
def test_service_GET_without_owner_bucket(self): bucket_list = [] for var in range(0, 10): user_id = 'test:other' bucket = 'bucket%s' % var owner = Owner(user_id, user_id) headers = encode_acl('container', ACL(owner, [])) self.swift.register('HEAD', '/v1/AUTH_test/%s' % bucket, swob.HTTPNoContent, headers, None) bucket_list.append((bucket, var, 300 + var)) status, headers, body = \ self._test_service_GET_for_check_bucket_owner(bucket_list) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListAllMyBucketsResult') resp_buckets = elem.find('./Buckets') buckets = resp_buckets.iterchildren('Bucket') self.assertEqual(len(list(buckets)), 0)
def _test_method_error(self, method, path, response_class, headers={}, env={}, expected_xml_tags=None): if not path.startswith('/'): path = '/' + path # add a missing slash before the path uri = '/v1/AUTH_test' if path != '/': uri += path self.swift.register(method, uri, response_class, headers, None) headers.update({'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) env.update({'REQUEST_METHOD': method}) req = swob.Request.blank(path, environ=env, headers=headers) status, headers, body = self.call_s3api(req) if expected_xml_tags is not None: elem = fromstring(body, 'Error') self.assertEqual(set(expected_xml_tags), {x.tag for x in elem}) return self._get_error_code(body)
def test_service_GET_bucket_list(self): bucket_list = [] for var in range(0, 10): if var % 3 == 0: user_id = 'test:tester' else: user_id = 'test:other' bucket = 'bucket%s' % var owner = Owner(user_id, user_id) headers = encode_acl('container', ACL(owner, [])) # set register to get owner of buckets if var % 3 == 2: self.swift.register('HEAD', '/v1/AUTH_test/%s' % bucket, swob.HTTPNotFound, {}, None) else: self.swift.register('HEAD', '/v1/AUTH_test/%s' % bucket, swob.HTTPNoContent, headers, None) bucket_list.append((bucket, var, 300 + var)) status, headers, body = \ self._test_service_GET_for_check_bucket_owner(bucket_list) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListAllMyBucketsResult') resp_buckets = elem.find('./Buckets') buckets = resp_buckets.iterchildren('Bucket') listing = list(list(buckets)[0]) self.assertEqual(len(listing), 2) names = [] for b in resp_buckets.iterchildren('Bucket'): names.append(b.find('./Name').text) # Check whether getting bucket only locate in multiples of 3 in # bucket_list which mean requested user is owner. expected_buckets = [b for i, b in enumerate(bucket_list) if i % 3 == 0] self.assertEqual(len(names), len(expected_buckets)) for i in expected_buckets: self.assertTrue(i[0] in names) self.assertEqual(len(self.swift.calls_with_headers), 11)
def _upload_part_copy(self, src_bucket, src_obj, dst_bucket, dst_key, upload_id, part_num=1, src_range=None): src_path = '%s/%s' % (src_bucket, src_obj) query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id) req_headers = {'X-Amz-Copy-Source': src_path} if src_range: req_headers['X-Amz-Copy-Source-Range'] = src_range status, headers, body = \ self.conn.make_request('PUT', dst_bucket, dst_key, headers=req_headers, query=query) elem = fromstring(body, 'CopyPartResult') etag = elem.find('ETag').text.strip('"') return status, headers, body, etag
def test_bucket_GET_v2_with_nonascii_queries(self): bucket_name = 'junk' req = Request.blank( '/%s?list-type=2&delimiter=\xef\xbc\xa1&start-after=\xef\xbc\xa2&' 'prefix=\xef\xbc\xa3' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'ListBucketResult') self.assertEqual(elem.find('./Prefix').text, '\xef\xbc\xa3') self.assertEqual(elem.find('./StartAfter').text, '\xef\xbc\xa2') self.assertEqual(elem.find('./Delimiter').text, '\xef\xbc\xa1') _, path = self.swift.calls[-1] _, query_string = path.split('?') args = dict(cgi.parse_qsl(query_string)) self.assertEqual(args['delimiter'], '\xef\xbc\xa1') self.assertEqual(args['marker'], '\xef\xbc\xa2') self.assertEqual(args['prefix'], '\xef\xbc\xa3')
def do_test(src_path=None): date_header = self.get_date_header() timestamp = mktime(date_header) last_modified = S3Timestamp(timestamp).s3xmlformat status, headers, body = self._test_object_PUT_copy( swob.HTTPOk, put_header={'Date': date_header}, timestamp=timestamp, src_path=src_path) self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Content-Type'], 'application/xml') self.assertTrue(headers.get('etag') is None) self.assertTrue(headers.get('x-amz-meta-something') is None) elem = fromstring(body, 'CopyObjectResult') self.assertEqual(elem.find('LastModified').text, last_modified) self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) _, _, headers = self.swift.calls_with_headers[-1] self.assertEqual(headers['X-Copy-From'], '/some/source') self.assertEqual(headers['Content-Length'], '0')
def test_object_multi_DELETE(self): self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk, {'x-static-large-object': 'True'}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk, {}, None) elem = Element('Delete') for key in ['Key1', 'Key2', 'Key3']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Content-Type': 'multipart/form-data', 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 3) self.assertEqual(self.swift.calls, [ ('HEAD', '/v1/AUTH_test/bucket'), ('HEAD', '/v1/AUTH_test/bucket/Key1'), ('DELETE', '/v1/AUTH_test/bucket/Key1'), ('HEAD', '/v1/AUTH_test/bucket/Key2'), ('HEAD', '/v1/AUTH_test/bucket/Key3'), ('DELETE', '/v1/AUTH_test/bucket/Key3?multipart-manifest=delete'), ])
def test_service_GET_subresource(self): req = Request.blank('/?acl', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListAllMyBucketsResult') all_buckets = elem.find('./Buckets') buckets = all_buckets.iterchildren('Bucket') listing = list(list(buckets)[0]) self.assertEqual(len(listing), 2) names = [] for b in all_buckets.iterchildren('Bucket'): names.append(b.find('./Name').text) self.assertEqual(len(names), len(self.buckets)) for i in self.buckets: self.assertTrue(i[0] in names)
def test_bucket_GET_other_account(self): self.assertIsNone(self.db.get_owner('bucket')) status, _, _ = self._bucket_put() self.assertEqual(status.split()[0], '200') self.assertEqual(self.db.get_owner('bucket'), 'AUTH_test') # Register request with account 'test'. expected_body = json.dumps([{ "name": "expected", "last_modified": "2017-04-21T16:30:34.133034", "hash": "0000", "bytes": 0 }]).encode('utf-8') self.swift.register('GET', '/v1/AUTH_test/bucket?format=json&limit=1001', swob.HTTPOk, {}, expected_body) # Then do a call with 'test2' account, that should be changed # to 'test' by the middleware (because the bucket 'bucket' belongs # to account 'test'). status, _, body = self._bucket_get('bucket', account='test2') elem = fromstring(body, "ListBucketResult") self.assertEqual(status.split()[0], '200') self.assertEqual(elem.find('Contents').find('Key').text, "expected")
def test_complete_upload_with_fewer_etags(self): bucket = 'bucket' key = 'obj' self.conn.make_request('PUT', bucket) query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, key, query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text etags = [] for i in xrange(1, 4): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, body='A' * 1024 * 1024 * 5, query=query) etags.append(headers['etag']) query = 'uploadId=%s' % upload_id xml = self._gen_comp_xml(etags[:-1]) status, headers, body = \ self.conn.make_request('POST', bucket, key, body=xml, query=query) self.assertEqual(status, 200)
def test_object_multi_DELETE_lots_of_keys(self): elem = Element('Delete') for i in range(self.conf.max_multi_delete_objects): name = 'x' * 1000 + str(i) self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name, swob.HTTPNotFound, {}, None) obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = name body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_s3api(req) self.assertEqual('200 OK', status) elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), self.conf.max_multi_delete_objects)
def test_bucket_GET_url_encoded(self): bucket_name = 'junk' req = Request.blank('/%s?encoding-type=url' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') name = elem.find('./Name').text self.assertEqual(name, bucket_name) objects = elem.iterchildren('Contents') items = [] for o in objects: items.append((o.find('./Key').text, o.find('./ETag').text)) self.assertEqual('2011-01-05T02:19:14.275Z', o.find('./LastModified').text) self.assertEqual(items, [ (quote(i[0].encode('utf-8')), '"0-N"' if i[0] == 'slo' else '"0"') for i in self.objects])
def test_object_multi_upload_part_copy_range(self): bucket = 'bucket' keys = ['obj1'] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( bucket, keys) # Initiate Multipart Upload for expected_key, (status, headers, body) in \ izip(keys, results_generator): self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'InitiateMultipartUploadResult') self.assertEqual(elem.find('Bucket').text, bucket) key = elem.find('Key').text self.assertEqual(expected_key, key) upload_id = elem.find('UploadId').text self.assertTrue(upload_id is not None) self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity # Upload Part Copy Range key, upload_id = uploads[0] src_bucket = 'bucket2' src_obj = 'obj4' src_content = 'y' * (self.min_segment_size / 2) + 'z' * \ self.min_segment_size src_range = 'bytes=0-%d' % (self.min_segment_size - 1) etag = md5(src_content[:self.min_segment_size]).hexdigest() # prepare src obj self.conn.make_request('PUT', src_bucket) self.conn.make_request('PUT', src_bucket, src_obj, body=src_content) _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj) self.assertCommonResponseHeaders(headers) status, headers, body, resp_etag = \ self._upload_part_copy(src_bucket, src_obj, bucket, key, upload_id, 1, src_range) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') last_modified = elem.find('LastModified').text self.assertTrue(last_modified is not None) self.assertEqual(resp_etag, etag) # Check last-modified timestamp key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('GET', bucket, key, query=query) elem = fromstring(body, 'ListPartsResult') # FIXME: COPY result drops milli/microseconds but GET doesn't last_modified_gets = [p.find('LastModified').text for p in elem.iterfind('Part')] self.assertEqual( last_modified_gets[0].rsplit('.', 1)[0], last_modified.rsplit('.', 1)[0], '%r != %r' % (last_modified_gets[0], last_modified)) # There should be *exactly* one parts in the result self.assertEqual(1, len(last_modified_gets)) # Abort Multipart Upload key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('DELETE', bucket, key, query=query) # sanity checks self.assertEqual(status, 204) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0')
def test_complete_upload_min_segment_size(self): bucket = 'bucket' key = 'obj' self.conn.make_request('PUT', bucket) query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, key, query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text # multi parts with no body etags = [] for i in xrange(1, 3): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query) etags.append(headers['etag']) xml = self._gen_comp_xml(etags) query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('POST', bucket, key, body=xml, query=query) self.assertEqual(get_error_code(body), 'EntityTooSmall') # multi parts with all parts less than min segment size etags = [] for i in xrange(1, 3): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query, body='AA') etags.append(headers['etag']) xml = self._gen_comp_xml(etags) query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('POST', bucket, key, body=xml, query=query) self.assertEqual(get_error_code(body), 'EntityTooSmall') # one part and less than min segment size etags = [] query = 'partNumber=1&uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query, body='AA') etags.append(headers['etag']) xml = self._gen_comp_xml(etags) query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('POST', bucket, key, body=xml, query=query) self.assertEqual(status, 200) # multi parts with all parts except the first part less than min # segment size query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, key, query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text etags = [] body_size = [self.min_segment_size, self.min_segment_size - 1, 2] for i in xrange(1, 3): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query, body='A' * body_size[i]) etags.append(headers['etag']) xml = self._gen_comp_xml(etags) query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('POST', bucket, key, body=xml, query=query) self.assertEqual(get_error_code(body), 'EntityTooSmall') # multi parts with all parts except last part more than min segment # size query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, key, query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text etags = [] body_size = [self.min_segment_size, self.min_segment_size, 2] for i in xrange(1, 3): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, key, query=query, body='A' * body_size[i]) etags.append(headers['etag']) xml = self._gen_comp_xml(etags) query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('POST', bucket, key, body=xml, query=query) self.assertEqual(status, 200)
def test_complete_multi_upload_error(self): bucket = 'bucket' keys = ['obj', 'obj2'] self.conn.make_request('PUT', bucket) query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, keys[0], query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text etags = [] for i in xrange(1, 3): query = 'partNumber=%s&uploadId=%s' % (i, upload_id) status, headers, body = \ self.conn.make_request('PUT', bucket, keys[0], query=query) etags.append(headers['etag']) xml = self._gen_comp_xml(etags) # part 1 too small query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('POST', bucket, keys[0], body=xml, query=query) self.assertEqual(get_error_code(body), 'EntityTooSmall') # invalid credentials auth_error_conn = Connection(aws_secret_key='invalid') status, headers, body = \ auth_error_conn.make_request('POST', bucket, keys[0], body=xml, query=query) self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') # wrong/missing bucket status, headers, body = \ self.conn.make_request('POST', 'nothing', keys[0], query=query) self.assertEqual(get_error_code(body), 'NoSuchBucket') # wrong upload ID query = 'uploadId=%s' % 'nothing' status, headers, body = \ self.conn.make_request('POST', bucket, keys[0], body=xml, query=query) self.assertEqual(get_error_code(body), 'NoSuchUpload') # without Part tag in xml query = 'uploadId=%s' % upload_id xml = self._gen_comp_xml([]) status, headers, body = \ self.conn.make_request('POST', bucket, keys[0], body=xml, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # with invalid etag in xml invalid_etag = 'invalid' xml = self._gen_comp_xml([invalid_etag]) status, headers, body = \ self.conn.make_request('POST', bucket, keys[0], body=xml, query=query) self.assertEqual(get_error_code(body), 'InvalidPart') # without part in Swift query = 'uploads' status, headers, body = \ self.conn.make_request('POST', bucket, keys[1], query=query) elem = fromstring(body, 'InitiateMultipartUploadResult') upload_id = elem.find('UploadId').text query = 'uploadId=%s' % upload_id xml = self._gen_comp_xml([etags[0]]) status, headers, body = \ self.conn.make_request('POST', bucket, keys[1], body=xml, query=query) self.assertEqual(get_error_code(body), 'InvalidPart')
def test_object_multi_upload(self): bucket = 'bucket' keys = ['obj1', 'obj2', 'obj3'] headers = [None, {'Content-MD5': base64.b64encode('a' * 16).strip()}, {'Etag': 'nonsense'}] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( bucket, keys, headers=headers) # Initiate Multipart Upload for expected_key, (status, headers, body) in \ izip(keys, results_generator): self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'InitiateMultipartUploadResult') self.assertEqual(elem.find('Bucket').text, bucket) key = elem.find('Key').text self.assertEqual(expected_key, key) upload_id = elem.find('UploadId').text self.assertTrue(upload_id is not None) self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity # List Multipart Uploads query = 'uploads' status, headers, body = \ self.conn.make_request('GET', bucket, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'ListMultipartUploadsResult') self.assertEqual(elem.find('Bucket').text, bucket) self.assertIsNone(elem.find('KeyMarker').text) self.assertEqual(elem.find('NextKeyMarker').text, uploads[-1][0]) self.assertIsNone(elem.find('UploadIdMarker').text) self.assertEqual(elem.find('NextUploadIdMarker').text, uploads[-1][1]) self.assertEqual(elem.find('MaxUploads').text, '1000') self.assertTrue(elem.find('EncodingType') is None) self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(len(elem.findall('Upload')), 3) for (expected_key, expected_upload_id), u in \ izip(uploads, elem.findall('Upload')): key = u.find('Key').text upload_id = u.find('UploadId').text self.assertEqual(expected_key, key) self.assertEqual(expected_upload_id, upload_id) self.assertEqual(u.find('Initiator/ID').text, self.conn.user_id) self.assertEqual(u.find('Initiator/DisplayName').text, self.conn.user_id) self.assertEqual(u.find('Owner/ID').text, self.conn.user_id) self.assertEqual(u.find('Owner/DisplayName').text, self.conn.user_id) self.assertEqual(u.find('StorageClass').text, 'STANDARD') self.assertTrue(u.find('Initiated').text is not None) # Upload Part key, upload_id = uploads[0] content = 'a' * self.min_segment_size etag = md5(content).hexdigest() status, headers, body = \ self._upload_part(bucket, key, upload_id, content) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers, etag) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0') expected_parts_list = [(headers['etag'], mktime(headers['date']))] # Upload Part Copy key, upload_id = uploads[1] src_bucket = 'bucket2' src_obj = 'obj3' src_content = 'b' * self.min_segment_size etag = md5(src_content).hexdigest() # prepare src obj self.conn.make_request('PUT', src_bucket) self.conn.make_request('PUT', src_bucket, src_obj, body=src_content) _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj) self.assertCommonResponseHeaders(headers) status, headers, body, resp_etag = \ self._upload_part_copy(src_bucket, src_obj, bucket, key, upload_id) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') last_modified = elem.find('LastModified').text self.assertTrue(last_modified is not None) self.assertEqual(resp_etag, etag) # Check last-modified timestamp key, upload_id = uploads[1] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('GET', bucket, key, query=query) self.assertEqual(200, status) elem = fromstring(body, 'ListPartsResult') # FIXME: COPY result drops milli/microseconds but GET doesn't last_modified_gets = [p.find('LastModified').text for p in elem.iterfind('Part')] self.assertEqual( last_modified_gets[0].rsplit('.', 1)[0], last_modified.rsplit('.', 1)[0], '%r != %r' % (last_modified_gets[0], last_modified)) # There should be *exactly* two parts in the result self.assertEqual(1, len(last_modified_gets)) # List Parts key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('GET', bucket, key, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'ListPartsResult') self.assertEqual(elem.find('Bucket').text, bucket) self.assertEqual(elem.find('Key').text, key) self.assertEqual(elem.find('UploadId').text, upload_id) self.assertEqual(elem.find('Initiator/ID').text, self.conn.user_id) self.assertEqual(elem.find('Initiator/DisplayName').text, self.conn.user_id) self.assertEqual(elem.find('Owner/ID').text, self.conn.user_id) self.assertEqual(elem.find('Owner/DisplayName').text, self.conn.user_id) self.assertEqual(elem.find('StorageClass').text, 'STANDARD') self.assertEqual(elem.find('PartNumberMarker').text, '0') self.assertEqual(elem.find('NextPartNumberMarker').text, '1') self.assertEqual(elem.find('MaxParts').text, '1000') self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(len(elem.findall('Part')), 1) # etags will be used to generate xml for Complete Multipart Upload etags = [] for (expected_etag, expected_date), p in \ izip(expected_parts_list, elem.findall('Part')): last_modified = p.find('LastModified').text self.assertTrue(last_modified is not None) # TODO: sanity check # (kota_) How do we check the sanity? # the last-modified header drops milli-seconds info # by the constraint of the format. # For now, we can do either the format check or round check # last_modified_from_xml = mktime(last_modified) # self.assertEqual(expected_date, # last_modified_from_xml) self.assertEqual(expected_etag, p.find('ETag').text) self.assertEqual(self.min_segment_size, int(p.find('Size').text)) etags.append(p.find('ETag').text) # Abort Multipart Uploads # note that uploads[1] has part data while uploads[2] does not for key, upload_id in uploads[1:]: query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('DELETE', bucket, key, query=query) self.assertEqual(status, 204) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0') # Complete Multipart Upload key, upload_id = uploads[0] xml = self._gen_comp_xml(etags) status, headers, body = \ self._complete_multi_upload(bucket, key, upload_id, xml) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'CompleteMultipartUploadResult') # TODO: use tf.config value self.assertEqual( 'http://%s:%s/bucket/obj1' % (self.conn.host, self.conn.port), elem.find('Location').text) self.assertEqual(elem.find('Bucket').text, bucket) self.assertEqual(elem.find('Key').text, key) concatted_etags = ''.join(etag.strip('"') for etag in etags) exp_etag = '"%s-%s"' % ( md5(concatted_etags.decode('hex')).hexdigest(), len(etags)) etag = elem.find('ETag').text self.assertEqual(etag, exp_etag) exp_size = self.min_segment_size * len(etags) swift_etag = '"%s"' % md5(concatted_etags).hexdigest() # TODO: GET via swift api, check against swift_etag # Check object def check_obj(req_headers, exp_status): status, headers, body = \ self.conn.make_request('HEAD', bucket, key, req_headers) self.assertEqual(status, exp_status) self.assertCommonResponseHeaders(headers) self.assertIn('content-length', headers) if exp_status == 412: self.assertNotIn('etag', headers) self.assertEqual(headers['content-length'], '0') else: self.assertIn('etag', headers) self.assertEqual(headers['etag'], exp_etag) if exp_status == 304: self.assertEqual(headers['content-length'], '0') else: self.assertEqual(headers['content-length'], str(exp_size)) check_obj({}, 200) # Sanity check conditionals check_obj({'If-Match': 'some other thing'}, 412) check_obj({'If-None-Match': 'some other thing'}, 200) # More interesting conditional cases check_obj({'If-Match': exp_etag}, 200) check_obj({'If-Match': swift_etag}, 412) check_obj({'If-None-Match': swift_etag}, 200) check_obj({'If-None-Match': exp_etag}, 304) # Check listings status, headers, body = self.conn.make_request('GET', bucket) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') resp_objects = elem.findall('./Contents') self.assertEqual(len(list(resp_objects)), 1) for o in resp_objects: self.assertEqual(o.find('Key').text, key) self.assertIsNotNone(o.find('LastModified').text) self.assertRegexpMatches( o.find('LastModified').text, r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.assertEqual(o.find('ETag').text, exp_etag) self.assertEqual(o.find('Size').text, str(exp_size)) self.assertIsNotNone(o.find('StorageClass').text is not None) self.assertEqual(o.find('Owner/ID').text, self.conn.user_id) self.assertEqual(o.find('Owner/DisplayName').text, self.conn.user_id)
def POST(self, req): """ Handles Delete Multiple Objects. """ def object_key_iter(elem): for obj in elem.iterchildren('Object'): key = obj.find('./Key').text if not key: raise UserKeyMustBeSpecified() version = obj.find('./VersionId') if version is not None: version = version.text yield key, version max_body_size = min( # FWIW, AWS limits multideletes to 1000 keys, and swift limits # object names to 1024 bytes (by default). Add a factor of two to # allow some slop. 2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH, # But, don't let operators shoot themselves in the foot 10 * 1024 * 1024) try: xml = req.xml(max_body_size) if not xml: raise MissingRequestBodyError() req.check_md5(xml) elem = fromstring(xml, 'Delete', self.logger) quiet = elem.find('./Quiet') if quiet is not None and quiet.text.lower() == 'true': self.quiet = True else: self.quiet = False delete_list = list(object_key_iter(elem)) if len(delete_list) > self.conf.max_multi_delete_objects: raise MalformedXML() except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: self.logger.error(e) raise elem = Element('DeleteResult') # check bucket existence try: req.get_response(self.app, 'HEAD') except AccessDenied as error: body = self._gen_error_body(error, elem, delete_list) return HTTPOk(body=body) if any(version is not None for _key, version in delete_list): # TODO: support deleting specific versions of objects raise S3NotImplemented() def do_delete(base_req, key, version): req = copy.copy(base_req) req.environ = copy.copy(base_req.environ) req.object_name = key try: query = req.gen_multipart_manifest_delete_query(self.app) resp = req.get_response(self.app, method='DELETE', query=query, headers={'Accept': 'application/json'}) # Have to read the response to actually do the SLO delete if query: try: delete_result = json.loads(resp.body) if delete_result['Errors']: # NB: bulk includes 404s in "Number Not Found", # not "Errors" msg_parts = [delete_result['Response Status']] msg_parts.extend( '%s: %s' % (obj, status) for obj, status in delete_result['Errors']) return key, { 'code': 'SLODeleteError', 'message': '\n'.join(msg_parts) } # else, all good except (ValueError, TypeError, KeyError): # Logs get all the gory details self.logger.exception( 'Could not parse SLO delete response: %r', resp.body) # Client gets something more generic return key, { 'code': 'SLODeleteError', 'message': 'Unexpected swift response' } except NoSuchKey: pass except ErrorResponse as e: return key, {'code': e.__class__.__name__, 'message': e._msg} return key, None with StreamingPile(self.conf.multi_delete_concurrency) as pile: for key, err in pile.asyncstarmap( do_delete, ((req, key, version) for key, version in delete_list)): if err: error = SubElement(elem, 'Error') SubElement(error, 'Key').text = key SubElement(error, 'Code').text = err['code'] SubElement(error, 'Message').text = err['message'] elif not self.quiet: deleted = SubElement(elem, 'Deleted') SubElement(deleted, 'Key').text = key body = tostring(elem) return HTTPOk(body=body)
def _get_error_message(self, body): elem = fromstring(body, 'Error') return elem.find('./Message').text
def test_object_multi_upload_part_copy_version(self): if 'object_versioning' not in tf.cluster_info: self.skipTest('Object Versioning not enabled') bucket = 'bucket' keys = ['obj1'] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( bucket, keys) # Initiate Multipart Upload for expected_key, (status, headers, body) in \ zip(keys, results_generator): self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'InitiateMultipartUploadResult') self.assertEqual(elem.find('Bucket').text, bucket) key = elem.find('Key').text self.assertEqual(expected_key, key) upload_id = elem.find('UploadId').text self.assertTrue(upload_id is not None) self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity key, upload_id = uploads[0] src_bucket = 'bucket2' src_obj = 'obj4' src_content = b'y' * (self.min_segment_size // 2) + b'z' * \ self.min_segment_size etags = [md5(src_content, usedforsecurity=False).hexdigest()] # prepare null-version src obj self.conn.make_request('PUT', src_bucket) self.conn.make_request('PUT', src_bucket, src_obj, body=src_content) _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj) self.assertCommonResponseHeaders(headers) # Turn on versioning elem = Element('VersioningConfiguration') SubElement(elem, 'Status').text = 'Enabled' xml = tostring(elem) status, headers, body = self.conn.make_request('PUT', src_bucket, body=xml, query='versioning') self.assertEqual(status, 200) src_obj2 = 'obj5' src_content2 = b'stub' etags.append(md5(src_content2, usedforsecurity=False).hexdigest()) # prepare src obj w/ real version self.conn.make_request('PUT', src_bucket, src_obj2, body=src_content2) _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj2) self.assertCommonResponseHeaders(headers) version_id2 = headers['x-amz-version-id'] status, headers, body, resp_etag = \ self._upload_part_copy(src_bucket, src_obj, bucket, key, upload_id, 1, src_version_id='null') self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') last_modifieds = [elem.find('LastModified').text] self.assertTrue(last_modifieds[0] is not None) self.assertEqual(resp_etag, etags[0]) status, headers, body, resp_etag = \ self._upload_part_copy(src_bucket, src_obj2, bucket, key, upload_id, 2, src_version_id=version_id2) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'application/xml') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], str(len(body))) self.assertTrue('etag' not in headers) elem = fromstring(body, 'CopyPartResult') last_modifieds.append(elem.find('LastModified').text) self.assertTrue(last_modifieds[1] is not None) self.assertEqual(resp_etag, etags[1]) # Check last-modified timestamp key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('GET', bucket, key, query=query) elem = fromstring(body, 'ListPartsResult') # FIXME: COPY result drops milli/microseconds but GET doesn't last_modified_gets = [ p.find('LastModified').text for p in elem.iterfind('Part') ] self.assertEqual([lm.rsplit('.', 1)[0] for lm in last_modified_gets], [lm.rsplit('.', 1)[0] for lm in last_modifieds]) # There should be *exactly* two parts in the result self.assertEqual(2, len(last_modified_gets)) # Abort Multipart Upload key, upload_id = uploads[0] query = 'uploadId=%s' % upload_id status, headers, body = \ self.conn.make_request('DELETE', bucket, key, query=query) # sanity checks self.assertEqual(status, 204) self.assertCommonResponseHeaders(headers) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0')
def test_delete_bucket_multi_upload_object_exisiting(self): bucket = 'bucket' keys = ['obj1'] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( bucket, keys) # Initiate Multipart Upload for expected_key, (status, _, body) in \ zip(keys, results_generator): self.assertEqual(status, 200) # sanity elem = fromstring(body, 'InitiateMultipartUploadResult') key = elem.find('Key').text self.assertEqual(expected_key, key) # sanity upload_id = elem.find('UploadId').text self.assertTrue(upload_id is not None) # sanity self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity # Upload Part key, upload_id = uploads[0] content = b'a' * self.min_segment_size status, headers, body = \ self._upload_part(bucket, key, upload_id, content) self.assertEqual(status, 200) # Complete Multipart Upload key, upload_id = uploads[0] etags = [md5(content, usedforsecurity=False).hexdigest()] xml = self._gen_comp_xml(etags) status, headers, body = \ self._complete_multi_upload(bucket, key, upload_id, xml) self.assertEqual(status, 200) # sanity # GET multipart object status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 200) # sanity self.assertEqual(content, body) # sanity # DELETE bucket while the object existing status, headers, body = \ self.conn.make_request('DELETE', bucket) self.assertEqual(status, 409) # sanity # The object must still be there. status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 200) # sanity self.assertEqual(content, body) # sanity # Can delete it with DeleteMultipleObjects request elem = Element('Delete') SubElement(elem, 'Quiet').text = 'true' obj_elem = SubElement(elem, 'Object') SubElement(obj_elem, 'Key').text = key body = tostring(elem, use_s3ns=False) status, headers, body = self.conn.make_request( 'POST', bucket, body=body, query='delete', headers={'Content-MD5': calculate_md5(body)}) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 404) # sanity # Now we can delete status, headers, body = \ self.conn.make_request('DELETE', bucket) self.assertEqual(status, 204) # sanity
def POST(self, req): """ Handles Complete Multipart Upload. """ upload_id = req.params['uploadId'] resp = _get_upload_info(req, self.app, upload_id) headers = {'Accept': 'application/json'} for key, val in resp.headers.items(): _key = key.lower() if _key.startswith('x-amz-meta-'): headers['x-object-meta-' + _key[11:]] = val hct_header = sysmeta_header('object', 'has-content-type') if resp.sysmeta_headers.get(hct_header) == 'yes': content_type = resp.sysmeta_headers.get( sysmeta_header('object', 'content-type')) elif hct_header in resp.sysmeta_headers: # has-content-type is present but false, so no content type was # set on initial upload. In that case, we won't set one on our # PUT request. Swift will end up guessing one based on the # object name. content_type = None else: content_type = resp.headers.get('Content-Type') if content_type: headers['Content-Type'] = content_type container = req.container_name + MULTIUPLOAD_SUFFIX s3_etag_hasher = md5() manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) if not xml: raise InvalidRequest(msg='You must specify at least one part') if 'content-md5' in req.headers: # If an MD5 was provided, we need to verify it. # Note that S3Request already took care of translating to ETag if req.headers['etag'] != md5(xml).hexdigest(): raise BadDigest(content_md5=req.headers['content-md5']) # We're only interested in the body here, in the # multipart-upload controller -- *don't* let it get # plumbed down to the object-server del req.headers['etag'] complete_elem = fromstring(xml, 'CompleteMultipartUpload', self.logger) for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = normalize_etag(part_elem.find('./ETag').text) if len(etag) != 32 or any(c not in '0123456789abcdef' for c in etag): raise InvalidPart(upload_id=upload_id, part_number=part_number) manifest.append({ 'path': '/%s/%s/%s/%d' % (container, req.object_name, upload_id, part_number), 'etag': etag }) s3_etag_hasher.update(binascii.a2b_hex(etag)) except (XMLSyntaxError, DocumentInvalid): # NB: our schema definitions catch uploads with no parts here raise MalformedXML() except ErrorResponse: raise except Exception as e: self.logger.error(e) raise s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest)) headers[sysmeta_header('object', 'etag')] = s3_etag # Leave base header value blank; SLO will populate c_etag = '; s3_etag=%s' % s3_etag headers[get_container_update_override_key('etag')] = c_etag too_small_message = ('s3api requires that each segment be at least ' '%d bytes' % self.conf.min_segment_size) def size_checker(manifest): # Check the size of each segment except the last and make sure # they are all more than the minimum upload chunk size. # Note that we need to use the *internal* keys, since we're # looking at the manifest that's about to be written. return [(item['name'], too_small_message) for item in manifest[:-1] if item and item['bytes'] < self.conf.min_segment_size] req.environ['swift.callback.slo_manifest_hook'] = size_checker start_time = time.time() def response_iter(): # NB: XML requires that the XML declaration, if present, be at the # very start of the document. Clients *will* call us out on not # being valid XML if we pass through whitespace before it. # Track whether we've sent anything yet so we can yield out that # declaration *first* yielded_anything = False try: try: # TODO: add support for versioning put_resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={ 'multipart-manifest': 'put', 'heartbeat': 'on' }, headers=headers) if put_resp.status_int == 202: body = [] put_resp.fix_conditional_response() for chunk in put_resp.response_iter: if not chunk.strip(): if time.time() - start_time < 10: # Include some grace period to keep # ceph-s3tests happy continue if not yielded_anything: yield (b'<?xml version="1.0" ' b'encoding="UTF-8"?>\n') yielded_anything = True yield chunk continue body.append(chunk) body = json.loads(b''.join(body)) if body['Response Status'] != '201 Created': for seg, err in body['Errors']: if err == too_small_message: raise EntityTooSmall() elif err in ('Etag Mismatch', '404 Not Found'): raise InvalidPart(upload_id=upload_id) raise InvalidRequest( status=body['Response Status'], msg='\n'.join(': '.join(err) for err in body['Errors'])) except BadSwiftRequest as e: msg = str(e) if too_small_message in msg: raise EntityTooSmall(msg) elif ', Etag Mismatch' in msg: raise InvalidPart(upload_id=upload_id) elif ', 404 Not Found' in msg: raise InvalidPart(upload_id=upload_id) else: raise # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) try: req.get_response(self.app, 'DELETE', container, obj) except NoSuchKey: # We know that this existed long enough for us to HEAD pass result_elem = Element('CompleteMultipartUploadResult') # NOTE: boto with sig v4 appends port to HTTP_HOST value at # the request header when the port is non default value and it # makes req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) # Why are we doing our own port parsing? Because py3 decided # to start raising ValueErrors on access after parsing such # an invalid port netloc = parsed_url.netloc.split('@')[-1].split(']')[-1] if ':' in netloc: port = netloc.split(':', 2)[1] host_url += ':%s' % port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag resp.headers.pop('ETag', None) if yielded_anything: yield b'\n' yield tostring(result_elem, xml_declaration=not yielded_anything) except ErrorResponse as err_resp: if yielded_anything: err_resp.xml_declaration = False yield b'\n' else: # Oh good, we can still change HTTP status code, too! resp.status = err_resp.status for chunk in err_resp({}, lambda *a: None): yield chunk resp = HTTPOk() # assume we're good for now... but see above! resp.app_iter = reiterate(response_iter()) resp.content_type = "application/xml" return resp
def POST(self, req): """ Handles Complete Multipart Upload. """ upload_id = req.params['uploadId'] resp = _get_upload_info(req, self.app, upload_id) headers = {} for key, val in resp.headers.iteritems(): _key = key.lower() if _key.startswith('x-amz-meta-'): headers['x-object-meta-' + _key[11:]] = val hct_header = sysmeta_header('object', 'has-content-type') if resp.sysmeta_headers.get(hct_header) == 'yes': content_type = resp.sysmeta_headers.get( sysmeta_header('object', 'content-type')) elif hct_header in resp.sysmeta_headers: # has-content-type is present but false, so no content type was # set on initial upload. In that case, we won't set one on our # PUT request. Swift will end up guessing one based on the # object name. content_type = None else: content_type = resp.headers.get('Content-Type') if content_type: headers['Content-Type'] = content_type # Query for the objects in the segments area to make sure it completed query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, 'GET', container, '', query=query) objinfo = json.loads(resp.body) objtable = dict((o['name'], { 'path': '/'.join(['', container, o['name']]), 'etag': o['hash'], 'size_bytes': o['bytes'] }) for o in objinfo) manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) if not xml: raise InvalidRequest(msg='You must specify at least one part') complete_elem = fromstring(xml, 'CompleteMultipartUpload', self.logger) for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = part_elem.find('./ETag').text if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"': # strip double quotes etag = etag[1:-1] info = objtable.get("%s/%s/%s" % (req.object_name, upload_id, part_number)) if info is None or info['etag'] != etag: raise InvalidPart(upload_id=upload_id, part_number=part_number) info['size_bytes'] = int(info['size_bytes']) manifest.append(info) except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: self.logger.error(e) raise # Check the size of each segment except the last and make sure they are # all more than the minimum upload chunk size for info in manifest[:-1]: if info['size_bytes'] < self.conf.min_segment_size: raise EntityTooSmall() try: # TODO: add support for versioning if manifest: resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={'multipart-manifest': 'put'}, headers=headers) else: # the upload must have consisted of a single zero-length part # just write it directly resp = req.get_response(self.app, 'PUT', body='', headers=headers) except BadSwiftRequest as e: msg = str(e) expected_msg = 'too small; each segment must be at least 1 byte' if expected_msg in msg: # FIXME: AWS S3 allows a smaller object than 5 MB if there is # only one part. Use a COPY request to copy the part object # from the segments container instead. raise EntityTooSmall(msg) else: raise # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) try: req.get_response(self.app, 'DELETE', container, obj) except NoSuchKey: pass # We know that this existed long enough for us to HEAD result_elem = Element('CompleteMultipartUploadResult') # NOTE: boto with sig v4 appends port to HTTP_HOST value at the # request header when the port is non default value and it makes # req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) if parsed_url.port: host_url += ':%s' % parsed_url.port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'ETag').text = resp.etag resp.body = tostring(result_elem) resp.status = 200 resp.content_type = "application/xml" return resp
def test_delete_multi_objects(self): bucket = 'bucket' put_objects = ['obj%s' % var for var in xrange(4)] self._prepare_test_delete_multi_objects(bucket, put_objects) query = 'delete' # Delete an object via MultiDelete API req_objects = ['obj0'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue(headers['content-type'] is not None) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body) resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API req_objects = ['obj1', 'obj2'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API but one (obj4) doesn't exist. req_objects = ['obj3', 'obj4'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') # S3 assumes a NoSuchKey object as deleted. self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API but no objects exist req_objects = ['obj4', 'obj5'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects)
def _check_acl(self, owner, body): elem = fromstring(body, 'AccessControlPolicy') permission = elem.find('./AccessControlList/Grant/Permission').text self.assertEqual(permission, 'FULL_CONTROL') name = elem.find('./AccessControlList/Grant/Grantee/ID').text self.assertEqual(name, owner)
def POST(self, req): """ Handles Complete Multipart Upload. """ upload_id = req.params['uploadId'] resp = _get_upload_info(req, self.app, upload_id) headers = { 'Accept': 'application/json', sysmeta_header('object', 'upload-id'): upload_id } for key, val in resp.headers.items(): _key = key.lower() if _key.startswith('x-amz-meta-'): headers['x-object-meta-' + _key[11:]] = val hct_header = sysmeta_header('object', 'has-content-type') if resp.sysmeta_headers.get(hct_header) == 'yes': content_type = resp.sysmeta_headers.get( sysmeta_header('object', 'content-type')) elif hct_header in resp.sysmeta_headers: # has-content-type is present but false, so no content type was # set on initial upload. In that case, we won't set one on our # PUT request. Swift will end up guessing one based on the # object name. content_type = None else: content_type = resp.headers.get('Content-Type') if content_type: headers['Content-Type'] = content_type container = req.container_name + MULTIUPLOAD_SUFFIX s3_etag_hasher = md5(usedforsecurity=False) manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) if not xml: raise InvalidRequest(msg='You must specify at least one part') if 'content-md5' in req.headers: # If an MD5 was provided, we need to verify it. # Note that S3Request already took care of translating to ETag if req.headers['etag'] != md5( xml, usedforsecurity=False).hexdigest(): raise BadDigest(content_md5=req.headers['content-md5']) # We're only interested in the body here, in the # multipart-upload controller -- *don't* let it get # plumbed down to the object-server del req.headers['etag'] complete_elem = fromstring(xml, 'CompleteMultipartUpload', self.logger) for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = normalize_etag(part_elem.find('./ETag').text) if len(etag) != 32 or any(c not in '0123456789abcdef' for c in etag): raise InvalidPart(upload_id=upload_id, part_number=part_number) manifest.append({ 'path': '/%s/%s/%s/%d' % (wsgi_to_str(container), wsgi_to_str( req.object_name), upload_id, part_number), 'etag': etag }) s3_etag_hasher.update(binascii.a2b_hex(etag)) except (XMLSyntaxError, DocumentInvalid): # NB: our schema definitions catch uploads with no parts here raise MalformedXML() except ErrorResponse: raise except Exception as e: self.logger.error(e) raise s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest)) s3_etag_header = sysmeta_header('object', 'etag') if resp.sysmeta_headers.get(s3_etag_header) == s3_etag: # This header should only already be present if the upload marker # has been cleaned up and the current target uses the same # upload-id; assuming the segments to use haven't changed, the work # is already done return HTTPOk(body=_make_complete_body(req, s3_etag, False), content_type='application/xml') headers[s3_etag_header] = s3_etag # Leave base header value blank; SLO will populate c_etag = '; s3_etag=%s' % s3_etag headers[get_container_update_override_key('etag')] = c_etag too_small_message = ('s3api requires that each segment be at least ' '%d bytes' % self.conf.min_segment_size) def size_checker(manifest): # Check the size of each segment except the last and make sure # they are all more than the minimum upload chunk size. # Note that we need to use the *internal* keys, since we're # looking at the manifest that's about to be written. return [(item['name'], too_small_message) for item in manifest[:-1] if item and item['bytes'] < self.conf.min_segment_size] req.environ['swift.callback.slo_manifest_hook'] = size_checker start_time = time.time() def response_iter(): # NB: XML requires that the XML declaration, if present, be at the # very start of the document. Clients *will* call us out on not # being valid XML if we pass through whitespace before it. # Track whether we've sent anything yet so we can yield out that # declaration *first* yielded_anything = False try: try: # TODO: add support for versioning put_resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={ 'multipart-manifest': 'put', 'heartbeat': 'on' }, headers=headers) if put_resp.status_int == 202: body = [] put_resp.fix_conditional_response() for chunk in put_resp.response_iter: if not chunk.strip(): if time.time() - start_time < 10: # Include some grace period to keep # ceph-s3tests happy continue if not yielded_anything: yield (b'<?xml version="1.0" ' b'encoding="UTF-8"?>\n') yielded_anything = True yield chunk continue body.append(chunk) body = json.loads(b''.join(body)) if body['Response Status'] != '201 Created': for seg, err in body['Errors']: if err == too_small_message: raise EntityTooSmall() elif err in ('Etag Mismatch', '404 Not Found'): raise InvalidPart(upload_id=upload_id) raise InvalidRequest( status=body['Response Status'], msg='\n'.join(': '.join(err) for err in body['Errors'])) except BadSwiftRequest as e: msg = str(e) if too_small_message in msg: raise EntityTooSmall(msg) elif ', Etag Mismatch' in msg: raise InvalidPart(upload_id=upload_id) elif ', 404 Not Found' in msg: raise InvalidPart(upload_id=upload_id) else: raise # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) try: req.get_response(self.app, 'DELETE', container, obj) except NoSuchKey: # The important thing is that we wrote out a tombstone to # make sure the marker got cleaned up. If it's already # gone (e.g., because of concurrent completes or a retried # complete), so much the better. pass yield _make_complete_body(req, s3_etag, yielded_anything) except ErrorResponse as err_resp: if yielded_anything: err_resp.xml_declaration = False yield b'\n' else: # Oh good, we can still change HTTP status code, too! resp.status = err_resp.status for chunk in err_resp({}, lambda *a: None): yield chunk resp = HTTPOk() # assume we're good for now... but see above! resp.app_iter = reiterate(response_iter()) resp.content_type = "application/xml" return resp
def test_object(self): obj = 'object name with %-sign' content = b'abc123' etag = md5(content).hexdigest() # PUT Object status, headers, body = \ self.conn.make_request('PUT', self.bucket, obj, body=content) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue('content-length' in headers) # sanity self.assertEqual(headers['content-length'], '0') self._assertObjectEtag(self.bucket, obj, etag) # PUT Object Copy dst_bucket = 'dst-bucket' dst_obj = 'dst_obj' self.conn.make_request('PUT', dst_bucket) headers = {'x-amz-copy-source': '/%s/%s' % (self.bucket, obj)} status, headers, body = \ self.conn.make_request('PUT', dst_bucket, dst_obj, headers=headers) self.assertEqual(status, 200) # PUT Object Copy with URL-encoded Source dst_bucket = 'dst-bucket' dst_obj = 'dst_obj' self.conn.make_request('PUT', dst_bucket) headers = {'x-amz-copy-source': quote('/%s/%s' % (self.bucket, obj))} status, headers, body = \ self.conn.make_request('PUT', dst_bucket, dst_obj, headers=headers) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body, 'CopyObjectResult') self.assertTrue(elem.find('LastModified').text is not None) last_modified_xml = elem.find('LastModified').text self.assertTrue(elem.find('ETag').text is not None) self.assertEqual(etag, elem.find('ETag').text.strip('"')) self._assertObjectEtag(dst_bucket, dst_obj, etag) # Check timestamp on Copy: status, headers, body = \ self.conn.make_request('GET', dst_bucket) self.assertEqual(status, 200) elem = fromstring(body, 'ListBucketResult') # FIXME: COPY result drops milli/microseconds but GET doesn't self.assertEqual( elem.find('Contents').find("LastModified").text.rsplit('.', 1)[0], last_modified_xml.rsplit('.', 1)[0]) # GET Object status, headers, body = \ self.conn.make_request('GET', self.bucket, obj) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers, etag) self.assertTrue(headers['last-modified'] is not None) self.assertTrue(headers['content-type'] is not None) self.assertEqual(headers['content-length'], str(len(content))) # HEAD Object status, headers, body = \ self.conn.make_request('HEAD', self.bucket, obj) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers, etag) self.assertTrue(headers['last-modified'] is not None) self.assertTrue('content-type' in headers) self.assertEqual(headers['content-length'], str(len(content))) # DELETE Object status, headers, body = \ self.conn.make_request('DELETE', self.bucket, obj) self.assertEqual(status, 204) self.assertCommonResponseHeaders(headers) # DELETE Non-Existent Object status, headers, body = \ self.conn.make_request('DELETE', self.bucket, 'does-not-exist') self.assertEqual(status, 204) self.assertCommonResponseHeaders(headers)