def GET(self, req): """ Handles GET Bucket versioning. """ req.get_response(self.app, method='HEAD') # Just report there is no versioning configured here. elem = Element('VersioningConfiguration') body = tostring(elem) return HTTPOk(body=body, content_type="text/plain")
def GET(self, req): """ Handles GET Bucket logging. """ req.get_response(self.app, method='HEAD') # logging disabled elem = Element('BucketLoggingStatus') body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def test_grant_email_xml(self): grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'AmazonCustomerByEmail') SubElement(grantee, 'EmailAddress').text = '*****@*****.**' xml = _make_xml(grantee=grantee) req = Request.blank('/bucket/object?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(self._get_error_code(body), 'NotImplemented')
def test_bucket_PUT_with_location(self): elem = Element('CreateBucketConfiguration') SubElement(elem, 'LocationConstraint').text = 'US' xml = tostring(elem) req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200')
def test_grant_invalid_group_xml(self): grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Invalid') xml = _make_xml(grantee=grantee) req = Request.blank('/bucket/object?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(self._get_error_code(body), 'MalformedACLError')
def _body_iter(self): error_elem = Element('Error') SubElement(error_elem, 'Code').text = self._code SubElement(error_elem, 'Message').text = self._msg if 'swift.trans_id' in self.environ: request_id = self.environ['swift.trans_id'] SubElement(error_elem, 'RequestId').text = request_id self._dict_to_etree(error_elem, self.info) yield tostring(error_elem, use_s3ns=False)
def _test_bucket_PUT_with_location(self, root_element): elem = Element(root_element) SubElement(elem, 'LocationConstraint').text = 'US' xml = tostring(elem) req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}, body=xml) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '200')
def GET(self, req): """ Handles GET Bucket location. """ req.get_response(self.app, method='HEAD') elem = Element('LocationConstraint') if CONF.location != 'US': elem.text = CONF.location body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def _make_xml(grantee): owner = 'test:tester' permission = 'READ' elem = Element('AccessControlPolicy') elem_owner = SubElement(elem, 'Owner') SubElement(elem_owner, 'ID').text = owner SubElement(elem_owner, 'DisplayName').text = owner acl_list_elem = SubElement(elem, 'AccessControlList') elem_grant = SubElement(acl_list_elem, 'Grant') elem_grant.append(grantee) SubElement(elem_grant, 'Permission').text = permission return tostring(elem)
def test_grant_invalid_uri_xml(self): grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = 'invalid' xml = _make_xml(grantee) req = Request.blank('/bucket/object?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_versioning_put_error(self): # Root tag is not VersioningConfiguration elem = Element('foo') SubElement(elem, 'Status').text = 'Enabled' xml = tostring(elem) status, headers, body = self.conn.make_request('PUT', 'bucket', body=xml, query='versioning') self.assertEqual(status, 400) self.assertEqual(get_error_code(body), 'MalformedXML') # Status is not "Enabled" or "Suspended" elem = Element('VersioningConfiguration') SubElement(elem, 'Status').text = '...' xml = tostring(elem) status, headers, body = self.conn.make_request('PUT', 'bucket', body=xml, query='versioning') self.assertEqual(status, 400) self.assertEqual(get_error_code(body), 'MalformedXML')
def test_object_multi_DELETE_without_md5(self): elem = Element('Delete') for key in ['Key1', 'Key2']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=body) status, headers, body = self.call_swift3(req) self.assertEquals(self._get_error_code(body), 'InvalidRequest')
def test_bucket_PUT_with_location_error(self): elem = Element('CreateBucketConfiguration') SubElement(elem, 'LocationConstraint').text = 'XXX' xml = tostring(elem) req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}, body=xml) status, headers, body = self.call_swift3(req) self.assertEqual(self._get_error_code(body), 'InvalidLocationConstraint')
def elem(self): """ Decode the value to an ACL instance. """ elem = Element(self.root_tag) owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = self.owner.id SubElement(owner, 'DisplayName').text = self.owner.name SubElement(elem, 'AccessControlList').extend(g.elem() for g in self.grants) return elem
def convert_urlquery_to_xml(val): """Convert x-amz-tagging to a Tagging XML.""" root = Element('Tagging') elem = SubElement(root, 'TagSet') # AWS support key1=&key2= items = parse_qs(val, keep_blank_values=True) for key, val in items.items(): if len(val) != 1: raise InvalidArgument(HTTP_HEADER_TAGGING_KEY, value=val, msg=INVALID_TAGGING) tag = SubElement(elem, 'Tag') SubElement(tag, 'Key').text = key SubElement(tag, 'Value').text = val[0] return tostring(root)
def _handle_create_multipart_upload(self, s3_key, req): req.headers.setdefault('content-type', 'application/octet-stream') boto3_resp = self.local_to_me_provider._create_multipart_upload( req.headers, s3_key) # NOTE: we forced swift3 to fully delegate to us, so we are # responsible for returning a valid S3 API response. result_elem = Element('InitiateMultipartUploadResult') SubElement(result_elem, 'Bucket').text = self.container_name SubElement(result_elem, 'Key').text = self.object_name SubElement(result_elem, 'UploadId').text = boto3_resp['UploadId'] body = tostring(result_elem) # Note: swift3 mw requires obj POST to return 202 return swob.HTTPAccepted(body=body, content_type='application/xml')
def test_object_multi_DELETE_to_object(self): elem = Element('Delete') obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = 'object' body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket/object?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_swift3(req) self.assertEqual(status.split()[0], '200')
def test_object_multi_DELETE_too_many_keys(self): elem = Element('Delete') for i in range(CONF.max_multi_delete_objects + 1): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = str(i) body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_swift3(req) self.assertEqual(self._get_error_code(body), 'MalformedXML')
def _handle_complete_multipart_upload(self, s3_key, req): upload_id = req.params['uploadId'] parts = [] xml = req.environ[CC_SWIFT_REQ_KEY].xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) complete_elem = fromstring(xml, 'CompleteMultipartUpload') for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) etag = part_elem.find('./ETag').text parts.append({'ETag': etag, 'PartNumber': part_number}) boto3_resp = self.local_to_me_provider._complete_multipart_upload( s3_key, upload_id, parts) # NOTE: we forced swift3 to fully delegate to us, so we are # responsible for returning a valid S3 API response. # NOTE (for the note): this workaround for a client library (boto) was # copied verbatim from the "swift3" codebase. It may or may not be # relevant (I think it is for any client talking to cloud-connector # using boto and a non-default port number), but this comment and the # workaround both come from "swift3" and should be safe for us as well. # (sic) vvvvvvvvvvvvvvvvvvvvvvvvvv # NOTE: boto with sig v4 appends port to HTTP_HOST value at the # request header when the port is non default value and it # makes req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 # (sic) ^^^^^^^^^^^^^^^^^^^^^^^^^^ parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) if parsed_url.port: host_url += ':%s' % parsed_url.port result_elem = Element('CompleteMultipartUploadResult') SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = self.container_name SubElement(result_elem, 'Key').text = self.object_name SubElement(result_elem, 'ETag').text = boto3_resp['ETag'] body = tostring(result_elem) # Note: swift3 mw requires obj POST to return 202 return swob.HTTPAccepted(body=body, content_type='application/xml')
def GET(self, req): """ Handle GET Service request """ log_s3api_command(req, 'list-buckets') resp = req.get_response(self.app, query={'format': 'json'}) containers = json.loads(resp.body) containers = filter( lambda item: validate_bucket_name(item['name']), containers) # we don't keep the creation time of a bucket (s3cmd doesn't # work without that) so we use something bogus. elem = Element('ListAllMyBucketsResult') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id buckets = SubElement(elem, 'Buckets') for c in containers: if 'last_modified' in c: ts = last_modified_date_to_timestamp(c['last_modified']) creation_date = S3Timestamp(ts).s3xmlformat else: creation_date = '2009-02-03T16:45:09.000Z' if CONF.s3_acl and CONF.check_bucket_owner: try: cname = c['name'].encode('utf-8') c_resp = req.get_response(self.app, 'HEAD', cname) if 'X-Timestamp' in c_resp.sw_headers: creation_date = S3Timestamp( c_resp.sw_headers['X-Timestamp']).s3xmlformat except AccessDenied: continue except NoSuchBucket: continue bucket = SubElement(buckets, 'Bucket') SubElement(bucket, 'Name').text = c['name'] SubElement(bucket, 'CreationDate').text = creation_date body = tostring(elem) return HTTPOk(content_type='application/xml', body=body)
def test_bucket_acl_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = 'id' acl = SubElement(elem, 'AccessControlList') grant = SubElement(acl, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.amazonaws.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' xml = tostring(elem) req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200')
def get_acl(account_name, headers): """ Attempts to construct an S3 ACL based on what is found in the swift headers """ elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = account_name SubElement(owner, 'DisplayName').text = account_name access_control_list = SubElement(elem, 'AccessControlList') # grant FULL_CONTROL to myself by default grant = SubElement(access_control_list, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'CanonicalUser') SubElement(grantee, 'ID').text = account_name SubElement(grantee, 'DisplayName').text = account_name SubElement(grant, 'Permission').text = 'FULL_CONTROL' referrers, _ = parse_acl(headers.get('x-container-read')) if referrer_allowed('unknown', referrers): # grant public-read access grant = SubElement(access_control_list, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.amazonaws.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' referrers, _ = parse_acl(headers.get('x-container-write')) if referrer_allowed('unknown', referrers): # grant public-write access grant = SubElement(access_control_list, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.amazonaws.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'WRITE' body = tostring(elem) return HTTPOk(body=body, content_type="text/plain")
def PUT(self, req): """ Handle PUT Object and PUT Object (Copy) request """ if CONF.s3_acl: if 'X-Amz-Copy-Source' in req.headers: src_path = req.headers['X-Amz-Copy-Source'] src_path = src_path if src_path.startswith('/') else \ ('/' + src_path) src_bucket, src_obj = split_path(src_path, 0, 2, True) req.get_response(self.app, 'HEAD', src_bucket, src_obj, permission='READ') b_resp = req.get_response(self.app, 'HEAD', obj='') # To avoid overwriting the existing object by unauthorized user, # we send HEAD request first before writing the object to make # sure that the target object does not exist or the user that sent # the PUT request have write permission. try: req.get_response(self.app, 'HEAD') except NoSuchKey: pass req_acl = ACL.from_headers(req.headers, b_resp.bucket_acl.owner, Owner(req.user_id, req.user_id)) req.object_acl = req_acl resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: elem = Element('CopyObjectResult') SubElement(elem, 'ETag').text = '"%s"' % resp.etag body = tostring(elem, use_s3ns=False) return HTTPOk(body=body, headers=resp.headers) resp.status = HTTP_OK return resp
def test_bucket_fails_with_both_acl_header_and_xml_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = 'id' acl = SubElement(elem, 'AccessControlList') grant = SubElement(acl, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.amazonaws.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' xml = tostring(elem) req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'X-AMZ-ACL': 'public-read'}, body=xml) status, headers, body = self.call_swift3(req) self.assertEqual(self._get_error_code(body), 'UnexpectedContent')
def test_object_multi_DELETE_no_key(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) elem = Element('Delete') SubElement(elem, 'Quiet').text = 'true' for key in ['Key1', 'Key2']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key') body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_swift3(req) self.assertEqual(self._get_error_code(body), 'UserKeyMustBeSpecified')
def POST(self, req): """ Handles Initiate Multipart Upload. """ # Create a unique S3 upload id from UUID to avoid duplicates. upload_id = unique_id() container = req.container_name + MULTIUPLOAD_SUFFIX content_type = req.headers.get('Content-Type') if content_type: req.headers[sysmeta_header('object', 'has-content-type')] = 'yes' req.headers[ sysmeta_header('object', 'content-type')] = content_type else: req.headers[sysmeta_header('object', 'has-content-type')] = 'no' req.headers['Content-Type'] = 'application/directory' try: req.get_response(self.app, 'PUT', container, '') except BucketAlreadyExists: pass obj = '%s/%s' % (req.object_name, upload_id) req.headers.pop('Etag', None) req.headers.pop('Content-Md5', None) req.get_response(self.app, 'PUT', container, obj, body='') result_elem = Element('InitiateMultipartUploadResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id body = tostring(result_elem) return HTTPOk(body=body, content_type='application/xml')
def test_bucket_acl_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = 'id' acl = SubElement(elem, 'AccessControlList') grant = SubElement(acl, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.amazonaws.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' xml = tostring(elem) req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header() }, body=xml) status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200') req = Request.blank('/bucket?acl', environ={ 'REQUEST_METHOD': 'PUT', 'wsgi.input': StringIO(xml) }, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Transfer-Encoding': 'chunked' }) self.assertIsNone(req.content_length) self.assertIsNone(req.message_length()) status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200')
def test_object_multi_DELETE(self): self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk, {'x-static-large-object': 'True'}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk, {}, None) elem = Element('Delete') for key in ['Key1', 'Key2', 'Key3']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) req.date = datetime.now() req.content_type = 'text/plain' status, headers, body = self.call_swift3(req) self.assertEquals(status.split()[0], '200') elem = fromstring(body) self.assertEquals(len(elem.findall('Deleted')), 3) _, path, _ = self.swift.calls_with_headers[-1] path, query_string = path.split('?', 1) self.assertEquals(path, '/v1/AUTH_test/bucket/Key3') query = dict(urllib.parse.parse_qsl(query_string)) self.assertEquals(query['multipart-manifest'], 'delete')
def GET(self, req): """ Handle GET Service request """ resp = req.get_response(self.app, query={'format': 'json'}) containers = json.loads(resp.body) containers = filter(lambda item: validate_bucket_name(item['name']), containers) # we don't keep the creation time of a backet (s3cmd doesn't # work without that) so we use something bogus. elem = Element('ListAllMyBucketsResult') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id buckets = SubElement(elem, 'Buckets') for c in containers: if CONF.s3_acl and CONF.check_bucket_owner: try: req.get_response(self.app, 'HEAD', c['name']) except AccessDenied: continue except NoSuchBucket: continue bucket = SubElement(buckets, 'Bucket') SubElement(bucket, 'Name').text = c['name'] SubElement(bucket, 'CreationDate').text = \ '2009-02-03T16:45:09.000Z' body = tostring(elem) return HTTPOk(content_type='application/xml', body=body)
def _test_object_multi_DELETE(self, account): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) elem = Element('Delete') for key in ['Key1', 'Key2']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'AWS %s:hmac' % account, 'Content-MD5': content_md5 }, body=body) req.date = datetime.now() req.content_type = 'text/plain' return self.call_swift3(req)