def test_object_multi_DELETE_quiet(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) elem = Element('Delete') SubElement(elem, 'Quiet').text = 'true' for key in ['Key1', 'Key2']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_ossns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) status, headers, body = self.call_oss2swift(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 0)
def GET(self, req): resp = req.get_response(self.app) if 'x-oss-meta-rules' in resp.headers: rules_string = resp.headers['x-oss-meta-rules'] if rules_string.startswith(','): rules_string = rules_string[1:] rule_lists = rules_string.split(',') rule_key = rule_lists[len(rule_lists) - 1] rule_id = rule_key.split(':')[0] rule_meta_name = 'x-oss-meta-' + rule_id rule = resp.headers[rule_meta_name] rule = eval(rule) elem = Element('LifecycleConfiguration') xml_rule = SubElement(elem, 'Rule') SubElement(xml_rule, 'ID').text = rule['ruleId'] SubElement(xml_rule, 'Prefix').text = rule['rulePrefix'] SubElement(xml_rule, 'Status').text = rule['ruleStatus'] expiration = SubElement(xml_rule, 'Expiration') if rule['expireDay'] != '': SubElement(expiration, 'Days').text = rule['expireDay'] else: SubElement(expiration, 'Date').text = rule['createDate'] body = tostring(elem) return HTTPOk(body=body, content_type='application/xml') else: elem = Element('Error') SubElement(elem, req.container_name) SubElement(elem, 'Code').text = 'NoSuchLifecycle' SubElement(elem, 'Message').text = 'No Row found in Lifecycle Table.' SubElement(elem, 'RequestId').text = resp.headers['x-oss-request-id'] SubElement(elem, 'HostId').text = req.headers['Host'] body = tostring(elem) return NoSuchLifecycle(req.container_name)
def test_grant_invalid_group_xml(self): grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Invalid') xml = _make_xml(grantee=grantee) req = Request.blank('/bucket/object?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header() }, body=xml) status, headers, body = self.call_oss2swift(req) self.assertEqual(self._get_error_code(body), 'MalformedACLError')
def test_grant_email_xml(self): grantee = Element('Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'AmazonCustomerByEmail') SubElement(grantee, 'EmailAddress').text = '*****@*****.**' xml = _make_xml(grantee=grantee) req = Request.blank('/bucket/object?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header() }, body=xml) status, headers, body = self.call_oss2swift(req) self.assertEqual(self._get_error_code(body), 'MalformedACLError')
def GET(self, req): """ Handles GET Bucket location. """ meta_location = dict(req.get_response(self.app, method='HEAD').headers)['x-oss-meta-location'] elem = Element('LocationConstraint') if meta_location is not None: elem.text = meta_location else: elem.text = '' body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def get_acl(account_name, headers): """ Attempts to construct an Oss ACL based on what is found in the swift headers """ elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = account_name SubElement(owner, 'DisplayName').text = account_name access_control_list = SubElement(elem, 'AccessControlList') # grant FULL_CONTROL to myself by default referrers, _ = parse_acl(headers.get('x-container-read')) if referrer_allowed('unknown', referrers): # grant public-read access SubElement(access_control_list, 'Grant').text = 'PUBLIC-READ' referrers, _ = parse_acl(headers.get('x-container-write')) if referrer_allowed('unknown', referrers): # grant public-write access SubElement(access_control_list, 'Grant').text = 'PUBLIC-READ-WRITE' body = tostring(elem) return HTTPOk(body=body, content_type="text/plain")
def _gen_comp_xml(self, etags): elem = Element('CompleteMultipartUpload') for i, etag in enumerate(etags): elem_part = SubElement(elem, 'Part') SubElement(elem_part, 'PartNumber').text = str(i + 1) SubElement(elem_part, 'ETag').text = etag return tostring(elem)
def _test_object_multi_DELETE(self, account): self.keys = ['Key1', 'Key2'] self.swift.register('DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[0], swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[1], swob.HTTPNotFound, {}, None) elem = Element('Delete') for key in self.keys: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_ossns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'OSS %s:hmac' % account, 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) req.date = datetime.now() req.content_type = 'text/plain' return self.call_oss2swift(req)
def POST(self, req): """ Handles Initiate Multipart Upload. """ upload_id = unique_id() container = req.container_name + MULTIUPLOAD_SUFFIX try: req.get_response(self.app, 'PUT', container, '') except BucketAlreadyExists: pass obj = '%s/%s' % (req.object_name, upload_id) req.get_response(self.app, 'PUT', container, obj, body='') result_elem = Element('InitiateMultipartUploadResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id body = tostring(result_elem) return HTTPOk(body=body, content_type='application/xml')
def _gen_invalid_multi_delete_xml(self, hasObjectTag=False): elem = Element('Delete') if hasObjectTag: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = '' return tostring(elem, use_ossns=False)
def GET(self, req): """ Handle GET Bucket Core (List Bucket CoreRules) request """ max_keys = req.get_validated_param('max-keys', CONF.max_corerule_listing) # TODO: Separate max_corerule_listing and default_corerule_listing max_keys = min(max_keys, CONF.max_corerule_listing) resp = req.get_response(self.app) if 'x-oss-meta-access-control-allow-origin' not in resp.headers: raise NoSuchCORSConfiguration() allowed_origins=resp.headers['x-oss-meta-access-control-allow-origin'] allowed_headers=resp.headers['x-oss-meta-access-control-allow-headers'] allowed_methods=resp.headers['x-oss-meta-access-control-allow-methods'] expose_headers=resp.headers['x-oss-meta-access-control-expose-headers'] max_age_seconds=resp.headers['x-oss-meta-access-control-max-age'] elem = Element('CORSConfiguration') rule_node =SubElement(elem, 'CORSRule') if rule_node is None and rule_node =='': raise NoSuchCORSConfiguration _add_node_list(rule_node, 'AllowedOrigin', _str_list(allowed_origins)) _add_node_list(rule_node, 'AllowedMethod', _str_list(allowed_methods)) _add_node_list(rule_node, 'AllowedHeader', _str_list(allowed_headers)) _add_node_list(rule_node, 'ExposeHeader', _str_list(expose_headers)) if max_age_seconds is not None: _add_text_child(rule_node, 'MaxAgeSeconds', str(max_age_seconds)) body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def elem(self): """ Create an etree element. """ elem = Element('AccessControlList') SubElement(elem, 'Grant').text = self.grant return elem
def _gen_multi_delete_xml(self, objects, quiet=None): elem = Element('Delete') if quiet: SubElement(elem, 'Quiet').text = quiet for key in objects: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key return tostring(elem, use_ossns=False)
def _body_iter(self): error_elem = Element('Error') SubElement(error_elem, 'Code').text = self._code SubElement(error_elem, 'Message').text = self._msg if 'swift.trans_id' in self.environ: request_id = self.environ['swift.trans_id'] SubElement(error_elem, 'RequestId').text = request_id self._dict_to_etree(error_elem, self.info) yield tostring(error_elem, use_ossns=False)
def GET(self, req): """ Handles GET Bucket logging. """ req.get_response(self.app, method='HEAD') # logging disabled elem = Element('BucketLoggingStatus') body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handles GET Bucket versioning. """ req.get_response(self.app, method='HEAD') # Just report there is no versioning configured here. elem = Element('VersioningConfiguration') body = tostring(elem) return HTTPOk(body=body, content_type="text/plain")
def GET(self, req): """ Handle GET Service request """ resp = req.get_response(self.app, query={'format': 'json'}) containers = json.loads(resp.body) containers = filter( lambda item: validate_bucket_name(item['name']), containers) # we don't keep the creation time of a bucket (osscmd doesn't # work without that) so we use something bogus. elem = Element('ListAllMyBucketsResult') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id buckets = SubElement(elem, 'Buckets') for c in containers: if CONF.oss_acl and CONF.check_bucket_owner: try: meta_headers = dict(req.get_response( self.app, 'HEAD', c['name']).headers) except AccessDenied: continue except NoSuchBucket: continue meta_headers = dict(req.get_response( self.app, 'HEAD', c['name']).headers) if meta_headers.has_key('x-oss-meta-create'): create_time = unixtime_to_iso8601(meta_headers['x-oss-meta-create']) else: create_time = unixtime_to_iso8601(0) if meta_headers.has_key('x-oss-meta-location') and \ meta_headers['x-oss-meta-location'] != '': location = meta_headers['x-oss-meta-location'] else: location = choice(CONF.location) bucket = SubElement(buckets, 'Bucket') SubElement(bucket, 'Name').text = c['name'] SubElement(bucket, 'CreationDate').text = create_time SubElement(bucket, 'Location').text = location SubElement( bucket, 'ExtranetEndpoint').text = 'oss-ostorage-'+location+'.com' SubElement( bucket, 'IntranetEndpoint').text = 'oss-ostorage-internal-'+location+'.com' body = tostring(elem) return HTTPOk(content_type='application/xml', body=body)
def _make_xml(grantee): owner = 'test:tester' permission = 'READ' elem = Element('AccessControlPolicy') elem_owner = SubElement(elem, 'Owner') SubElement(elem_owner, 'ID').text = owner SubElement(elem_owner, 'DisplayName').text = owner acl_list_elem = SubElement(elem, 'AccessControlList') elem_grant = SubElement(acl_list_elem, 'Grant') elem_grant.append(grantee) SubElement(elem_grant, 'Permission').text = permission return tostring(elem)
def elem(self): """ Decode the value to an ACL instance. """ elem = Element(self.root_tag) owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = self.owner.id SubElement(owner, 'DisplayName').text = self.owner.name grant = SubElement(elem, 'AccessControlList') SubElement(grant, 'Grant').text = str(self.grant) return elem
def test_bucket_PUT_with_location(self): elem = Element('CreateBucketConfiguration') SubElement(elem, 'LocationConstraint').text = 'US' xml = tostring(elem) req = Request.blank('/bucket', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header() }, body=xml) status, headers, body = self.call_oss2swift(req) self.assertEqual(status.split()[0], '400')
def test_object_multi_DELETE(self): self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk, {'x-static-large-object': 'True'}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key3', swob.HTTPOk, {}, None) elem = Element('Delete') for key in ['Key1', 'Key2', 'Key3']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_ossns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) req.date = datetime.now() req.content_type = 'text/plain' status, headers, body = self.call_oss2swift(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 3) _, path, _ = self.swift.calls_with_headers[-1] path, query_string = path.split('?', 1) self.assertEqual(path, '/v1/AUTH_test/bucket/Key3') query = dict(urllib.parse.parse_qsl(query_string)) self.assertEqual(query['multipart-manifest'], 'delete')
def test_object_multi_DELETE_without_md5(self): elem = Element('Delete') for key in ['Key1', 'Key2']: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_ossns=False) req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header() }, body=body) status, headers, body = self.call_oss2swift(req) self.assertEqual(self._get_error_code(body), 'InvalidRequest')
def test_object_multi_DELETE_too_many_keys(self): elem = Element('Delete') for i in range(CONF.max_multi_delete_objects + 1): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = str(i) body = tostring(elem, use_ossns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) status, headers, body = self.call_oss2swift(req) self.assertEqual(self._get_error_code(body), 'MalformedXML')
def test_object_multi_DELETE_to_object(self): elem = Element('Delete') obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = 'object' body = tostring(elem, use_ossns=False) content_md5 = md5(body).digest().encode('base64').strip() req = Request.blank('/bucket/object?delete', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header(), 'Content-MD5': content_md5 }, body=body) status, headers, body = self.call_oss2swift(req) self.assertEqual(status.split()[0], '200')
def GET(self, req): """ Handle GET Bucket Referer request """ resp = req.get_response(self.app) referers=resp.headers['X-Container-Read'] real_referers=get_oss_refer(referers) elem = Element('RefererConfiguration') SubElement(elem, 'AllowEmptyReferer').text='true' referer_list=SubElement(elem, 'RefererList') if real_referers is None: SubElement(referer_list, 'Referer').text='' body = tostring(elem) return HTTPOk(body=body, content_type='application/xml') if real_referers =='*': SubElement(referer_list, 'Referer').text='*' body = tostring(elem) return HTTPOk(body=body, content_type='application/xml') for refer in real_referers.split(','): SubElement(referer_list, 'Referer').text=refer body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handle GET Bucket website request """ resp = req.get_response(self.app) if resp.bucket_acl == 'private': raise AccessDenied() if 'x-oss-web-index' not in resp.headers: raise NoSuchWebsiteConfiguration() web_index = resp.headers['x-oss-web-index'] web_error = resp.headers['x-oss-web-error'] elem = Element('WebsiteConfiguration') index = SubElement(elem, 'IndexDocument') SubElement(index, 'Suffix').text = web_index if web_error is not None: error_doc = SubElement(elem, 'ErrorDocument') key = SubElement(error_doc, 'Key') if key is None and key == '': raise NoSuchKey() key.text = web_error body = tostring(elem) return HTTPOk(body=body, content_type='application/xml')
def test_bucket_fails_with_both_acl_header_and_xml_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = 'id' acl = SubElement(elem, 'AccessControlList') grant = SubElement(acl, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.oss.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' xml = tostring(elem) req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header(), 'X-Oss-ACL': 'public-read' }, body=xml) status, headers, body = self.call_oss2swift(req) self.assertEqual(self._get_error_code(body), 'UnexpectedContent')
def test_bucket_acl_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = 'id' acl = SubElement(elem, 'AccessControlList') grant = SubElement(acl, 'Grant') grantee = SubElement(grant, 'Grantee', nsmap={'xsi': XMLNS_XSI}) grantee.set('{%s}type' % XMLNS_XSI, 'Group') SubElement(grantee, 'URI').text = \ 'http://acs.oss.com/groups/global/AllUsers' SubElement(grant, 'Permission').text = 'READ' xml = tostring(elem) req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header() }, body=xml) status, headers, body = self.call_oss2swift(req) self.assertEqual(status.split()[0], '200') req = Request.blank('/bucket?acl', environ={ 'REQUEST_METHOD': 'PUT', 'wsgi.input': StringIO(xml) }, headers={ 'Authorization': 'OSS test:tester:hmac', 'Date': self.get_date_header(), 'Transfer-Encoding': 'chunked' }) self.assertIsNone(req.content_length) self.assertIsNone(req.message_length()) status, headers, body = self.call_oss2swift(req) self.assertEqual(status.split()[0], '200')
def elem(self): elem = Element('Owner') SubElement(elem, 'ID').text = self.id SubElement(elem, 'DisplayName').text = self.display_name return elem
def POST(self, req): """ Handles Complete Multipart Upload. """ upload_id = req.params['uploadId'] req.headers['x-object-meta-object-type'] = 'Multipart' resp = _get_upload_info(req, self.app, upload_id) headers = {} for key, val in resp.headers.iteritems(): _key = key.lower() if _key.startswith('x-oss-meta-'): headers['x-object-meta-' + _key[11:]] = val elif _key == 'content-type': headers['Content-Type'] = val # Query for the objects in the segments area to make sure it completed query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, 'GET', container, '', query=query) objinfo = json.loads(resp.body) objtable = dict((o['name'], {'path': '/'.join(['', container, o['name']]), 'etag': o['hash'], 'size_bytes': o['bytes']}) for o in objinfo) manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) complete_elem = fromstring(xml, 'CompleteMultipartUpload') for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = part_elem.find('./ETag').text if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"': # strip double quotes etag = etag[1:-1] info = objtable.get("%s/%s/%s" % (req.object_name, upload_id, part_number)) if info is None or info['etag'] != etag: raise InvalidPart(upload_id=upload_id, part_number=part_number) manifest.append(info) except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() LOGGER.error(e) raise exc_type, exc_value, exc_traceback # Following swift commit 7f636a5, zero-byte segments aren't allowed, # even as the final segment if int(info['size_bytes']) == 0: manifest.pop() # Ordinarily, we just let SLO check segment sizes. However, we # just popped off a zero-byte segment; if there was a second # zero-byte segment and it was at the end, it would succeed on # Swift < 2.6.0 and fail on newer Swift. It seems reasonable that # it should always fail. if manifest and int(manifest[-1]['size_bytes']) == 0: raise EntityTooSmall() try: # TODO: add support for versioning if manifest: resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={'multipart-manifest': 'put'}, headers=headers) else: # the upload must have consisted of a single zero-length part # just write it directly resp = req.get_response(self.app, 'PUT', body='', headers=headers) except BadSwiftRequest as e: msg = str(e) msg_pre_260 = 'Each segment, except the last, must be at least ' # see https://github.com/openstack/swift/commit/c0866ce msg_260 = ('too small; each segment, except the last, must be ' 'at least ') # see https://github.com/openstack/swift/commit/7f636a5 msg_post_260 = 'too small; each segment must be at least 1 byte' if msg.startswith(msg_pre_260) or \ msg_260 in msg or msg_post_260 in msg: # FIXME: Alibaba OSS allows a smaller object than 5 MB if there is # only one part. Use a COPY request to copy the part object # from the segments container instead. raise EntityTooSmall(msg) else: raise if int(info['size_bytes']) == 0: # clean up the zero-byte segment empty_seg_cont, empty_seg_name = info['path'].split('/', 2)[1:] req.get_response(self.app, 'DELETE', container=empty_seg_cont, obj=empty_seg_name) # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) req.get_response(self.app, 'DELETE', container, obj) result_elem = Element('CompleteMultipartUploadResult') # NOTE: boto with sig v4 appends port to HTTP_HOST value at the # request header when the port is non default value and it makes # req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) if parsed_url.port: host_url += ':%s' % parsed_url.port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'ETag').text = resp.etag resp.body = tostring(result_elem) resp.status = 200 resp.content_type = "application/xml" return resp