def test_account_listing_reserved_names(self): broker = backend.AccountBroker(':memory:', account='a') put_timestamp = next(self.ts) now = time.time() with mock.patch('time.time', new=lambda: now): broker.initialize(put_timestamp.internal) container_timestamp = next(self.ts) broker.put_container(get_reserved_name('foo'), container_timestamp.internal, 0, 10, 100, 0) req = Request.blank('') resp = utils.account_listing_response( 'a', req, 'application/json', broker) self.assertEqual(resp.status_int, 200) expected = HeaderKeyDict({ 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': 2, 'X-Account-Container-Count': 1, 'X-Account-Object-Count': 10, 'X-Account-Bytes-Used': 100, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': put_timestamp.normal, 'X-Account-Storage-Policy-Zero-Container-Count': 1, 'X-Account-Storage-Policy-Zero-Object-Count': 10, 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, }) self.assertEqual(expected, resp.headers) self.assertEqual(b'[]', resp.body) req = Request.blank('', headers={ 'X-Backend-Allow-Reserved-Names': 'true'}) resp = utils.account_listing_response( 'a', req, 'application/json', broker) self.assertEqual(resp.status_int, 200) expected = HeaderKeyDict({ 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': 97, 'X-Account-Container-Count': 1, 'X-Account-Object-Count': 10, 'X-Account-Bytes-Used': 100, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': put_timestamp.normal, 'X-Account-Storage-Policy-Zero-Container-Count': 1, 'X-Account-Storage-Policy-Zero-Object-Count': 10, 'X-Account-Storage-Policy-Zero-Bytes-Used': 100, }) self.assertEqual(expected, resp.headers) expected = [{ "last_modified": container_timestamp.isoformat, "count": 10, "bytes": 100, "name": get_reserved_name('foo'), }] self.assertEqual(sorted(json.dumps(expected).encode('ascii')), sorted(resp.body))
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), constraints.MAX_ACCOUNT_NAME_LENGTH) return resp partition = self.app.account_ring.get_part(self.account_name) concurrency = self.app.account_ring.replica_count \ if self.app.concurrent_gets else 1 node_iter = self.app.iter_nodes(self.app.account_ring, partition) resp = self.GETorHEAD_base( req, _('Account'), node_iter, partition, req.swift_entity_path.rstrip('/'), concurrency) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if req.environ.get('swift_owner'): self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: resp.headers.pop(header, None) return resp
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter)
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), constraints.MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base(req, _('Account'), self.app.account_ring, partition, req.swift_entity_path.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if req.environ.get('swift_owner'): self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: resp.headers.pop(header, None) return resp
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = get_account_name_and_placement(req) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') reverse = config_true_value(get_param(req, 'reverse')) limit = constrain_req_limit(req, constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, "prefix") delimiter = get_param(req, "delimiter") if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body="Bad delimiter") limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, "limit") if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body="Maximum limit is %d" % constraints.ACCOUNT_LISTING_LIMIT ) marker = get_param(req, "marker", "") end_marker = get_param(req, "end_marker") out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response( account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter )
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') limit = constraints.ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') reverse = config_true_value(get_param(req, 'reverse')) if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" print 'in GETorHEAD function of accountcontroller class' if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), constraints.MAX_ACCOUNT_NAME_LENGTH) return resp partition = self.app.account_ring.get_part(self.account_name) print 'partition',partition node_iter = self.app.iter_nodes(self.app.account_ring, partition) print 'node_iter',node_iter resp = self.GETorHEAD_base( req, _('Account'), node_iter, partition, req.swift_entity_path.rstrip('/')) print 'resp',resp if resp.status_int == HTTP_NOT_FOUND: print 'resp.status_int == HTTP_NOT_FOUND' if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if req.environ.get('swift_owner'): print 'req.environ.get(swift_owner), true' self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: print 'header',header resp.headers.pop(header, None) print 'resp',resp print 'in GETorHEAD function of accountcontroller class end' return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), constraints.MAX_ACCOUNT_NAME_LENGTH) # Don't cache this. We know the account doesn't exist because # the name is bad; we don't need to cache that because it's # really cheap to recompute. return resp partition = self.app.account_ring.get_part(self.account_name) concurrency = self.app.account_ring.replica_count \ if self.app.concurrent_gets else 1 node_iter = self.app.iter_nodes(self.app.account_ring, partition) params = req.params params['format'] = 'json' req.params = params resp = self.GETorHEAD_base(req, _('Account'), node_iter, partition, req.swift_entity_path.rstrip('/'), concurrency) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: # This is kind of a lie; we pretend like the account is # there, but it's not. We'll create it as soon as something # tries to write to it, but we don't need databases on disk # to tell us that nothing's there. # # We set a header so that certain consumers can tell it's a # fake listing. The important one is the PUT of a container # to an autocreate account; the proxy checks to see if the # account exists before actually performing the PUT and # creates the account if necessary. If we feed it a perfect # lie, it'll just try to create the container without # creating the account, and that'll fail. req.params = {} # clear our format override resp = account_listing_response( self.account_name, req, listing_formats.get_listing_content_type(req)) resp.headers['X-Backend-Fake-Account-Listing'] = 'yes' # Cache this. We just made a request to a storage node and got # up-to-date information for the account. resp.headers['X-Backend-Recheck-Account-Existence'] = str( self.app.recheck_account_existence) set_info_cache(self.app, req.environ, self.account_name, None, resp) if req.environ.get('swift_owner'): self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: resp.headers.pop(header, None) return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" length_limit = self.get_name_length_limit() if len(self.account_name) > length_limit: resp = HTTPBadRequest(request=req) resp.body = b'Account name length of %d longer than %d' % \ (len(self.account_name), length_limit) # Don't cache this. We know the account doesn't exist because # the name is bad; we don't need to cache that because it's # really cheap to recompute. return resp partition = self.app.account_ring.get_part(self.account_name) concurrency = self.app.account_ring.replica_count \ if self.app.concurrent_gets else 1 node_iter = self.app.iter_nodes(self.app.account_ring, partition) params = req.params params['format'] = 'json' req.params = params resp = self.GETorHEAD_base( req, _('Account'), node_iter, partition, req.swift_entity_path.rstrip('/'), concurrency) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: # This is kind of a lie; we pretend like the account is # there, but it's not. We'll create it as soon as something # tries to write to it, but we don't need databases on disk # to tell us that nothing's there. # # We set a header so that certain consumers can tell it's a # fake listing. The important one is the PUT of a container # to an autocreate account; the proxy checks to see if the # account exists before actually performing the PUT and # creates the account if necessary. If we feed it a perfect # lie, it'll just try to create the container without # creating the account, and that'll fail. resp = account_listing_response( self.account_name, req, listing_formats.get_listing_content_type(req)) resp.headers['X-Backend-Fake-Account-Listing'] = 'yes' # Cache this. We just made a request to a storage node and got # up-to-date information for the account. resp.headers['X-Backend-Recheck-Account-Existence'] = str( self.app.recheck_account_existence) set_info_cache(self.app, req.environ, self.account_name, None, resp) if req.environ.get('swift_owner'): self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: resp.headers.pop(header, None) return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = "Account name length of %d longer than %d" % (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base(req, _("Account"), self.app.account_ring, partition, req.path_info.rstrip("/")) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if not req.environ.get("swift_owner", False): for key in self.app.swift_owner_headers: if key in resp.headers: del resp.headers[key] return resp
def test_account_listing_response(self): req = Request.blank('') now = time.time() with mock.patch('time.time', new=lambda: now): resp = utils.account_listing_response('a', req, 'text/plain') self.assertEqual(resp.status_int, 204) expected = HeaderKeyDict({ 'Content-Type': 'text/plain; charset=utf-8', 'X-Account-Container-Count': 0, 'X-Account-Object-Count': 0, 'X-Account-Bytes-Used': 0, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': Timestamp(now).normal, }) self.assertEqual(expected, resp.headers) self.assertEqual(b'', resp.body)
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base(req, _('Account'), self.app.account_ring, partition, req.path_info.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: content_type, error = account_listing_content_type(req) if error: return error return account_listing_response(self.account_name, req, content_type) return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base( req, _('Account'), self.app.account_ring, partition, req.path_info.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: content_type, error = account_listing_content_type(req) if error: return error return account_listing_response(self.account_name, req, content_type) return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base(req, _('Account'), self.app.account_ring, partition, req.path_info.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if not req.environ.get('swift_owner', False): for key in self.app.swift_owner_headers: if key in resp.headers: del resp.headers[key] return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = "Account name length of %d longer than %d" % (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base( req, _("Account"), self.app.account_ring, partition, req.swift_entity_path.rstrip("/") ) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get("X-Account-Status", "").lower() == "deleted": resp.status = HTTP_GONE elif self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if req.environ.get("swift_owner"): self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: resp.headers.pop(header, None) return resp
def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base( req, _('Account'), self.app.account_ring, partition, req.swift_entity_path.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if not req.environ.get('swift_owner', False): for key in self.app.swift_owner_headers: if key in resp.headers: del resp.headers[key] return resp
return HTTPPreconditionFailed(request=req, body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter) @public @replication @timing_stats() def REPLICATE(self, req): """ Handle HTTP REPLICATE request. Handler for RPC calls for account replication. """ try: post_args = req.split_path(3) drive, partition, hash = post_args validate_device_partition(drive, partition) except ValueError, err:
ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type, error = account_listing_content_type(req) if error: return error if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, stale_reads_ok=True) broker.pending_timeout = 0.1 if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter) @public @replication @timing_stats() def REPLICATE(self, req): """ Handle HTTP REPLICATE request. Handler for RPC calls for account replication. """ try: post_args = req.split_path(3) drive, partition, hash = post_args validate_device_partition(drive, partition) except ValueError, err: