def test_account_listing_reserved_names(self):
broker = backend.AccountBroker(':memory:', account='a')
put_timestamp = next(self.ts)
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(put_timestamp.internal)
container_timestamp = next(self.ts)
broker.put_container(get_reserved_name('foo'),
container_timestamp.internal, 0, 10, 100, 0)
req = Request.blank('')
resp = utils.account_listing_response(
'a', req, 'application/json', broker)
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 2,
'X-Account-Container-Count': 1,
'X-Account-Object-Count': 10,
'X-Account-Bytes-Used': 100,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'[]', resp.body)
req = Request.blank('', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = utils.account_listing_response(
'a', req, 'application/json', broker)
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 97,
'X-Account-Container-Count': 1,
'X-Account-Object-Count': 10,
'X-Account-Bytes-Used': 100,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
expected = [{
"last_modified": container_timestamp.isoformat,
"count": 10,
"bytes": 100,
"name": get_reserved_name('foo'),
}]
self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
sorted(resp.body))
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name),
constraints.MAX_ACCOUNT_NAME_LENGTH)
return resp
partition = self.app.account_ring.get_part(self.account_name)
concurrency = self.app.account_ring.replica_count \
if self.app.concurrent_gets else 1
node_iter = self.app.iter_nodes(self.app.account_ring, partition)
resp = self.GETorHEAD_base(
req, _('Account'), node_iter, partition,
req.swift_entity_path.rstrip('/'), concurrency)
if resp.status_int == HTTP_NOT_FOUND:
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
if req.environ.get('swift_owner'):
self.add_acls_from_sys_metadata(resp)
else:
for header in self.app.swift_owner_headers:
resp.headers.pop(header, None)
return resp
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name),
constraints.MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(req, _('Account'),
self.app.account_ring, partition,
req.swift_entity_path.rstrip('/'))
if resp.status_int == HTTP_NOT_FOUND:
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
if req.environ.get('swift_owner'):
self.add_acls_from_sys_metadata(resp)
else:
for header in self.app.swift_owner_headers:
resp.headers.pop(header, None)
return resp
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = get_account_name_and_placement(req)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
reverse = config_true_value(get_param(req, 'reverse'))
limit = constrain_req_limit(req, constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, "prefix")
delimiter = get_param(req, "delimiter")
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body="Bad delimiter")
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, "limit")
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req, body="Maximum limit is %d" % constraints.ACCOUNT_LISTING_LIMIT
)
marker = get_param(req, "marker", "")
end_marker = get_param(req, "end_marker")
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(
account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter
)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
print 'in GETorHEAD function of accountcontroller class'
if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name),
constraints.MAX_ACCOUNT_NAME_LENGTH)
return resp
partition = self.app.account_ring.get_part(self.account_name)
print 'partition',partition
node_iter = self.app.iter_nodes(self.app.account_ring, partition)
print 'node_iter',node_iter
resp = self.GETorHEAD_base(
req, _('Account'), node_iter, partition,
req.swift_entity_path.rstrip('/'))
print 'resp',resp
if resp.status_int == HTTP_NOT_FOUND:
print 'resp.status_int == HTTP_NOT_FOUND'
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
if req.environ.get('swift_owner'):
print 'req.environ.get(swift_owner), true'
self.add_acls_from_sys_metadata(resp)
else:
for header in self.app.swift_owner_headers:
print 'header',header
resp.headers.pop(header, None)
print 'resp',resp
print 'in GETorHEAD function of accountcontroller class end'
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > constraints.MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name),
constraints.MAX_ACCOUNT_NAME_LENGTH)
# Don't cache this. We know the account doesn't exist because
# the name is bad; we don't need to cache that because it's
# really cheap to recompute.
return resp
partition = self.app.account_ring.get_part(self.account_name)
concurrency = self.app.account_ring.replica_count \
if self.app.concurrent_gets else 1
node_iter = self.app.iter_nodes(self.app.account_ring, partition)
params = req.params
params['format'] = 'json'
req.params = params
resp = self.GETorHEAD_base(req, _('Account'), node_iter, partition,
req.swift_entity_path.rstrip('/'),
concurrency)
if resp.status_int == HTTP_NOT_FOUND:
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
# This is kind of a lie; we pretend like the account is
# there, but it's not. We'll create it as soon as something
# tries to write to it, but we don't need databases on disk
# to tell us that nothing's there.
#
# We set a header so that certain consumers can tell it's a
# fake listing. The important one is the PUT of a container
# to an autocreate account; the proxy checks to see if the
# account exists before actually performing the PUT and
# creates the account if necessary. If we feed it a perfect
# lie, it'll just try to create the container without
# creating the account, and that'll fail.
req.params = {} # clear our format override
resp = account_listing_response(
self.account_name, req,
listing_formats.get_listing_content_type(req))
resp.headers['X-Backend-Fake-Account-Listing'] = 'yes'
# Cache this. We just made a request to a storage node and got
# up-to-date information for the account.
resp.headers['X-Backend-Recheck-Account-Existence'] = str(
self.app.recheck_account_existence)
set_info_cache(self.app, req.environ, self.account_name, None, resp)
if req.environ.get('swift_owner'):
self.add_acls_from_sys_metadata(resp)
else:
for header in self.app.swift_owner_headers:
resp.headers.pop(header, None)
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
length_limit = self.get_name_length_limit()
if len(self.account_name) > length_limit:
resp = HTTPBadRequest(request=req)
resp.body = b'Account name length of %d longer than %d' % \
(len(self.account_name), length_limit)
# Don't cache this. We know the account doesn't exist because
# the name is bad; we don't need to cache that because it's
# really cheap to recompute.
return resp
partition = self.app.account_ring.get_part(self.account_name)
concurrency = self.app.account_ring.replica_count \
if self.app.concurrent_gets else 1
node_iter = self.app.iter_nodes(self.app.account_ring, partition)
params = req.params
params['format'] = 'json'
req.params = params
resp = self.GETorHEAD_base(
req, _('Account'), node_iter, partition,
req.swift_entity_path.rstrip('/'), concurrency)
if resp.status_int == HTTP_NOT_FOUND:
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
# This is kind of a lie; we pretend like the account is
# there, but it's not. We'll create it as soon as something
# tries to write to it, but we don't need databases on disk
# to tell us that nothing's there.
#
# We set a header so that certain consumers can tell it's a
# fake listing. The important one is the PUT of a container
# to an autocreate account; the proxy checks to see if the
# account exists before actually performing the PUT and
# creates the account if necessary. If we feed it a perfect
# lie, it'll just try to create the container without
# creating the account, and that'll fail.
resp = account_listing_response(
self.account_name, req,
listing_formats.get_listing_content_type(req))
resp.headers['X-Backend-Fake-Account-Listing'] = 'yes'
# Cache this. We just made a request to a storage node and got
# up-to-date information for the account.
resp.headers['X-Backend-Recheck-Account-Existence'] = str(
self.app.recheck_account_existence)
set_info_cache(self.app, req.environ, self.account_name, None, resp)
if req.environ.get('swift_owner'):
self.add_acls_from_sys_metadata(resp)
else:
for header in self.app.swift_owner_headers:
resp.headers.pop(header, None)
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = "Account name length of %d longer than %d" % (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(req, _("Account"), self.app.account_ring, partition, req.path_info.rstrip("/"))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
resp = account_listing_response(self.account_name, req, get_listing_content_type(req))
if not req.environ.get("swift_owner", False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
return resp
def test_account_listing_response(self):
req = Request.blank('')
now = time.time()
with mock.patch('time.time', new=lambda: now):
resp = utils.account_listing_response('a', req, 'text/plain')
self.assertEqual(resp.status_int, 204)
expected = HeaderKeyDict({
'Content-Type': 'text/plain; charset=utf-8',
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'', resp.body)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(req, _('Account'), self.app.account_ring,
partition, req.path_info.rstrip('/'))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
content_type, error = account_listing_content_type(req)
if error:
return error
return account_listing_response(self.account_name, req,
content_type)
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(
req, _('Account'), self.app.account_ring, partition,
req.path_info.rstrip('/'))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
content_type, error = account_listing_content_type(req)
if error:
return error
return account_listing_response(self.account_name, req,
content_type)
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(req, _('Account'), self.app.account_ring,
partition, req.path_info.rstrip('/'))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = "Account name length of %d longer than %d" % (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(
req, _("Account"), self.app.account_ring, partition, req.swift_entity_path.rstrip("/")
)
if resp.status_int == HTTP_NOT_FOUND:
if resp.headers.get("X-Account-Status", "").lower() == "deleted":
resp.status = HTTP_GONE
elif self.app.account_autocreate:
resp = account_listing_response(self.account_name, req, get_listing_content_type(req))
if req.environ.get("swift_owner"):
self.add_acls_from_sys_metadata(resp)
else:
for header in self.app.swift_owner_headers:
resp.headers.pop(header, None)
return resp
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
resp = HTTPBadRequest(request=req)
resp.body = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return resp
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
resp = self.GETorHEAD_base(
req, _('Account'), self.app.account_ring, partition,
req.swift_entity_path.rstrip('/'))
if resp.status_int == HTTP_NOT_FOUND:
if resp.headers.get('X-Account-Status', '').lower() == 'deleted':
resp.status = HTTP_GONE
elif self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
return resp
return HTTPPreconditionFailed(request=req,
body='Maximum limit is %d' %
ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter)
@public
@replication
@timing_stats()
def REPLICATE(self, req):
"""
Handle HTTP REPLICATE request.
Handler for RPC calls for account replication.
"""
try:
post_args = req.split_path(3)
drive, partition, hash = post_args
validate_device_partition(drive, partition)
except ValueError, err:
ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type, error = account_listing_content_type(req)
if error:
return error
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
stale_reads_ok=True)
broker.pending_timeout = 0.1
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter)
@public
@replication
@timing_stats()
def REPLICATE(self, req):
"""
Handle HTTP REPLICATE request.
Handler for RPC calls for account replication.
"""
try:
post_args = req.split_path(3)
drive, partition, hash = post_args
validate_device_partition(drive, partition)
except ValueError, err: