def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = get_account_name_and_placement(req)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
reverse = config_true_value(get_param(req, 'reverse'))
limit = constrain_req_limit(req, constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, "prefix")
delimiter = get_param(req, "delimiter")
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body="Bad delimiter")
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, "limit")
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req, body="Maximum limit is %d" % constraints.ACCOUNT_LISTING_LIMIT
)
marker = get_param(req, "marker", "")
end_marker = get_param(req, "end_marker")
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(
account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter
)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter)
def get_account_listing_resp(self, req):
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'prefix')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker')
end_marker = get_param(req, 'end_marker')
try:
listing, info = self.app.storage.container_list(
self.account_name, limit=limit, marker=marker,
end_marker=end_marker, prefix=prefix,
delimiter=delimiter)
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req),
info=info,
listing=listing)
except exceptions.NotFound:
if self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
else:
resp = HTTPNotFound(request=req)
return resp
def get_account_listing_resp(self, req):
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'prefix')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker')
end_marker = get_param(req, 'end_marker')
s3_buckets_only = False
oio_headers = {REQID_HEADER: self.trans_id}
info = None
if req.environ.get('swift.source') == 'S3':
if hasattr(self.app.storage.account, 'bucket_list'):
# Call directly AccountClient.bucket_list()
info = self.app.storage.account.bucket_list(
self.account_name,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
headers=oio_headers)
listing = info.pop('listing')
return account_listing_bucket_response(
self.account_name,
req,
get_listing_content_type(req),
listing=listing)
# Fallback: use legacy method to list buckets
s3_buckets_only = True
delimiter = '%' # first character of encoded delimiter of CH
info = self.app.storage.account.container_list(
self.account_name,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
headers=oio_headers,
s3_buckets_only=s3_buckets_only)
listing = info.pop('listing')
return account_listing_response(self.account_name,
req,
get_listing_content_type(req),
info=info,
listing=listing,
s3_buckets_only=s3_buckets_only)
def get_account_listing_resp(self, req):
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'prefix')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker')
end_marker = get_param(req, 'end_marker')
oio_headers = {'X-oio-req-id': self.trans_id}
try:
info = None
if hasattr(self.app.storage, 'account'):
# Call directly AccountClient.container_list()
# because storage.container_list() does not return
# account metadata
info = self.app.storage.account.container_list(
self.account_name,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
headers=oio_headers)
listing = info.pop('listing')
else:
# Legacy call to account service
listing, info = self.app.storage.container_list(
self.account_name,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
delimiter=delimiter,
headers=oio_headers)
resp = account_listing_response(self.account_name,
req,
get_listing_content_type(req),
info=info,
listing=listing)
except (exceptions.NotFound, exceptions.NoSuchAccount):
if self.app.account_autocreate:
resp = account_listing_response(self.account_name, req,
get_listing_content_type(req))
else:
resp = HTTPNotFound(request=req)
return resp
def _filter_resp_shard_ranges(self, req, cached_ranges):
# filter returned shard ranges according to request constraints
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
includes = get_param(req, 'includes')
reverse = config_true_value(get_param(req, 'reverse'))
if reverse:
marker, end_marker = end_marker, marker
shard_ranges = [
ShardRange.from_dict(shard_range) for shard_range in cached_ranges
]
shard_ranges = filter_shard_ranges(shard_ranges, includes, marker,
end_marker)
if reverse:
shard_ranges.reverse()
return json.dumps([dict(sr) for sr in shard_ranges]).encode('ascii')
def get_account_listing_resp(self, req):
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'prefix')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker')
end_marker = get_param(req, 'end_marker')
s3_buckets_only = False
if req.environ.get('swift.source') == 'S3':
s3_buckets_only = True
delimiter = CH.ENCODED_DELIMITER[0]
oio_headers = {REQID_HEADER: self.trans_id}
info = None
if hasattr(self.app.storage, 'account'):
# Call directly AccountClient.container_list()
# because storage.container_list() does not return
# account metadata
info = self.app.storage.account.container_list(
self.account_name, limit=limit, marker=marker,
end_marker=end_marker, prefix=prefix,
delimiter=delimiter, headers=oio_headers,
s3_buckets_only=s3_buckets_only)
listing = info.pop('listing')
else:
# Legacy call to account service
listing, info = self.app.storage.container_list(
self.account_name, limit=limit, marker=marker,
end_marker=end_marker, prefix=prefix,
delimiter=delimiter, headers=oio_headers)
return account_listing_response(
self.account_name, req, get_listing_content_type(req),
info=info, listing=listing, s3_buckets_only=s3_buckets_only)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account = split_and_validate_path(req, 3)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter, reverse)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
return self.create_listing(req, out_content_type, info,
broker.metadata, container_list, container)
def create_listing(self, req, out_content_type, info, metadata,
container_list, container):
list_meta = get_param(req, 'list_meta', 'f').lower() in TRUE_VALUES
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in metadata.iteritems():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record, list_meta)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj, list_meta)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
if list_meta and field == 'metadata':
meta = SubElement(obj_element, field)
for k, v in record[field].iteritems():
SubElement(meta, k).text = str(
v.decode('utf-8'))
else:
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
return ret
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
# import pydevd
# pydevd.settrace('172.29.132.122', port=5678, stdoutToServer=True, stderrToServer=True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
container_list = broker.list_objects_iter(
limit,
marker,
end_marker,
prefix,
delimiter,
path,
storage_policy_index=info['storage_policy_index'],
reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
def account_listing_content_type(req):
"""
Figure out the content type of an account-listing response.
Returns a 2-tuple: (content_type, error). Only one of them will be set;
the other will be None.
"""
query_format = get_param(req, 'format')
if query_format:
req.accept = FORMAT2CONTENT_TYPE.get(query_format.lower(),
FORMAT2CONTENT_TYPE['plain'])
content_type = req.accept.best_match(
['text/plain', 'application/json', 'application/xml', 'text/xml'])
if not content_type:
return (None, HTTPNotAcceptable(request=req))
else:
return (content_type, None)
def GET(self, req):
"""Handle HTTP GET request."""
# 从原始请求中获取相关信息
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
# 如果设置的对象数量限制超过系统定义的阈值，则报错412
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
# 返回一个container broker实例，用于代理其数据块操作
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
# info = SELECT account, container, created_at, put_timestamp,
# delete_timestamp, status_changed_at,
# object_count, bytes_used,
# reported_put_timestamp, reported_delete_timestamp,
# reported_object_count, reported_bytes_used, hash,
# id, %s, %s
# FROM container_stat
info, is_deleted = broker.get_info_is_deleted()
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
# 如果container元数据不存在，则报错404
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
# 获取objects的排序列表
container_list = broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'])
# 根据json和xml的不同创建列表
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
def GET(self, req):
"""
Handle HTTP GET request.
This method is exact copy of swift.container.server.GET() except
that this version of it passes 'out_content_type' information to
broker.list_objects_iter()
"""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
container_list = broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'],
out_content_type=out_content_type, reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
def get_container_list_resp(self, req):
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if path is not None:
prefix = path
if path:
prefix = path.rstrip('/') + '/'
delimiter = '/'
opts = req.environ.get('oio.query', {})
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
result = self.app.storage.object_list(
self.account_name,
self.container_name,
prefix=prefix,
limit=limit,
delimiter=delimiter,
marker=marker,
end_marker=end_marker,
properties=True,
versions=opts.get('versions', False),
deleted=opts.get('deleted', False),
force_master=opts.get('force_master', False),
headers=oio_headers,
cache=oio_cache,
perfdata=perfdata)
resp_headers = self.get_metadata_resp_headers(result)
resp = self.create_listing(req, out_content_type, resp_headers, result,
self.container_name, **opts)
return resp
def get_listing_content_type(req):
"""
Determine the content type to use for an account or container listing
response.
:param req: request object
:returns: content type as a string (e.g. text/plain, application/json)
:raises HTTPNotAcceptable: if the requested content type is not acceptable
:raises HTTPBadRequest: if the 'format' query param is provided and
not valid UTF-8
"""
query_format = get_param(req, 'format')
if query_format:
req.accept = FORMAT2CONTENT_TYPE.get(
query_format.lower(), FORMAT2CONTENT_TYPE['plain'])
out_content_type = req.accept.best_match(
['text/plain', 'application/json', 'application/xml', 'text/xml'])
if not out_content_type:
raise HTTPNotAcceptable(request=req)
return out_content_type
def get_listing_content_type(req):
"""
Determine the content type to use for an account or container listing
response.
:param req: request object
:returns: content type as a string (e.g. text/plain, application/json)
:raises HTTPNotAcceptable: if the requested content type is not acceptable
:raises HTTPBadRequest: if the 'format' query param is provided and
not valid UTF-8
"""
query_format = get_param(req, 'format')
if query_format:
req.accept = FORMAT2CONTENT_TYPE.get(
query_format.lower(), FORMAT2CONTENT_TYPE['plain'])
try:
out_content_type = req.accept.best_match(
['text/plain', 'application/json', 'application/xml', 'text/xml'])
except ValueError:
raise HTTPBadRequest(request=req, body='Invalid Accept header')
if not out_content_type:
raise HTTPNotAcceptable(request=req)
return out_content_type
def get_container_list_resp(self, req):
storage = self.app.storage
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
if path is not None:
prefix = path
if path:
prefix = path.rstrip('/') + '/'
delimiter = '/'
out_content_type = get_listing_content_type(req)
try:
result_list = storage. \
object_list(self.account_name, self.container_name,
prefix=prefix, limit=limit, delimiter=delimiter,
marker=marker, end_marker=end_marker)
# TODO get container info
info = {}
resp_headers = gen_resp_headers(info)
resp = self.create_listing(req, out_content_type, resp_headers,
info, result_list,
self.container_name)
except exceptions.NoSuchContainer:
return HTTPNotFound(request=req)
return resp
def get_container_list_resp(self, req):
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if path is not None:
prefix = path
if path:
prefix = path.rstrip('/') + '/'
delimiter = '/'
opts = req.environ.get('oio.query', {})
oio_headers = {'X-oio-req-id': self.trans_id}
result = self.app.storage.object_list(
self.account_name, self.container_name, prefix=prefix,
limit=limit, delimiter=delimiter, marker=marker,
end_marker=end_marker, properties=True,
versions=opts.get('versions', False),
deleted=opts.get('deleted', False),
headers=oio_headers)
resp_headers = self.get_metadata_resp_headers(result)
resp = self.create_listing(
req, out_content_type, resp_headers, result,
self.container_name, **opts)
return resp
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
return self.create_listing(req, out_content_type, info,
broker.metadata, container_list, container)
else:
content_type += ';%s=%s' % (key, value)
response['content_type'] = content_type
return response
@public
@timing_stats()
def GET(self, req):
"""Handle HTTP GET request."""
try:
drive, part, account, container, obj = req.split_path(4, 5, True)
validate_device_partition(drive, part)
except ValueError, err:
return HTTPBadRequest(body=str(err), content_type='text/plain',
request=req)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' % CONTAINER_LISTING_LIMIT)
def GET(self, req):
"""
Handle HTTP GET request.
The body of the response to a successful GET request contains a listing
of either objects or shard ranges. The exact content of the listing is
determined by a combination of request headers and query string
parameters, as follows:
* The type of the listing is determined by the
``X-Backend-Record-Type`` header. If this header has value ``shard``
then the response body will be a list of shard ranges; if this header
has value ``auto``, and the container state is ``sharding`` or
``sharded``, then the listing will be a list of shard ranges;
otherwise the response body will be a list of objects.
* Both shard range and object listings may be constrained to a name
range by the ``marker`` and ``end_marker`` query string parameters.
Object listings will only contain objects whose names are greater
than any ``marker`` value and less than any ``end_marker`` value.
Shard range listings will only contain shard ranges whose namespace
is greater than or includes any ``marker`` value and is less than or
includes any ``end_marker`` value.
* Shard range listings may also be constrained by an ``includes`` query
string parameter. If this parameter is present the listing will only
contain shard ranges whose namespace includes the value of the
parameter; any ``marker`` or ``end_marker`` parameters are ignored
* The length of an object listing may be constrained by the ``limit``
parameter. Object listings may also be constrained by ``prefix``,
``delimiter`` and ``path`` query string parameters.
* Shard range listings will include deleted shard ranges if and only if
the ``X-Backend-Include-Deleted`` header value is one of
:attr:`swift.common.utils.TRUE_VALUES`. Object listings never
include deleted objects.
* Shard range listings may be constrained to include only shard ranges
whose state is specified by a query string ``states`` parameter. If
present, the ``states`` parameter should be a comma separated list of
either the string or integer representation of
:data:`~swift.common.utils.ShardRange.STATES`.
Two alias values may be used in a ``states`` parameter value:
``listing`` will cause the listing to include all shard ranges in a
state suitable for contributing to an object listing; ``updating``
will cause the listing to include all shard ranges in a state
suitable to accept an object update.
If either of these aliases is used then the shard range listing will
if necessary be extended with a synthesised 'filler' range in order
to satisfy the requested name range when insufficient actual shard
ranges are found. Any 'filler' shard range will cover the otherwise
uncovered tail of the requested name range and will point back to the
same container.
* Listings are not normally returned from a deleted container. However,
the ``X-Backend-Override-Deleted`` header may be used with a value in
:attr:`swift.common.utils.TRUE_VALUES` to force a shard range
listing to be returned from a deleted container whose DB file still
exists.
:param req: an instance of :class:`swift.common.swob.Request`
:returns: an instance of :class:`swift.common.swob.Response`
"""
drive, part, account, container, obj = get_obj_name_and_placement(req)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.CONTAINER_LISTING_LIMIT)
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
record_type = req.headers.get('x-backend-record-type', '').lower()
if record_type == 'auto' and info.get('db_state') in (SHARDING,
SHARDED):
record_type = 'shard'
if record_type == 'shard':
override_deleted = info and config_true_value(
req.headers.get('x-backend-override-deleted', False))
resp_headers = gen_resp_headers(info,
is_deleted=is_deleted
and not override_deleted)
if is_deleted and not override_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'shard'
includes = get_param(req, 'includes')
states = get_param(req, 'states')
fill_gaps = False
if states:
states = list_from_csv(states)
fill_gaps = any(('listing' in states, 'updating' in states))
try:
states = broker.resolve_shard_range_states(states)
except ValueError:
return HTTPBadRequest(request=req, body='Bad state')
include_deleted = config_true_value(
req.headers.get('x-backend-include-deleted', False))
container_list = broker.get_shard_ranges(
marker,
end_marker,
includes,
reverse,
states=states,
include_deleted=include_deleted,
fill_gaps=fill_gaps)
else:
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'object'
# Use the retired db while container is in process of sharding,
# otherwise use current db
src_broker = broker.get_brokers()[0]
container_list = src_broker.list_objects_iter(
limit,
marker,
end_marker,
prefix,
delimiter,
path,
storage_policy_index=info['storage_policy_index'],
reverse=reverse,
allow_reserved=req.allow_reserved_names)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(request=req,
body='Maximum limit is %d' %
CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers
or is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req,
headers=resp_headers,
content_type=out_content_type,
charset='utf-8')
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
ret.body = json.dumps(
[self.update_data_record(record) for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in [
"name", "hash", "bytes", "content_type",
"last_modified"
]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
return ret
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
@public
@timing_stats(sample_rate=0.1)
def HEAD(self, req):
"""Handle HTTP HEAD request."""
try:
drive, part, account, container, obj = req.split_path(4, 5, True)
validate_device_partition(drive, part)
except ValueError, err:
return HTTPBadRequest(body=str(err),
content_type='text/plain',
request=req)
query_format = get_param(req, 'format')
if query_format:
req.accept = FORMAT2CONTENT_TYPE.get(query_format.lower(),
FORMAT2CONTENT_TYPE['plain'])
out_content_type = req.accept.best_match(
['text/plain', 'application/json', 'application/xml', 'text/xml'])
if not out_content_type:
return HTTPNotAcceptable(request=req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
ai = self.account_info(self.account_name, req)
auto_account = self.account_name.startswith(
self.app.auto_create_account_prefix)
if not (auto_account or ai[1]):
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
# Don't cache this. The lack of account will be cached, and that
# is sufficient.
return HTTPNotFound(request=req)
# The read-modify-write of params here is because the Request.params
# getter dynamically generates a dict of params from the query string;
# the setter must be called for new params to update the query string.
params = req.params
params['format'] = 'json'
# x-backend-record-type may be sent via internal client e.g. from
# the sharder or in probe tests
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
if not record_type:
record_type = 'auto'
req.headers['X-Backend-Record-Type'] = 'auto'
params['states'] = 'listing'
req.params = params
memcache = cache_from_env(req.environ, True)
if (req.method == 'GET' and record_type != 'object'
and self.app.recheck_listing_shard_ranges > 0 and memcache
and get_param(req, 'states') == 'listing'
and not config_true_value(
req.headers.get('x-backend-include-deleted', False))):
# This GET might be served from cache or might populate cache.
# 'x-backend-include-deleted' is not usually expected in requests
# to the proxy (it is used from sharder to container servers) but
# it is included in the conditions just in case because we don't
# cache deleted shard ranges.
resp = self._GET_using_cache(req)
else:
resp = self._GETorHEAD_from_backend(req)
resp_record_type = resp.headers.get('X-Backend-Record-Type', '')
if all((req.method == "GET", record_type == 'auto',
resp_record_type.lower() == 'shard')):
resp = self._get_from_shards(req, resp)
if not config_true_value(resp.headers.get('X-Backend-Cached-Results')):
# Cache container metadata. We just made a request to a storage
# node and got up-to-date information for the container.
resp.headers['X-Backend-Recheck-Container-Existence'] = str(
self.app.recheck_container_existence)
set_info_cache(self.app, req.environ, self.account_name,
self.container_name, resp)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
if aresp:
# Don't cache this. It doesn't reflect the state of the
# container, just that the user can't access it.
return aresp
if not req.environ.get('swift_owner', False):
for key in self.app.swift_owner_headers:
if key in resp.headers:
del resp.headers[key]
# Expose sharding state in reseller requests
if req.environ.get('reseller_request', False):
resp.headers['X-Container-Sharding'] = config_true_value(
resp.headers.get(
get_sys_meta_prefix('container') + 'Sharding', 'False'))
return resp
def GET(self, req):
"""
Handle HTTP GET request.
The body of the response to a successful GET request contains a listing
of either objects or shard ranges. The exact content of the listing is
determined by a combination of request headers and query string
parameters, as follows:
* The type of the listing is determined by the
``X-Backend-Record-Type`` header. If this header has value ``shard``
then the response body will be a list of shard ranges; if this header
has value ``auto``, and the container state is ``sharding`` or
``sharded``, then the listing will be a list of shard ranges;
otherwise the response body will be a list of objects.
* Both shard range and object listings may be constrained to a name
range by the ``marker`` and ``end_marker`` query string parameters.
Object listings will only contain objects whose names are greater
than any ``marker`` value and less than any ``end_marker`` value.
Shard range listings will only contain shard ranges whose namespace
is greater than or includes any ``marker`` value and is less than or
includes any ``end_marker`` value.
* Shard range listings may also be constrained by an ``includes`` query
string parameter. If this parameter is present the listing will only
contain shard ranges whose namespace includes the value of the
parameter; any ``marker`` or ``end_marker`` parameters are ignored
* The length of an object listing may be constrained by the ``limit``
parameter. Object listings may also be constrained by ``prefix``,
``delimiter`` and ``path`` query string parameters.
* Shard range listings will include deleted shard ranges if and only if
the ``X-Backend-Include-Deleted`` header value is one of
:attr:`swift.common.utils.TRUE_VALUES`. Object listings never
include deleted objects.
* Shard range listings may be constrained to include only shard ranges
whose state is specified by a query string ``states`` parameter. If
present, the ``states`` parameter should be a comma separated list of
either the string or integer representation of
:data:`~swift.common.utils.ShardRange.STATES`.
Two alias values may be used in a ``states`` parameter value:
``listing`` will cause the listing to include all shard ranges in a
state suitable for contributing to an object listing; ``updating``
will cause the listing to include all shard ranges in a state
suitable to accept an object update.
If either of these aliases is used then the shard range listing will
if necessary be extended with a synthesised 'filler' range in order
to satisfy the requested name range when insufficient actual shard
ranges are found. Any 'filler' shard range will cover the otherwise
uncovered tail of the requested name range and will point back to the
same container.
* Listings are not normally returned from a deleted container. However,
the ``X-Backend-Override-Deleted`` header may be used with a value in
:attr:`swift.common.utils.TRUE_VALUES` to force a shard range
listing to be returned from a deleted container whose DB file still
exists.
:param req: an instance of :class:`swift.common.swob.Request`
:returns: an instance of :class:`swift.common.swob.Response`
"""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
record_type = req.headers.get('x-backend-record-type', '').lower()
if record_type == 'auto' and info.get('db_state') in (SHARDING,
SHARDED):
record_type = 'shard'
if record_type == 'shard':
override_deleted = info and config_true_value(
req.headers.get('x-backend-override-deleted', False))
resp_headers = gen_resp_headers(
info, is_deleted=is_deleted and not override_deleted)
if is_deleted and not override_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'shard'
includes = get_param(req, 'includes')
states = get_param(req, 'states')
fill_gaps = False
if states:
states = list_from_csv(states)
fill_gaps = any(('listing' in states, 'updating' in states))
try:
states = broker.resolve_shard_range_states(states)
except ValueError:
return HTTPBadRequest(request=req, body='Bad state')
include_deleted = config_true_value(
req.headers.get('x-backend-include-deleted', False))
container_list = broker.get_shard_ranges(
marker, end_marker, includes, reverse, states=states,
include_deleted=include_deleted, fill_gaps=fill_gaps)
else:
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'object'
# Use the retired db while container is in process of sharding,
# otherwise use current db
src_broker = broker.get_brokers()[0]
container_list = src_broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'],
reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
for key, (value, timestamp) in
broker.metadata.iteritems() if value != '')
headers['Content-Type'] = out_content_type
return HTTPNoContent(request=req, headers=headers, charset='utf-8')
@public
@timing_stats()
def GET(self, req):
"""Handle HTTP GET request."""
try:
drive, part, account = req.split_path(3)
validate_device_partition(drive, part)
except ValueError, err:
return HTTPBadRequest(body=str(err), content_type='text/plain',
request=req)
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
limit = ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > ACCOUNT_LISTING_LIMIT:
return HTTPPreconditionFailed(request=req,
body='Maximum limit is %d' %
ACCOUNT_LISTING_LIMIT)
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
out_content_type = get_listing_content_type(req)
return resp
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
@public
@timing_stats(sample_rate=0.1)
def HEAD(self, req):
"""Handle HTTP HEAD request."""
try:
drive, part, account, container, obj = req.split_path(4, 5, True)
validate_device_partition(drive, part)
except ValueError, err:
return HTTPBadRequest(body=str(err), content_type="text/plain", request=req)
query_format = get_param(req, "format")
if query_format:
req.accept = FORMAT2CONTENT_TYPE.get(query_format.lower(), FORMAT2CONTENT_TYPE["plain"])
out_content_type = req.accept.best_match(["text/plain", "application/json", "application/xml", "text/xml"])
if not out_content_type:
return HTTPNotAcceptable(request=req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
headers = {
"X-Container-Object-Count": info["object_count"],
"X-Container-Bytes-Used": info["bytes_used"],
"X-Timestamp": info["created_at"],
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d' %
constraints.CONTAINER_LISTING_LIMIT)
out_content_type = listing_formats.get_listing_content_type(req)
if not check_drive(self.root, drive, self.mount_check):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
record_type = req.headers.get('x-backend-record-type', '').lower()
if record_type == 'auto' and info.get('db_state') in (SHARDING,
SHARDED):
record_type = 'shard'
if record_type == 'shard':
override_deleted = info and config_true_value(
req.headers.get('x-backend-override-deleted', False))
resp_headers = gen_resp_headers(info,
is_deleted=is_deleted
and not override_deleted)
if is_deleted and not override_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'shard'
includes = get_param(req, 'includes')
states = get_param(req, 'states')
fill_gaps = False
if states:
states = list_from_csv(states)
fill_gaps = any(('listing' in states, 'updating' in states))
try:
states = broker.resolve_shard_range_states(states)
except ValueError:
return HTTPBadRequest(request=req, body='Bad state')
include_deleted = config_true_value(
req.headers.get('x-backend-include-deleted', False))
container_list = broker.get_shard_ranges(
marker,
end_marker,
includes,
reverse,
states=states,
include_deleted=include_deleted,
fill_gaps=fill_gaps)
else:
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
resp_headers['X-Backend-Record-Type'] = 'object'
# Use the retired db while container is in process of sharding,
# otherwise use current db
src_broker = broker.get_brokers()[0]
container_list = src_broker.list_objects_iter(
limit,
marker,
end_marker,
prefix,
delimiter,
path,
storage_policy_index=info['storage_policy_index'],
reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
return ret
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
@public
@timing_stats()
def HEAD(self, req):
"""Handle HTTP HEAD request."""
try:
drive, part, account = req.split_path(3)
validate_device_partition(drive, part)
except ValueError, err:
return HTTPBadRequest(body=str(err), content_type='text/plain',
request=req)
query_format = get_param(req, 'format')
if query_format:
req.accept = FORMAT2CONTENT_TYPE.get(
query_format.lower(), FORMAT2CONTENT_TYPE['plain'])
out_content_type = req.accept.best_match(
['text/plain', 'application/json', 'application/xml', 'text/xml'])
if not out_content_type:
return HTTPNotAcceptable(request=req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account,
stale_reads_ok=True)
broker.pending_timeout = 0.1
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
info = broker.get_info()
