def HEAD(self, req):
"""Handle HTTP HEAD request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=headers)
headers.update(
(str_to_wsgi(key), str_to_wsgi(value))
for key, (value, timestamp) in broker.metadata.items()
if value != '' and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)))
headers['Content-Type'] = out_content_type
resp = HTTPNoContent(request=req, headers=headers, charset='utf-8')
resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp']))
return resp
def _process_delete(self,
resp,
pile,
obj_name,
resp_dict,
failed_files,
failed_file_response,
retry=0):
if resp.status_int // 100 == 2:
resp_dict['Number Deleted'] += 1
elif resp.status_int == HTTP_NOT_FOUND:
resp_dict['Number Not Found'] += 1
elif resp.status_int == HTTP_UNAUTHORIZED:
failed_files.append(
[wsgi_quote(str_to_wsgi(obj_name)),
HTTPUnauthorized().status])
elif resp.status_int == HTTP_CONFLICT and pile and \
self.retry_count > 0 and self.retry_count > retry:
retry += 1
sleep(self.retry_interval**retry)
delete_obj_req = Request.blank(resp.environ['PATH_INFO'],
resp.environ)
def _retry(req, app, obj_name, retry):
return req.get_response(app), obj_name, retry
pile.spawn(_retry, delete_obj_req, self.app, obj_name, retry)
else:
if resp.status_int // 100 == 5:
failed_file_response['type'] = HTTPBadGateway
failed_files.append(
[wsgi_quote(str_to_wsgi(obj_name)), resp.status])
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest(
req.environ,
path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
if req.method == 'HEAD':
con_resp.body = b''
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(b''.join(con_resp.app_iter))
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest(
req.environ,
path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
if req.method == 'HEAD':
con_resp.body = b''
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(b''.join(con_resp.app_iter))
def HEAD(self, req):
"""Handle HTTP HEAD request."""
drive, part, account, container, obj = get_obj_name_and_placement(req)
out_content_type = listing_formats.get_listing_content_type(req)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
info, is_deleted = broker.get_info_is_deleted()
headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=headers)
headers.update(
(str_to_wsgi(key), str_to_wsgi(value))
for key, (value, timestamp) in broker.metadata.items()
if value != '' and (key.lower() in self.save_headers
or is_sys_or_user_meta('container', key)))
headers['Content-Type'] = out_content_type
resp = HTTPNoContent(request=req, headers=headers, charset='utf-8')
resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp']))
return resp
def create_listing(self, req, out_content_type, info, resp_headers,
metadata, container_list, container):
for key, (value, _timestamp) in metadata.items():
if value and (key.lower() in self.save_headers
or is_sys_or_user_meta('container', key)):
resp_headers[str_to_wsgi(key)] = str_to_wsgi(value)
listing = [
self.update_data_record(record) for record in container_list
]
if out_content_type.endswith('/xml'):
body = listing_formats.container_to_xml(listing, container)
elif out_content_type.endswith('/json'):
body = json.dumps(listing).encode('ascii')
else:
body = listing_formats.listing_to_text(listing)
ret = Response(request=req,
headers=resp_headers,
body=body,
content_type=out_content_type,
charset='utf-8')
ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
if not ret.body:
ret.status_int = HTTP_NO_CONTENT
return ret
def delete_filter(predicate, objs_to_delete):
for obj_to_delete in objs_to_delete:
obj_name = obj_to_delete['name']
if not obj_name:
continue
if not predicate(obj_name):
continue
if obj_to_delete.get('error'):
if obj_to_delete['error']['code'] == HTTP_NOT_FOUND:
resp_dict['Number Not Found'] += 1
else:
failed_files.append([
wsgi_quote(str_to_wsgi(obj_name)),
obj_to_delete['error']['message']
])
continue
delete_path = '/'.join(
['', vrs, account,
obj_name.lstrip('/')])
if not constraints.check_utf8(delete_path):
failed_files.append([
wsgi_quote(str_to_wsgi(obj_name)),
HTTPPreconditionFailed().status
])
continue
yield (obj_name, delete_path,
obj_to_delete.get('version_id'))
def make_headers(self, hdrs, cfg=None):
if cfg is None:
cfg = {}
headers = {}
if not cfg.get('no_auth_token'):
headers['X-Auth-Token'] = self.storage_token
if cfg.get('use_token'):
headers['X-Auth-Token'] = cfg.get('use_token')
if isinstance(hdrs, dict):
headers.update(
(str_to_wsgi(h), str_to_wsgi(v)) for h, v in hdrs.items())
return headers
def make_headers(self, hdrs, cfg=None):
if cfg is None:
cfg = {}
headers = {}
if not cfg.get('no_auth_token'):
headers['X-Auth-Token'] = self.storage_token
if cfg.get('use_token'):
headers['X-Auth-Token'] = cfg.get('use_token')
if isinstance(hdrs, dict):
headers.update((str_to_wsgi(h), str_to_wsgi(v))
for h, v in hdrs.items())
return headers
def _copy_object(self, app, destination):
req = Request.blank(str_to_wsgi(self.object_path), method='COPY',
headers={'Destination': destination})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
return resp
def _post_object(self, app):
req = Request.blank(str_to_wsgi(self.object_path), method='POST',
headers={'Content-Type': 'application/test',
'X-Object-Meta-Fruit': 'Kiwi'})
resp = req.get_response(app)
self.assertEqual('202 Accepted', resp.status)
return resp
def _copy_object(self, app, destination):
req = Request.blank(str_to_wsgi(self.object_path), method='COPY',
headers={'Destination': destination})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
return resp
def _post_object(self, app):
req = Request.blank(str_to_wsgi(self.object_path), method='POST',
headers={'Content-Type': 'application/test',
'X-Object-Meta-Fruit': 'Kiwi'})
resp = req.get_response(app)
self.assertEqual('202 Accepted', resp.status)
return resp
def delete_actual_object(self, actual_obj, timestamp, is_async_delete):
"""
Deletes the end-user object indicated by the actual object name given
'<account>/<container>/<object>' if and only if the X-Delete-At value
of the object is exactly the timestamp given.
:param actual_obj: The name of the end-user object to delete:
'<account>/<container>/<object>'
:param timestamp: The swift.common.utils.Timestamp instance the
X-Delete-At value must match to perform the actual
delete.
:param is_async_delete: False if the object should be deleted because
of "normal" expiration, or True if it should
be async-deleted.
:raises UnexpectedResponse: if the delete was unsuccessful and
should be retried later
"""
path = '/v1/' + wsgi_quote(str_to_wsgi(actual_obj.lstrip('/')))
if is_async_delete:
headers = {'X-Timestamp': timestamp.normal}
acceptable_statuses = (2, HTTP_CONFLICT, HTTP_NOT_FOUND)
else:
headers = {
'X-Timestamp': timestamp.normal,
'X-If-Delete-At': timestamp.normal,
'X-Backend-Clean-Expiring-Object-Queue': 'no'
}
acceptable_statuses = (2, HTTP_CONFLICT)
self.swift.make_request('DELETE', path, headers, acceptable_statuses)
def test_dlo_post_with_manifest_header(self):
# verify that performing a POST to a DLO manifest
# preserves the fact that it is a manifest file.
# verify that the x-object-manifest header may be updated.
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write(b'manifest-contents',
hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST a modified x-object-manifest value
new_x_o_m = x_o_m.rstrip('lower') + 'upper'
file_item.post({
'x-object-meta-foo': 'bar',
'x-object-manifest': new_x_o_m
})
# verify that x-object-manifest was updated
file_item.info()
resp_headers = [(h.lower(), v)
for h, v in file_item.conn.response.getheaders()]
self.assertIn(('x-object-manifest', str_to_wsgi(new_x_o_m)),
resp_headers)
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that updated manifest points to new content
expected_contents = ''.join((c * 10) for c in 'ABCDE').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# Now revert the manifest to point to original segments, including a
# multipart-manifest=get param just to check that has no effect
file_item.post({'x-object-manifest': x_o_m},
parms={'multipart-manifest': 'get'})
# verify that x-object-manifest was reverted
info = file_item.info()
self.assertIn('x_object_manifest', info)
self.assertEqual(x_o_m, info['x_object_manifest'])
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that updated manifest points new content
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
def test_dlo_post_with_manifest_regular_object(self):
# verify that performing a POST to a regular object
# with a manifest header will create a DLO.
# Put a regular object
file_item = self.env.container.file(Utils.create_name())
file_item.write(b'file contents', hdrs={})
# sanity checks
file_contents = file_item.read(parms={})
self.assertEqual(b'file contents', file_contents)
# get the path associated with man1
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
# POST a x-object-manifest value to the regular object
file_item.post({'x-object-manifest': x_o_m})
# verify that the file is now a manifest
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'file contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde']).encode()
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
file_item.info()
resp_headers = [(h.lower(), v)
for h, v in file_item.conn.response.getheaders()]
self.assertIn(('x-object-manifest', str_to_wsgi(x_o_m)), resp_headers)
def delete_actual_object(self, actual_obj, timestamp, is_async_delete):
"""
Deletes the end-user object indicated by the actual object name given
'<account>/<container>/<object>' if and only if the X-Delete-At value
of the object is exactly the timestamp given.
:param actual_obj: The name of the end-user object to delete:
'<account>/<container>/<object>'
:param timestamp: The swift.common.utils.Timestamp instance the
X-Delete-At value must match to perform the actual
delete.
:param is_async_delete: False if the object should be deleted because
of "normal" expiration, or True if it should
be async-deleted.
:raises UnexpectedResponse: if the delete was unsuccessful and
should be retried later
"""
path = '/v1/' + wsgi_quote(str_to_wsgi(actual_obj.lstrip('/')))
if is_async_delete:
headers = {'X-Timestamp': timestamp.normal}
acceptable_statuses = (2, HTTP_CONFLICT, HTTP_NOT_FOUND)
else:
headers = {'X-Timestamp': timestamp.normal,
'X-If-Delete-At': timestamp.normal,
'X-Backend-Clean-Expiring-Object-Queue': 'no'}
acceptable_statuses = (2, HTTP_CONFLICT)
self.swift.make_request('DELETE', path, headers, acceptable_statuses)
def _put_object(self, app, body):
req = Request.blank(
str_to_wsgi(self.object_path), method='PUT', body=body,
headers={'Content-Type': 'application/test'})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
return resp
def do_delete(obj_name, delete_path):
delete_obj_req = make_subrequest(
req.environ, method='DELETE',
path=wsgi_quote(str_to_wsgi(delete_path)),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
body='', agent='%(orig)s ' + user_agent,
swift_source=swift_source)
return (delete_obj_req.get_response(self.app), obj_name, 0)
def _put_object(self, app, body):
req = Request.blank(
str_to_wsgi(self.object_path), method='PUT', body=body,
headers={'Content-Type': 'application/test'})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
return resp
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
"""
Handle request for object resource.
Note that account, container, obj should be unquoted by caller
if the url path is under url encoding (e.g. %FF)
:param req: swift.common.swob.Request instance
:param api_version: should be v1 unless swift bumps api version
:param account: account name string
:param container: container name string
:param object: object name string
"""
resp = None
is_enabled = config_true_value(allow_versioned_writes)
container_info = get_container_info(
req.environ, self.app)
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
versions_cont = container_info.get(
'sysmeta', {}).get('versions-location')
versioning_mode = container_info.get(
'sysmeta', {}).get('versions-mode', 'stack')
if not versions_cont:
versions_cont = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and versions_cont:
is_enabled = True
if is_enabled and versions_cont:
versions_cont = wsgi_unquote(str_to_wsgi(
versions_cont)).split('/')[0]
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method == 'PUT':
resp = vw_ctx.handle_obj_versions_put(
req, versions_cont, api_version, account,
obj)
# handle DELETE
elif versioning_mode == 'history':
resp = vw_ctx.handle_obj_versions_delete_push(
req, versions_cont, api_version, account,
container, obj)
else:
resp = vw_ctx.handle_obj_versions_delete_pop(
req, versions_cont, api_version, account,
container, obj)
if resp:
return resp
else:
return self.app
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
"""
Handle request for object resource.
Note that account, container, obj should be unquoted by caller
if the url path is under url encoding (e.g. %FF)
:param req: swift.common.swob.Request instance
:param api_version: should be v1 unless swift bumps api version
:param account: account name string
:param container: container name string
:param object: object name string
"""
resp = None
is_enabled = config_true_value(allow_versioned_writes)
container_info = get_container_info(
req.environ, self.app, swift_source='VW')
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
versions_cont = container_info.get(
'sysmeta', {}).get('versions-location')
versioning_mode = container_info.get(
'sysmeta', {}).get('versions-mode', 'stack')
if not versions_cont:
versions_cont = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and versions_cont:
is_enabled = True
if is_enabled and versions_cont:
versions_cont = wsgi_unquote(str_to_wsgi(
versions_cont)).split('/')[0]
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method == 'PUT':
resp = vw_ctx.handle_obj_versions_put(
req, versions_cont, api_version, account,
obj)
# handle DELETE
elif versioning_mode == 'history':
resp = vw_ctx.handle_obj_versions_delete_push(
req, versions_cont, api_version, account,
container, obj)
else:
resp = vw_ctx.handle_obj_versions_delete_pop(
req, versions_cont, api_version, account,
container, obj)
if resp:
return resp
else:
return self.app
def _make_key_id(self, path, secret_id, version):
if version in ('1', '2'):
path = str_to_wsgi(path)
key_id = {'v': version, 'path': path}
if secret_id:
# stash secret_id so that decrypter can pass it back to get the
# same keys
key_id['secret_id'] = secret_id
return key_id
def _check_match_requests(self, method, app, object_path=None):
object_path = str_to_wsgi(object_path or self.object_path)
# verify conditional match requests
expected_body = self.plaintext if method == 'GET' else b''
# If-Match matches
req = Request.blank(object_path, method=method,
headers={'If-Match': '"%s"' % self.plaintext_etag})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
# If-Match wildcard
req = Request.blank(object_path, method=method,
headers={'If-Match': '*'})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
# If-Match does not match
req = Request.blank(object_path, method=method,
headers={'If-Match': '"not the etag"'})
resp = req.get_response(app)
self.assertEqual('412 Precondition Failed', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match matches
req = Request.blank(
object_path, method=method,
headers={'If-None-Match': '"%s"' % self.plaintext_etag})
resp = req.get_response(app)
self.assertEqual('304 Not Modified', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match wildcard
req = Request.blank(object_path, method=method,
headers={'If-None-Match': '*'})
resp = req.get_response(app)
self.assertEqual('304 Not Modified', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match does not match
req = Request.blank(object_path, method=method,
headers={'If-None-Match': '"not the etag"'})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def _check_match_requests(self, method, app, object_path=None):
object_path = str_to_wsgi(object_path or self.object_path)
# verify conditional match requests
expected_body = self.plaintext if method == 'GET' else b''
# If-Match matches
req = Request.blank(object_path, method=method,
headers={'If-Match': '"%s"' % self.plaintext_etag})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
# If-Match wildcard
req = Request.blank(object_path, method=method,
headers={'If-Match': '*'})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
# If-Match does not match
req = Request.blank(object_path, method=method,
headers={'If-Match': '"not the etag"'})
resp = req.get_response(app)
self.assertEqual('412 Precondition Failed', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match matches
req = Request.blank(
object_path, method=method,
headers={'If-None-Match': '"%s"' % self.plaintext_etag})
resp = req.get_response(app)
self.assertEqual('304 Not Modified', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match wildcard
req = Request.blank(object_path, method=method,
headers={'If-None-Match': '*'})
resp = req.get_response(app)
self.assertEqual('304 Not Modified', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match does not match
req = Request.blank(object_path, method=method,
headers={'If-None-Match': '"not the etag"'})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def pax_key_to_swift_header(pax_key):
if (pax_key == u"SCHILY.xattr.user.mime_type" or
pax_key == u"LIBARCHIVE.xattr.user.mime_type"):
return "Content-Type"
elif pax_key.startswith(u"SCHILY.xattr.user.meta."):
useful_part = pax_key[len(u"SCHILY.xattr.user.meta."):]
if six.PY2:
return "X-Object-Meta-" + useful_part.encode("utf-8")
return str_to_wsgi("X-Object-Meta-" + useful_part)
elif pax_key.startswith(u"LIBARCHIVE.xattr.user.meta."):
useful_part = pax_key[len(u"LIBARCHIVE.xattr.user.meta."):]
if six.PY2:
return "X-Object-Meta-" + useful_part.encode("utf-8")
return str_to_wsgi("X-Object-Meta-" + useful_part)
else:
# You can get things like atime/mtime/ctime or filesystem ACLs in
# pax headers; those aren't really user metadata. The same goes for
# other, non-user metadata.
return None
def _create_container(self, app, policy_name='one', container_path=None):
if not container_path:
# choose new container name so that the policy can be specified
self.container_name = uuid.uuid4().hex
self.container_path = 'http://foo:8080/v1/a/' + self.container_name
self.object_name = 'o'
self.object_path = self.container_path + '/' + self.object_name
container_path = self.container_path
req = Request.blank(str_to_wsgi(container_path),
method='PUT',
headers={'X-Storage-Policy': policy_name})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
# sanity check
req = Request.blank(str_to_wsgi(container_path),
method='HEAD',
headers={'X-Storage-Policy': policy_name})
resp = req.get_response(app)
self.assertEqual(policy_name, resp.headers['X-Storage-Policy'])
def do_delete(obj_name, delete_path):
delete_obj_req = make_subrequest(
req.environ,
method='DELETE',
path=wsgi_quote(str_to_wsgi(delete_path)),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
body='',
agent='%(orig)s ' + user_agent,
swift_source=swift_source)
return (delete_obj_req.get_response(self.app), obj_name, 0)
def _create_container(self, app, policy_name='one', container_path=None):
if not container_path:
# choose new container name so that the policy can be specified
self.container_name = uuid.uuid4().hex
self.container_path = 'http://foo:8080/v1/a/' + self.container_name
self.object_name = 'o'
self.object_path = self.container_path + '/' + self.object_name
container_path = self.container_path
req = Request.blank(
str_to_wsgi(container_path), method='PUT',
headers={'X-Storage-Policy': policy_name})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
# sanity check
req = Request.blank(
str_to_wsgi(container_path), method='HEAD',
headers={'X-Storage-Policy': policy_name})
resp = req.get_response(app)
self.assertEqual(policy_name, resp.headers['X-Storage-Policy'])
def pax_key_to_swift_header(pax_key):
if (pax_key == u"SCHILY.xattr.user.mime_type"
or pax_key == u"LIBARCHIVE.xattr.user.mime_type"):
return "Content-Type"
elif pax_key.startswith(u"SCHILY.xattr.user.meta."):
useful_part = pax_key[len(u"SCHILY.xattr.user.meta."):]
if six.PY2:
return "X-Object-Meta-" + useful_part.encode("utf-8")
return str_to_wsgi("X-Object-Meta-" + useful_part)
elif pax_key.startswith(u"LIBARCHIVE.xattr.user.meta."):
useful_part = pax_key[len(u"LIBARCHIVE.xattr.user.meta."):]
if six.PY2:
return "X-Object-Meta-" + useful_part.encode("utf-8")
return str_to_wsgi("X-Object-Meta-" + useful_part)
else:
# You can get things like atime/mtime/ctime or filesystem ACLs in
# pax headers; those aren't really user metadata. The same goes for
# other, non-user metadata.
return None
def create_listing(self, req, out_content_type, info, resp_headers,
metadata, container_list, container):
for key, (value, timestamp) in metadata.items():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[str_to_wsgi(key)] = str_to_wsgi(value)
listing = [self.update_data_record(record)
for record in container_list]
if out_content_type.endswith('/xml'):
body = listing_formats.container_to_xml(listing, container)
elif out_content_type.endswith('/json'):
body = json.dumps(listing).encode('ascii')
else:
body = listing_formats.listing_to_text(listing)
ret = Response(request=req, headers=resp_headers, body=body,
content_type=out_content_type, charset='utf-8')
ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
if not ret.body:
ret.status_int = HTTP_NO_CONTENT
return ret
def handle_delete(self, req, start_response):
"""
Handle request to delete a user's container.
As part of deleting a container, this middleware will also delete
the hidden container holding object versions.
Before a user's container can be deleted, swift must check
if there are still old object versions from that container.
Only after disabling versioning and deleting *all* object versions
can a container be deleted.
"""
container_info = get_container_info(req.environ,
self.app,
swift_source='OV')
versions_cont = unquote(
container_info.get('sysmeta', {}).get('versions-container', ''))
if versions_cont:
account = req.split_path(3, 3, True)[1]
# using a HEAD request here as opposed to get_container_info
# to make sure we get an up-to-date value
versions_req = make_pre_authed_request(
req.environ,
method='HEAD',
swift_source='OV',
path=wsgi_quote('/v1/%s/%s' %
(account, str_to_wsgi(versions_cont))),
headers={'X-Backend-Allow-Reserved-Names': 'true'})
vresp = versions_req.get_response(self.app)
drain_and_close(vresp)
if vresp.is_success and int(
vresp.headers.get('X-Container-Object-Count', 0)) > 0:
raise HTTPConflict(
'Delete all versions before deleting container.',
request=req)
elif not vresp.is_success and vresp.status_int != 404:
raise HTTPInternalServerError(
'Error deleting versioned container')
else:
versions_req.method = 'DELETE'
resp = versions_req.get_response(self.app)
drain_and_close(resp)
if not is_success(resp.status_int) and resp.status_int != 404:
raise HTTPInternalServerError(
'Error deleting versioned container')
app_resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
def get_response_headers(broker):
info = broker.get_info()
resp_headers = {
'X-Account-Container-Count': info['container_count'],
'X-Account-Object-Count': info['object_count'],
'X-Account-Bytes-Used': info['bytes_used'],
'X-Timestamp': Timestamp(info['created_at']).normal,
'X-PUT-Timestamp': Timestamp(info['put_timestamp']).normal
}
policy_stats = broker.get_policy_stats()
for policy_idx, stats in policy_stats.items():
policy = POLICIES.get_by_index(policy_idx)
if not policy:
continue
header_prefix = 'X-Account-Storage-Policy-%s-%%s' % policy.name
for key, value in stats.items():
header_name = header_prefix % key.replace('_', '-')
resp_headers[header_name] = value
resp_headers.update(
(str_to_wsgi(key), str_to_wsgi(value))
for key, (value, _timestamp) in broker.metadata.items() if value != '')
return resp_headers
def delete_filter(predicate, objs_to_delete):
for obj_to_delete in objs_to_delete:
obj_name = obj_to_delete['name']
if not obj_name:
continue
if not predicate(obj_name):
continue
if obj_to_delete.get('error'):
if obj_to_delete['error']['code'] == HTTP_NOT_FOUND:
resp_dict['Number Not Found'] += 1
else:
failed_files.append([
wsgi_quote(str_to_wsgi(obj_name)),
obj_to_delete['error']['message']])
continue
delete_path = '/'.join(['', vrs, account,
obj_name.lstrip('/')])
if not constraints.check_utf8(delete_path):
failed_files.append([wsgi_quote(str_to_wsgi(obj_name)),
HTTPPreconditionFailed().status])
continue
yield (obj_name, delete_path)
def _check_GET_and_HEAD(self, app, object_path=None):
object_path = str_to_wsgi(object_path or self.object_path)
req = Request.blank(object_path, method='GET')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(self.plaintext, resp.body)
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
req = Request.blank(object_path, method='HEAD')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def _check_GET_and_HEAD(self, app, object_path=None):
object_path = str_to_wsgi(object_path or self.object_path)
req = Request.blank(object_path, method='GET')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(self.plaintext, resp.body)
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
req = Request.blank(object_path, method='HEAD')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(b'', resp.body)
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def _test_get_path(self, host, path, anonymous=False, expected_status=200,
expected_in=[], expected_not_in=[]):
self.env.account.conn.make_request(
'GET', str_to_wsgi(path),
hdrs={'X-Web-Mode': str(not anonymous), 'Host': host},
cfg={'no_auth_token': anonymous, 'absolute_path': True})
self.assert_status(expected_status)
body = self.env.account.conn.response.read()
if not six.PY2:
body = body.decode('utf8')
for string in expected_in:
self.assertIn(string, body)
for string in expected_not_in:
self.assertNotIn(string, body)
def _check_listing(self, app, expect_mismatch=False, container_path=None):
container_path = str_to_wsgi(container_path or self.container_path)
req = Request.blank(
container_path, method='GET', query_string='format=json')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
listing = json.loads(resp.body)
self.assertEqual(1, len(listing))
self.assertEqual(self.object_name, listing[0]['name'])
self.assertEqual(len(self.plaintext), listing[0]['bytes'])
if expect_mismatch:
self.assertNotEqual(self.plaintext_etag, listing[0]['hash'])
else:
self.assertEqual(self.plaintext_etag, listing[0]['hash'])
def _check_listing(self, app, expect_mismatch=False, container_path=None):
container_path = str_to_wsgi(container_path or self.container_path)
req = Request.blank(
container_path, method='GET', query_string='format=json')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
listing = json.loads(resp.body)
self.assertEqual(1, len(listing))
self.assertEqual(self.object_name, listing[0]['name'])
self.assertEqual(len(self.plaintext), listing[0]['bytes'])
if expect_mismatch:
self.assertNotEqual(self.plaintext_etag, listing[0]['hash'])
else:
self.assertEqual(self.plaintext_etag, listing[0]['hash'])
def _process_delete(self, resp, pile, obj_name, resp_dict,
failed_files, failed_file_response, retry=0):
if resp.status_int // 100 == 2:
resp_dict['Number Deleted'] += 1
elif resp.status_int == HTTP_NOT_FOUND:
resp_dict['Number Not Found'] += 1
elif resp.status_int == HTTP_UNAUTHORIZED:
failed_files.append([wsgi_quote(str_to_wsgi(obj_name)),
HTTPUnauthorized().status])
elif resp.status_int == HTTP_CONFLICT and pile and \
self.retry_count > 0 and self.retry_count > retry:
retry += 1
sleep(self.retry_interval ** retry)
delete_obj_req = Request.blank(resp.environ['PATH_INFO'],
resp.environ)
def _retry(req, app, obj_name, retry):
return req.get_response(app), obj_name, retry
pile.spawn(_retry, delete_obj_req, self.app, obj_name, retry)
else:
if resp.status_int // 100 == 5:
failed_file_response['type'] = HTTPBadGateway
failed_files.append([wsgi_quote(str_to_wsgi(obj_name)),
resp.status])
def object_request(self, req, api_version, account, container, obj):
"""
Handle request for object resource.
Note that account, container, obj should be unquoted by caller
if the url path is under url encoding (e.g. %FF)
:param req: swift.common.swob.Request instance
:param api_version: should be v1 unless swift bumps api version
:param account: account name string
:param container: container name string
:param object: object name string
"""
resp = None
container_info = get_container_info(req.environ,
self.app,
swift_source='OV')
versions_cont = container_info.get('sysmeta',
{}).get('versions-container', '')
is_enabled = config_true_value(
container_info.get('sysmeta', {}).get('versions-enabled'))
if versions_cont:
versions_cont = wsgi_unquote(
str_to_wsgi(versions_cont)).split('/')[0]
if req.params.get('version-id'):
vw_ctx = OioObjectContext(self.app, self.logger)
resp = vw_ctx.handle_versioned_request(req, versions_cont,
api_version, account,
container, obj, is_enabled,
req.params['version-id'])
elif versions_cont:
# handle object request for a enabled versioned container
vw_ctx = OioObjectContext(self.app, self.logger)
resp = vw_ctx.handle_request(req, versions_cont, api_version,
account, container, obj, is_enabled)
if resp:
return resp
else:
return self.app
def verify_v1_keys_for_path(self, wsgi_path, expected_keys, key_id=None):
put_keys = None
self.app.meta_version_to_write = '1'
for method, resp_class, status in (('PUT', swob.HTTPCreated, '201'),
('POST', swob.HTTPAccepted, '202'),
('GET', swob.HTTPOk, '200'),
('HEAD', swob.HTTPNoContent,
'204')):
resp_headers = {}
self.swift.register(method, '/v1' + wsgi_path, resp_class,
resp_headers, b'')
req = Request.blank('/v1' + wsgi_path,
environ={'REQUEST_METHOD': method})
start_response, calls = capture_start_response()
self.app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertTrue(calls[0][0].startswith(status))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)(key_id=key_id)
self.assertIn('id', keys)
id = keys.pop('id')
path = swob.wsgi_to_str(wsgi_path)
if '//' in path:
path = path[path.index('//') + 1:]
if six.PY2:
self.assertEqual(path, id['path'])
else:
self.assertEqual(swob.str_to_wsgi(path), id['path'])
self.assertEqual('1', id['v'])
keys.pop('all_ids')
self.assertListEqual(
sorted(expected_keys), sorted(keys.keys()),
'%s %s got keys %r, but expected %r' %
(method, path, keys.keys(), expected_keys))
if put_keys is not None:
# check all key sets were consistent for this path
self.assertDictEqual(put_keys, keys)
else:
put_keys = keys
self.app.meta_version_to_write = '2' # Clean up after ourselves
return put_keys
def handle_container(self, env, start_response):
"""
Handles a possible static web request for a container.
:param env: The original WSGI environment dict.
:param start_response: The original WSGI start_response hook.
"""
container_info = self._get_container_info(env)
req = Request(env)
req.acl = container_info['read_acl']
# we checked earlier that swift.authorize is set in env
aresp = env['swift.authorize'](req)
if aresp:
resp = aresp(env, self._start_response)
return self._error_response(resp, env, start_response)
if not self._listings and not self._index:
if config_true_value(env.get('HTTP_X_WEB_MODE', 'f')):
return HTTPNotFound()(env, start_response)
return self.app(env, start_response)
if not env['PATH_INFO'].endswith('/'):
return self._redirect_with_slash(env, start_response)
if not self._index:
return self._listing(env, start_response)
tmp_env = dict(env)
tmp_env['HTTP_USER_AGENT'] = \
'%s StaticWeb' % env.get('HTTP_USER_AGENT')
tmp_env['swift.source'] = 'SW'
tmp_env['PATH_INFO'] += str_to_wsgi(self._index)
resp = self._app_call(tmp_env)
status_int = self._get_status_int()
if status_int == HTTP_NOT_FOUND:
return self._listing(env, start_response)
elif not is_success(self._get_status_int()) and \
not is_redirection(self._get_status_int()):
return self._error_response(resp, env, start_response)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
def _get_from_shards(self, req, resp):
# construct listing using shards described by the response body
shard_ranges = [ShardRange.from_dict(data)
for data in json.loads(resp.body)]
self.app.logger.debug('GET listing from %s shards for: %s',
len(shard_ranges), req.path_qs)
if not shard_ranges:
# can't find ranges or there was a problem getting the ranges. So
# return what we have.
return resp
objects = []
req_limit = int(req.params.get('limit', CONTAINER_LISTING_LIMIT))
params = req.params.copy()
params.pop('states', None)
req.headers.pop('X-Backend-Record-Type', None)
reverse = config_true_value(params.get('reverse'))
marker = params.get('marker')
end_marker = params.get('end_marker')
limit = req_limit
for shard_range in shard_ranges:
params['limit'] = limit
# Always set marker to ensure that object names less than or equal
# to those already in the listing are not fetched; if the listing
# is empty then the original request marker, if any, is used. This
# allows misplaced objects below the expected shard range to be
# included in the listing.
if objects:
last_name = objects[-1].get('name',
objects[-1].get('subdir', u''))
params['marker'] = last_name.encode('utf-8')
elif marker:
params['marker'] = marker
else:
params['marker'] = ''
# Always set end_marker to ensure that misplaced objects beyond the
# expected shard range are not fetched. This prevents a misplaced
# object obscuring correctly placed objects in the next shard
# range.
if end_marker and end_marker in shard_range:
params['end_marker'] = end_marker
elif reverse:
params['end_marker'] = str_to_wsgi(shard_range.lower_str)
else:
params['end_marker'] = str_to_wsgi(shard_range.end_marker)
if (shard_range.account == self.account_name and
shard_range.container == self.container_name):
# directed back to same container - force GET of objects
headers = {'X-Backend-Record-Type': 'object'}
else:
headers = None
self.app.logger.debug('Getting from %s %s with %s',
shard_range, shard_range.name, headers)
objs, shard_resp = self._get_container_listing(
req, shard_range.account, shard_range.container,
headers=headers, params=params)
if not objs:
# tolerate errors or empty shard containers
continue
objects.extend(objs)
limit -= len(objs)
if limit <= 0:
break
if (end_marker and reverse and
(wsgi_to_bytes(end_marker) >=
objects[-1]['name'].encode('utf-8'))):
break
if (end_marker and not reverse and
(wsgi_to_bytes(end_marker) <=
objects[-1]['name'].encode('utf-8'))):
break
resp.body = json.dumps(objects).encode('ascii')
constrained = any(req.params.get(constraint) for constraint in (
'marker', 'end_marker', 'path', 'prefix', 'delimiter'))
if not constrained and len(objects) < req_limit:
self.app.logger.debug('Setting object count to %s' % len(objects))
# prefer the actual listing stats over the potentially outdated
# root stats. This condition is only likely when a sharded
# container is shrinking or in tests; typically a sharded container
# will have more than CONTAINER_LISTING_LIMIT objects so any
# unconstrained listing will be capped by the limit and total
# object stats cannot therefore be inferred from the listing.
resp.headers['X-Container-Object-Count'] = len(objects)
resp.headers['X-Container-Bytes-Used'] = sum(
[o['bytes'] for o in objects])
return resp
def _perform_subrequest(self, orig_env, attributes, fp, keys):
"""
Performs the subrequest and returns the response.
:param orig_env: The WSGI environment dict; will only be used
to form a new env for the subrequest.
:param attributes: dict of the attributes of the form so far.
:param fp: The file-like object containing the request body.
:param keys: The account keys to validate the signature with.
:returns: (status_line, headers_list)
"""
if not keys:
raise FormUnauthorized('invalid signature')
try:
max_file_size = int(attributes.get('max_file_size') or 0)
except ValueError:
raise FormInvalid('max_file_size not an integer')
subenv = make_pre_authed_env(orig_env, 'PUT', agent=None,
swift_source='FP')
if 'QUERY_STRING' in subenv:
del subenv['QUERY_STRING']
subenv['HTTP_TRANSFER_ENCODING'] = 'chunked'
subenv['wsgi.input'] = _CappedFileLikeObject(fp, max_file_size)
if not subenv['PATH_INFO'].endswith('/') and \
subenv['PATH_INFO'].count('/') < 4:
subenv['PATH_INFO'] += '/'
subenv['PATH_INFO'] += str_to_wsgi(
attributes['filename'] or 'filename')
if 'x_delete_at' in attributes:
try:
subenv['HTTP_X_DELETE_AT'] = int(attributes['x_delete_at'])
except ValueError:
raise FormInvalid('x_delete_at not an integer: '
'Unix timestamp required.')
if 'x_delete_after' in attributes:
try:
subenv['HTTP_X_DELETE_AFTER'] = int(
attributes['x_delete_after'])
except ValueError:
raise FormInvalid('x_delete_after not an integer: '
'Number of seconds required.')
if 'content-type' in attributes:
subenv['CONTENT_TYPE'] = \
attributes['content-type'] or 'application/octet-stream'
if 'content-encoding' in attributes:
subenv['HTTP_CONTENT_ENCODING'] = attributes['content-encoding']
try:
if int(attributes.get('expires') or 0) < time():
raise FormUnauthorized('form expired')
except ValueError:
raise FormInvalid('expired not an integer')
hmac_body = '%s\n%s\n%s\n%s\n%s' % (
wsgi_to_str(orig_env['PATH_INFO']),
attributes.get('redirect') or '',
attributes.get('max_file_size') or '0',
attributes.get('max_file_count') or '0',
attributes.get('expires') or '0')
if six.PY3:
hmac_body = hmac_body.encode('utf-8')
has_valid_sig = False
for key in keys:
# Encode key like in swift.common.utls.get_hmac.
if not isinstance(key, six.binary_type):
key = key.encode('utf8')
sig = hmac.new(key, hmac_body, sha1).hexdigest()
if streq_const_time(sig, (attributes.get('signature') or
'invalid')):
has_valid_sig = True
if not has_valid_sig:
raise FormUnauthorized('invalid signature')
substatus = [None]
subheaders = [None]
wsgi_input = subenv['wsgi.input']
def _start_response(status, headers, exc_info=None):
if wsgi_input.file_size_exceeded:
raise EOFError("max_file_size exceeded")
substatus[0] = status
subheaders[0] = headers
# reiterate to ensure the response started,
# but drop any data on the floor
close_if_possible(reiterate(self.app(subenv, _start_response)))
return substatus[0], subheaders[0]
def __call__(self, env, start_response):
if not self.storage_domain:
return self.app(env, start_response)
if 'HTTP_HOST' in env:
requested_host = env['HTTP_HOST']
else:
requested_host = env['SERVER_NAME']
given_domain = wsgi_to_str(requested_host)
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if is_valid_ip(given_domain):
return self.app(env, start_response)
a_domain = given_domain
if not self._domain_endswith_in_storage_domain(a_domain):
if self.memcache is None:
self.memcache = cache_from_env(env)
error = True
for tries in range(self.lookup_depth):
found_domain = None
if self.memcache:
memcache_key = ''.join(['cname-', a_domain])
found_domain = self.memcache.get(memcache_key)
if six.PY2 and found_domain:
found_domain = found_domain.encode('utf-8')
if found_domain is None:
ttl, found_domain = lookup_cname(a_domain, self.resolver)
if self.memcache and ttl > 0:
memcache_key = ''.join(['cname-', given_domain])
self.memcache.set(memcache_key, found_domain, time=ttl)
if not found_domain or found_domain == a_domain:
# no CNAME records or we're at the last lookup
error = True
found_domain = None
break
elif self._domain_endswith_in_storage_domain(found_domain):
# Found it!
self.logger.info(
_('Mapped %(given_domain)s to %(found_domain)s') % {
'given_domain': given_domain,
'found_domain': found_domain
})
if port:
env['HTTP_HOST'] = ':'.join(
[str_to_wsgi(found_domain), port])
else:
env['HTTP_HOST'] = str_to_wsgi(found_domain)
error = False
break
else:
# try one more deep in the chain
self.logger.debug(
_('Following CNAME chain for '
'%(given_domain)s to %(found_domain)s') % {
'given_domain': given_domain,
'found_domain': found_domain
})
a_domain = found_domain
if error:
if found_domain:
msg = 'CNAME lookup failed after %d tries' % \
self.lookup_depth
else:
msg = 'CNAME lookup failed to resolve to a valid domain'
resp = HTTPBadRequest(request=Request(env),
body=msg,
content_type='text/plain')
return resp(env, start_response)
else:
context = _CnameLookupContext(self.app, requested_host,
env['HTTP_HOST'])
return context.handle_request(env, start_response)
return self.app(env, start_response)
def _get_from_shards(self, req, resp):
# Construct listing using shards described by the response body.
# The history of containers that have returned shard ranges is
# maintained in the request environ so that loops can be avoided by
# forcing an object listing if the same container is visited again.
# This can happen in at least two scenarios:
# 1. a container has filled a gap in its shard ranges with a
# shard range pointing to itself
# 2. a root container returns a (stale) shard range pointing to a
# shard that has shrunk into the root, in which case the shrunken
# shard may return the root's shard range.
shard_listing_history = req.environ.setdefault(
'swift.shard_listing_history', [])
shard_listing_history.append((self.account_name, self.container_name))
shard_ranges = [
ShardRange.from_dict(data) for data in json.loads(resp.body)
]
self.app.logger.debug('GET listing from %s shards for: %s',
len(shard_ranges), req.path_qs)
if not shard_ranges:
# can't find ranges or there was a problem getting the ranges. So
# return what we have.
return resp
objects = []
req_limit = constrain_req_limit(req, CONTAINER_LISTING_LIMIT)
params = req.params.copy()
params.pop('states', None)
req.headers.pop('X-Backend-Record-Type', None)
reverse = config_true_value(params.get('reverse'))
marker = wsgi_to_str(params.get('marker'))
end_marker = wsgi_to_str(params.get('end_marker'))
prefix = wsgi_to_str(params.get('prefix'))
limit = req_limit
for i, shard_range in enumerate(shard_ranges):
params['limit'] = limit
# Always set marker to ensure that object names less than or equal
# to those already in the listing are not fetched; if the listing
# is empty then the original request marker, if any, is used. This
# allows misplaced objects below the expected shard range to be
# included in the listing.
if objects:
last_name = objects[-1].get('name',
objects[-1].get('subdir', u''))
params['marker'] = bytes_to_wsgi(last_name.encode('utf-8'))
elif marker:
params['marker'] = str_to_wsgi(marker)
else:
params['marker'] = ''
# Always set end_marker to ensure that misplaced objects beyond the
# expected shard range are not fetched. This prevents a misplaced
# object obscuring correctly placed objects in the next shard
# range.
if end_marker and end_marker in shard_range:
params['end_marker'] = str_to_wsgi(end_marker)
elif reverse:
params['end_marker'] = str_to_wsgi(shard_range.lower_str)
else:
params['end_marker'] = str_to_wsgi(shard_range.end_marker)
headers = {}
if ((shard_range.account, shard_range.container)
in shard_listing_history):
# directed back to same container - force GET of objects
headers['X-Backend-Record-Type'] = 'object'
if config_true_value(req.headers.get('x-newest', False)):
headers['X-Newest'] = 'true'
if prefix:
if prefix > shard_range:
continue
try:
just_past = prefix[:-1] + chr(ord(prefix[-1]) + 1)
except ValueError:
pass
else:
if just_past < shard_range:
continue
self.app.logger.debug(
'Getting listing part %d from shard %s %s with %s', i,
shard_range, shard_range.name, headers)
objs, shard_resp = self._get_container_listing(
req,
shard_range.account,
shard_range.container,
headers=headers,
params=params)
sharding_state = shard_resp.headers.get('x-backend-sharding-state',
'unknown')
if objs is None:
# tolerate errors
self.app.logger.debug(
'Failed to get objects from shard (state=%s), total = %d',
sharding_state, len(objects))
continue
self.app.logger.debug(
'Found %d objects in shard (state=%s), total = %d', len(objs),
sharding_state,
len(objs) + len(objects))
if not objs:
# tolerate empty shard containers
continue
objects.extend(objs)
limit -= len(objs)
if limit <= 0:
break
last_name = objects[-1].get('name', objects[-1].get('subdir', u''))
if six.PY2:
last_name = last_name.encode('utf8')
if end_marker and reverse and end_marker >= last_name:
break
if end_marker and not reverse and end_marker <= last_name:
break
resp.body = json.dumps(objects).encode('ascii')
constrained = any(
req.params.get(constraint)
for constraint in ('marker', 'end_marker', 'path', 'prefix',
'delimiter'))
if not constrained and len(objects) < req_limit:
self.app.logger.debug('Setting object count to %s' % len(objects))
# prefer the actual listing stats over the potentially outdated
# root stats. This condition is only likely when a sharded
# container is shrinking or in tests; typically a sharded container
# will have more than CONTAINER_LISTING_LIMIT objects so any
# unconstrained listing will be capped by the limit and total
# object stats cannot therefore be inferred from the listing.
resp.headers['X-Container-Object-Count'] = len(objects)
resp.headers['X-Container-Bytes-Used'] = sum(
[o['bytes'] for o in objects])
return resp
def _get_from_shards(self, req, resp):
# construct listing using shards described by the response body
shard_ranges = [
ShardRange.from_dict(data) for data in json.loads(resp.body)
]
self.app.logger.debug('GET listing from %s shards for: %s',
len(shard_ranges), req.path_qs)
if not shard_ranges:
# can't find ranges or there was a problem getting the ranges. So
# return what we have.
return resp
objects = []
req_limit = int(req.params.get('limit', CONTAINER_LISTING_LIMIT))
params = req.params.copy()
params.pop('states', None)
req.headers.pop('X-Backend-Record-Type', None)
reverse = config_true_value(params.get('reverse'))
marker = params.get('marker')
end_marker = params.get('end_marker')
limit = req_limit
for shard_range in shard_ranges:
params['limit'] = limit
# Always set marker to ensure that object names less than or equal
# to those already in the listing are not fetched; if the listing
# is empty then the original request marker, if any, is used. This
# allows misplaced objects below the expected shard range to be
# included in the listing.
if objects:
last_name = objects[-1].get('name',
objects[-1].get('subdir', u''))
params['marker'] = last_name.encode('utf-8')
elif marker:
params['marker'] = marker
else:
params['marker'] = ''
# Always set end_marker to ensure that misplaced objects beyond the
# expected shard range are not fetched. This prevents a misplaced
# object obscuring correctly placed objects in the next shard
# range.
if end_marker and end_marker in shard_range:
params['end_marker'] = end_marker
elif reverse:
params['end_marker'] = str_to_wsgi(shard_range.lower_str)
else:
params['end_marker'] = str_to_wsgi(shard_range.end_marker)
if (shard_range.account == self.account_name
and shard_range.container == self.container_name):
# directed back to same container - force GET of objects
headers = {'X-Backend-Record-Type': 'object'}
else:
headers = None
self.app.logger.debug('Getting from %s %s with %s', shard_range,
shard_range.name, headers)
objs, shard_resp = self._get_container_listing(
req,
shard_range.account,
shard_range.container,
headers=headers,
params=params)
if not objs:
# tolerate errors or empty shard containers
continue
objects.extend(objs)
limit -= len(objs)
if limit <= 0:
break
if (end_marker and reverse
and (wsgi_to_bytes(end_marker) >=
objects[-1]['name'].encode('utf-8'))):
break
if (end_marker and not reverse
and (wsgi_to_bytes(end_marker) <=
objects[-1]['name'].encode('utf-8'))):
break
resp.body = json.dumps(objects).encode('ascii')
constrained = any(
req.params.get(constraint)
for constraint in ('marker', 'end_marker', 'path', 'prefix',
'delimiter'))
if not constrained and len(objects) < req_limit:
self.app.logger.debug('Setting object count to %s' % len(objects))
# prefer the actual listing stats over the potentially outdated
# root stats. This condition is only likely when a sharded
# container is shrinking or in tests; typically a sharded container
# will have more than CONTAINER_LISTING_LIMIT objects so any
# unconstrained listing will be capped by the limit and total
# object stats cannot therefore be inferred from the listing.
resp.headers['X-Container-Object-Count'] = len(objects)
resp.headers['X-Container-Bytes-Used'] = sum(
[o['bytes'] for o in objects])
return resp