def create_listing(self, req, out_content_type, resp_headers, result,
container, **kwargs):
container_list = result['objects']
for p in result.get('prefixes', []):
record = {'name': p, 'subdir': True}
container_list.append(record)
container_list.sort(key=lambda x: x['name'])
ret = Response(request=req,
headers=resp_headers,
content_type=out_content_type,
charset='utf-8')
versions = kwargs.get('versions', False)
slo = kwargs.get('slo', False)
if out_content_type == 'application/json':
ret.body = json.dumps([
self.update_data_record(r, versions, slo)
for r in container_list
]).encode('utf-8')
req.environ['swift.format_listing'] = False
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj, versions, slo)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in [
"name", "hash", "bytes", "content_type",
"last_modified"
]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
req.environ['swift.format_listing'] = False
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = ('\n'.join(rec['name'] for rec in container_list) +
'\n').encode('utf-8')
return ret
def create_listing(self, req, out_content_type, info, metadata,
container_list, container):
list_meta = get_param(req, 'list_meta', 'f').lower() in TRUE_VALUES
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in metadata.iteritems():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record, list_meta)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj, list_meta)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
if list_meta and field == 'metadata':
meta = SubElement(obj_element, field)
for k, v in record[field].iteritems():
SubElement(meta, k).text = str(
v.decode('utf-8'))
else:
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
return ret
def best_response(self, req, statuses, reasons, bodies, server_type,
etag=None):
"""
Given a list of responses from several servers, choose the best to
return to the API.
:param req: swob.Request object
:param statuses: list of statuses returned
:param reasons: list of reasons for each status
:param bodies: bodies of each response
:param server_type: type of server the responses came from
:param etag: etag
:returns: swob.Response object with the correct status, body, etc. set
"""
resp = Response(request=req)
if len(statuses):
for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST):
hstatuses = \
[s for s in statuses if hundred <= s < hundred + 100]
if len(hstatuses) > len(statuses) / 2:
status = max(hstatuses)
status_index = statuses.index(status)
resp.status = '%s %s' % (status, reasons[status_index])
resp.body = bodies[status_index]
if etag:
resp.headers['etag'] = etag.strip('"')
return resp
self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'),
{'type': server_type, 'statuses': statuses})
resp.status = '503 Internal Server Error'
return resp
def best_response(self, req, statuses, reasons, bodies, server_type,
etag=None, headers=None):
"""
Given a list of responses from several servers, choose the best to
return to the API.
:param req: swob.Request object
:param statuses: list of statuses returned
:param reasons: list of reasons for each status
:param bodies: bodies of each response
:param server_type: type of server the responses came from
:param etag: etag
:param headers: headers of each response
:returns: swob.Response object with the correct status, body, etc. set
"""
resp = Response(request=req)
if len(statuses):
for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST):
hstatuses = \
[s for s in statuses if hundred <= s < hundred + 100]
if len(hstatuses) >= quorum_size(len(statuses)):
status = max(hstatuses)
status_index = statuses.index(status)
resp.status = '%s %s' % (status, reasons[status_index])
resp.body = bodies[status_index]
if headers:
update_headers(resp, headers[status_index])
if etag:
resp.headers['etag'] = etag.strip('"')
return resp
self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'),
{'type': server_type, 'statuses': statuses})
resp.status = '503 Internal Server Error'
return resp
def zip_get_content_response(fp, files, boundary=_boundary):
def part_with_attachment(fn):
part = Response(headers={
'Content-Type': 'application/octet-stream',
'Content-Disposition': 'attachment; filename=%s' %
fn.encode('utf-8'),
})
part.body = zip_file.read(fn)
return part
with zipfile.ZipFile(fp) as zip_file:
try:
total_size = sum(zip_file.getinfo(fn).file_size for fn in files)
except KeyError:
raise HTTPNotFound(body='File not found in the zip\r\n')
if total_size > MAX_CONTENT_SIZE:
raise HTTPRequestEntityTooLarge()
if len(files) == 1:
resp = part_with_attachment(files[0])
else:
resp = Response(
content_type='multipart/mixed; boundary=%s' % boundary)
body = io.BytesIO()
for fn in files:
part_resp = part_with_attachment(fn)
body.write('\r\n--%s\r\n' % boundary)
body.write(dump_response(part_resp))
body.write('\r\n--%s--\r\n' % boundary)
resp.body = body.getvalue()
return resp
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None):
"""
Given a list of responses from several servers, choose the best to
return to the API.
:param req: swob.Request object
:param statuses: list of statuses returned
:param reasons: list of reasons for each status
:param bodies: bodies of each response
:param server_type: type of server the responses came from
:param etag: etag
:returns: swob.Response object with the correct status, body, etc. set
"""
resp = Response(request=req)
if len(statuses):
for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST):
hstatuses = [s for s in statuses if hundred <= s < hundred + 100]
if len(hstatuses) > len(statuses) / 2:
status = max(hstatuses)
status_index = statuses.index(status)
resp.status = "%s %s" % (status, reasons[status_index])
resp.body = bodies[status_index]
if etag:
resp.headers["etag"] = etag.strip('"')
return resp
self.app.logger.error(_("%(type)s returning 503 for %(statuses)s"), {"type": server_type, "statuses": statuses})
resp.status = "503 Internal Server Error"
return resp
def part_with_attachment(fn):
part = Response(headers={
'Content-Type': 'application/octet-stream',
'Content-Disposition': 'attachment; filename=%s' %
fn.encode('utf-8'),
})
part.body = zip_file.read(fn)
return part
def get_err_response(self, msg='Unable to process requested file'):
self.logger.error(msg)
resp = Response(content_type='text/xml')
resp.status = HTTP_BAD_REQUEST
resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \
'<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \
% (HTTP_BAD_REQUEST, msg)
return resp
def get_err_response(self, msg="Unable to process requested file"):
self.logger.error(msg)
resp = Response(content_type="text/xml")
resp.status = HTTP_BAD_REQUEST
resp.body = (
'<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n '
"<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n" % (HTTP_BAD_REQUEST, msg)
)
return resp
def get_err_response(code):
"""
Given an HTTP response code, create a properly formatted xml error response
:param code: error code
:returns: webob.response object
"""
error_table = {
'AccessDenied':
(HTTP_FORBIDDEN, 'Access denied'),
'BucketAlreadyExists':
(HTTP_CONFLICT, 'The requested bucket name is not available'),
'BucketNotEmpty':
(HTTP_CONFLICT, 'The bucket you tried to delete is not empty'),
'InvalidArgument':
(HTTP_BAD_REQUEST, 'Invalid Argument'),
'InvalidBucketName':
(HTTP_BAD_REQUEST, 'The specified bucket is not valid'),
'InvalidURI':
(HTTP_BAD_REQUEST, 'Could not parse the specified URI'),
'InvalidDigest':
(HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'),
'BadDigest':
(HTTP_BAD_REQUEST, 'The Content-Length you specified was invalid'),
'NoSuchBucket':
(HTTP_NOT_FOUND, 'The specified bucket does not exist'),
'SignatureDoesNotMatch':
(HTTP_FORBIDDEN, 'The calculated request signature does not '
'match your provided one'),
'RequestTimeTooSkewed':
(HTTP_FORBIDDEN, 'The difference between the request time and the'
' current time is too large'),
'NoSuchKey':
(HTTP_NOT_FOUND, 'The resource you requested does not exist'),
'Unsupported':
(HTTP_NOT_IMPLEMENTED, 'The feature you requested is not yet'
' implemented'),
'MissingContentLength':
(HTTP_LENGTH_REQUIRED, 'Length Required'),
'ServiceUnavailable':
(HTTP_SERVICE_UNAVAILABLE, 'Please reduce your request rate'),
'IllegalVersioningConfigurationException':
(HTTP_BAD_REQUEST, 'The specified versioning configuration invalid'),
'MalformedACLError':
(HTTP_BAD_REQUEST, 'The XML you provided was not well-formed or did '
'not validate against our published schema')
}
resp = Response(content_type='text/xml')
resp.status = error_table[code][0]
resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \
'<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \
% (code, error_table[code][1])
return resp
def create_listing(self, req, out_content_type, resp_headers,
metadata, result_list, container):
container_list = result_list['objects']
for p in result_list.get('prefixes', []):
record = {'name': p,
'subdir': True}
container_list.append(record)
for (k, v) in metadata.iteritems():
if v and (k.lower() in self.pass_through_headers or
is_sys_or_user_meta('container', k)):
resp_headers[k] = v
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec['name'] for rec in container_list) + '\n'
return ret
def create_listing(self, req, out_content_type, info, resp_headers,
metadata, container_list, container):
for key, (value, timestamp) in metadata.items():
if value and (key.lower() in self.save_headers
or is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req,
headers=resp_headers,
content_type=out_content_type,
charset='utf-8')
if out_content_type == 'application/json':
ret.body = json.dumps(
[self.update_data_record(record) for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in [
"name", "hash", "bytes", "content_type",
"last_modified"
]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
return ret
def get_err_response(self, code):
"""
Given an HTTP response code, create a properly formatted xml error response
:param code: error code
:returns: webob.response object
"""
error_table = {
'AccessDenied': (HTTP_FORBIDDEN, 'Access denied'),
'BucketAlreadyExists':
(HTTP_CONFLICT, 'The requested bucket name is not available'),
'BucketNotEmpty':
(HTTP_CONFLICT, 'The bucket you tried to delete is not empty'),
'InvalidArgument': (HTTP_BAD_REQUEST, 'Invalid Argument'),
'InvalidBucketName': (HTTP_BAD_REQUEST,
'The specified bucket is not valid'),
'InvalidURI': (HTTP_BAD_REQUEST,
'Could not parse the specified URI'),
'InvalidDigest': (HTTP_BAD_REQUEST,
'The Content-MD5 you specified was invalid'),
'BadDigest': (HTTP_BAD_REQUEST,
'The Content-Length you specified was invalid'),
'NoSuchBucket': (HTTP_NOT_FOUND,
'The specified bucket does not exist'),
'SignatureDoesNotMatch':
(HTTP_FORBIDDEN, 'The calculated request signature does not '
'match your provided one'),
'RequestTimeTooSkewed':
(HTTP_FORBIDDEN, 'The difference between the request time and the'
' current time is too large'),
'NoSuchKey': (HTTP_NOT_FOUND,
'The resource you requested does not exist'),
'NotSuchPolicy': (HTTP_NOT_FOUND,
'The Policy you requested does not exist'),
'NotSuchWebsite': (HTTP_NOT_FOUND,
'The Website you requested does not exist'),
'Unsupported': (HTTP_NOT_IMPLEMENTED,
'The feature you requested is not yet'
' implemented'),
'MissingContentLength': (HTTP_LENGTH_REQUIRED, 'Length Required'),
'ServiceUnavailable': (HTTP_SERVICE_UNAVAILABLE,
'Please reduce your request rate')
}
resp = Response(content_type='text/xml')
resp.status = error_table[code][0]
resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \
'<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \
% (code, error_table[code][1])
return resp
def create_listing(self, req, out_content_type, info, resp_headers,
metadata, container_list, container):
for key, (value, timestamp) in metadata.items():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
return ret
def _deny_request(self, code):
error_table = {
'AccessDenied': (401, 'Access denied'),
'InvalidURI': (400, 'Could not parse the specified URI'),
}
resp = Response(content_type='text/xml')
resp.status = error_table[code][0]
error_msg = ('<?xml version="1.0" encoding="UTF-8"?>\r\n'
'<Error>\r\n <Code>%s</Code>\r\n '
'<Message>%s</Message>\r\n</Error>\r\n' %
(code, error_table[code][1]))
if six.PY3:
error_msg = error_msg.encode()
resp.body = error_msg
return resp
def get_err_response(code):
"""
Given an HTTP response code, create a properly formatted xml error response
:param code: error code
:returns: webob.response object
"""
error_table = {
'AccessDenied': (HTTP_FORBIDDEN, 'Access denied'),
'BucketAlreadyExists': (
HTTP_CONFLICT, 'The requested bucket name is not available'),
'BucketNotEmpty': (
HTTP_CONFLICT, 'The bucket you tried to delete is not empty'),
'InvalidArgument': (HTTP_BAD_REQUEST, 'Invalid Argument'),
'InvalidBucketName': (
HTTP_BAD_REQUEST, 'The specified bucket is not valid'),
'InvalidURI': (HTTP_BAD_REQUEST, 'Could not parse the specified URI'),
'InvalidDigest': (
HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'),
'BadDigest': (
HTTP_BAD_REQUEST, 'The Content-Length you specified was invalid'),
'EntityTooLarge': (
HTTP_BAD_REQUEST, 'Your proposed upload exceeds the maximum '
'allowed object size.'),
'NoSuchBucket': (
HTTP_NOT_FOUND, 'The specified bucket does not exist'),
'SignatureDoesNotMatch': (
HTTP_FORBIDDEN, 'The calculated request signature does not match '
'your provided one'),
'RequestTimeTooSkewed': (
HTTP_FORBIDDEN, 'The difference between the request time and the '
'current time is too large'),
'NoSuchKey': (
HTTP_NOT_FOUND, 'The resource you requested does not exist'),
'Unsupported': (
HTTP_NOT_IMPLEMENTED, 'The feature you requested is not yet '
'implemented'),
'MissingContentLength': (HTTP_LENGTH_REQUIRED, 'Length Required'),
'ServiceUnavailable': (
HTTP_SERVICE_UNAVAILABLE, 'Please reduce your request rate')}
resp = Response(content_type='text/xml')
resp.status = error_table[code][0]
resp.body = """%s
<Error><Code>%s</Code><Message>%s</Message></Error>
""" % (XML_HEADER, code, error_table[code][1])
return resp
def GET(self, req):
"""
Load ring data from .gz files.
"""
self._reload()
ring_info = self._to_dict()
gz_file_to_ring_info = json.dumps(ring_info, sort_keys=True)
ring_info_json_md5 = self._ringinfo_md5(gz_file_to_ring_info)
resp = Response(request=req)
resp.etag = ring_info_json_md5
resp.content_length = len(gz_file_to_ring_info)
resp.body = gz_file_to_ring_info
resp.status_init = 200
resp.content_type = 'application/json; charset=UTF-8'
return resp
def get_err_response(code):
"""
Given an HTTP response code, create a properly formatted xml error response
:param code: error code
:returns: webob.response object
"""
error_table = {
"AccessDenied": (HTTP_FORBIDDEN, "Access denied"),
"BucketAlreadyExists": (HTTP_CONFLICT, "The requested bucket name is not available"),
"BucketNotEmpty": (HTTP_CONFLICT, "The bucket you tried to delete is not empty"),
"InvalidArgument": (HTTP_BAD_REQUEST, "Invalid Argument"),
"InvalidBucketName": (HTTP_BAD_REQUEST, "The specified bucket is not valid"),
"InvalidURI": (HTTP_BAD_REQUEST, "Could not parse the specified URI"),
"InvalidDigest": (HTTP_BAD_REQUEST, "The Content-MD5 you specified was invalid"),
"BadDigest": (HTTP_BAD_REQUEST, "The Content-Length you specified was invalid"),
"NoSuchObject": (HTTP_NOT_FOUND, "The specified object does not exist"),
"NoSuchBucket": (HTTP_NOT_FOUND, "The specified bucket does not exist"),
"NoSuchUpload": (HTTP_NOT_FOUND, "The specified bucket does not exist"),
"SignatureDoesNotMatch": (
HTTP_FORBIDDEN,
"The calculated request signature does not " "match your provided one",
),
"RequestTimeTooSkewed": (
HTTP_FORBIDDEN,
"The difference between the request time and the" " current time is too large",
),
"NoSuchKey": (HTTP_NOT_FOUND, "The resource you requested does not exist"),
"Unsupported": (HTTP_NOT_IMPLEMENTED, "The feature you requested is not yet" " implemented"),
"MissingContentLength": (HTTP_LENGTH_REQUIRED, "Length Required"),
"ServiceUnavailable": (HTTP_SERVICE_UNAVAILABLE, "Please reduce your request rate"),
}
resp = Response(content_type="text/xml")
resp.status = error_table[code][0]
resp.body = (
'<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n '
"<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n" % (code, error_table[code][1])
)
return resp
(obj_status, objct) = get_object(storage_url, token, cont, obj)
except ClientException, e:
resp = Response(charset='utf8')
resp.status = e.http_status
return resp
except err:
pass
resp = Response()
resp.set_cookie('_token',
token,
path=self.page_path,
max_age=self.cookie_max_age,
secure=self.secure)
resp.status = HTTP_OK
resp.headers = obj_status
resp.body = objct
self.token_bank[token].update({'msg': ''})
self.memcache_update(token)
return resp
return HTTPFound(location=self.add_prefix(storage_url))
def page_cont_list(self, req, storage_url, token, template=None):
""" """
if template is None:
tmpl = self.tmpl
path = urlparse(self.del_prefix(req.url)).path
if len(path.split('/')) <= 2:
path = urlparse(storage_url).path
vrs, acc, cont, obj = split_path(path, 1, 4, True)
lang = self.get_lang(req)
base = self.add_prefix(urlparse(storage_url).path)
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers
or key.lower().startswith('x-container-meta-')):
resp_headers[key] = value
ret = Response(request=req,
headers=resp_headers,
content_type=out_content_type,
charset='utf-8')
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
ret.body = json.dumps(
[self.update_data_record(record) for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in [
"name", "hash", "bytes", "content_type",
"last_modified"
]:
SubElement(obj_element, field).text = str(
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(request=req,
body='Maximum limit is %d' %
CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive,
part,
account,
container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers
or is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req,
headers=resp_headers,
content_type=out_content_type,
charset='utf-8')
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
ret.body = json.dumps(
[self.update_data_record(record) for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in [
"name", "hash", "bytes", "content_type",
"last_modified"
]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
return ret
def _GET_using_cache(self, req):
# It may be possible to fulfil the request from cache: we only reach
# here if request record_type is 'shard' or 'auto', so if the container
# state is 'sharded' then look for cached shard ranges. However, if
# X-Newest is true then we always fetch from the backend servers.
get_newest = config_true_value(req.headers.get('x-newest', False))
if get_newest:
self.app.logger.debug(
'Skipping shard cache lookup (x-newest) for %s', req.path_qs)
info = None
else:
info = _get_info_from_caches(self.app, req.environ,
self.account_name,
self.container_name)
if (info and is_success(info['status'])
and info.get('sharding_state') == 'sharded'):
# container is sharded so we may have the shard ranges cached
headers = headers_from_container_info(info)
if headers:
# only use cached values if all required headers available
infocache = req.environ.setdefault('swift.infocache', {})
memcache = cache_from_env(req.environ, True)
cache_key = get_cache_key(self.account_name,
self.container_name,
shard='listing')
cached_ranges = infocache.get(cache_key)
if cached_ranges is None and memcache:
cached_ranges = memcache.get(cache_key)
if cached_ranges is not None:
infocache[cache_key] = tuple(cached_ranges)
# shard ranges can be returned from cache
self.app.logger.debug('Found %d shards in cache for %s',
len(cached_ranges), req.path_qs)
headers.update({
'x-backend-record-type': 'shard',
'x-backend-cached-results': 'true'
})
shard_range_body = self._filter_resp_shard_ranges(
req, cached_ranges)
# mimic GetOrHeadHandler.get_working_response...
# note: server sets charset with content_type but proxy
# GETorHEAD_base does not, so don't set it here either
resp = Response(request=req, body=shard_range_body)
update_headers(resp, headers)
resp.last_modified = math.ceil(
float(headers['x-put-timestamp']))
resp.environ['swift_x_timestamp'] = headers.get(
'x-timestamp')
resp.accept_ranges = 'bytes'
resp.content_type = 'application/json'
return resp
# The request was not fulfilled from cache so send to the backend
# server, but instruct the backend server to ignore name constraints in
# request params if returning shard ranges so that the response can
# potentially be cached. Only do this if the container state is
# 'sharded'. We don't attempt to cache shard ranges for a 'sharding'
# container as they may include the container itself as a 'gap filler'
# for shard ranges that have not yet cleaved; listings from 'gap
# filler' shard ranges are likely to become stale as the container
# continues to cleave objects to its shards and caching them is
# therefore more likely to result in stale or incomplete listings on
# subsequent container GETs.
req.headers['x-backend-override-shard-name-filter'] = 'sharded'
resp = self._GETorHEAD_from_backend(req)
sharding_state = resp.headers.get('x-backend-sharding-state',
'').lower()
resp_record_type = resp.headers.get('x-backend-record-type',
'').lower()
complete_listing = config_true_value(
resp.headers.pop('x-backend-override-shard-name-filter', False))
# given that we sent 'x-backend-override-shard-name-filter=sharded' we
# should only receive back 'x-backend-override-shard-name-filter=true'
# if the sharding state is 'sharded', but check them both anyway...
if (resp_record_type == 'shard' and sharding_state == 'sharded'
and complete_listing):
# backend returned unfiltered listing state shard ranges so parse
# them and replace response body with filtered listing
cache_key = get_cache_key(self.account_name,
self.container_name,
shard='listing')
data = self._parse_listing_response(req, resp)
backend_shard_ranges = self._parse_shard_ranges(req, data, resp)
if backend_shard_ranges is not None:
cached_ranges = [dict(sr) for sr in backend_shard_ranges]
if resp.headers.get('x-backend-sharding-state') == 'sharded':
# cache in infocache even if no shard ranges returned; this
# is unexpected but use that result for this request
infocache = req.environ.setdefault('swift.infocache', {})
infocache[cache_key] = tuple(cached_ranges)
memcache = cache_from_env(req.environ, True)
if memcache and cached_ranges:
# cache in memcache only if shard ranges as expected
self.app.logger.debug('Caching %d shards for %s',
len(cached_ranges), req.path_qs)
memcache.set(
cache_key,
cached_ranges,
time=self.app.recheck_listing_shard_ranges)
# filter returned shard ranges according to request constraints
resp.body = self._filter_resp_shard_ranges(req, cached_ranges)
return resp
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-')):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record.keys()):
def GET(self, req):
"""Handle HTTP GET request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
path = get_param(req, 'path')
prefix = get_param(req, 'prefix')
delimiter = get_param(req, 'delimiter')
if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
# delimiters can be made more flexible later
return HTTPPreconditionFailed(body='Bad delimiter')
marker = get_param(req, 'marker', '')
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
return HTTPPreconditionFailed(
request=req,
body='Maximum limit is %d'
% constraints.CONTAINER_LISTING_LIMIT)
out_content_type = get_listing_content_type(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers,
content_type=out_content_type, charset='utf-8')
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
ret.body = json.dumps([self.update_data_record(record)
for record in container_list])
elif out_content_type.endswith('/xml'):
doc = Element('container', name=container.decode('utf-8'))
for obj in container_list:
record = self.update_data_record(obj)
if 'subdir' in record:
name = record['subdir'].decode('utf-8')
sub = SubElement(doc, 'subdir', name=name)
SubElement(sub, 'name').text = name
else:
obj_element = SubElement(doc, 'object')
for field in ["name", "hash", "bytes", "content_type",
"last_modified"]:
SubElement(obj_element, field).text = str(
record.pop(field)).decode('utf-8')
for field in sorted(record):
SubElement(obj_element, field).text = str(
record[field]).decode('utf-8')
ret.body = tostring(doc, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
return ret
if path_type == 4: # object
try:
(obj_status, objct) = get_object(storage_url, token, cont, obj)
except ClientException, e:
resp = Response(charset='utf8')
resp.status = e.http_status
return resp
except err:
pass
resp = Response()
resp.set_cookie('_token', token, path=self.page_path,
max_age=self.cookie_max_age,
secure=self.secure)
resp.status = HTTP_OK
resp.headers = obj_status
resp.body = objct
self.token_bank[token].update({'msg': ''})
self.memcache_update(token)
return resp
return HTTPFound(location=self.add_prefix(storage_url))
def page_cont_list(self, req, storage_url, token, template=None):
""" """
if template is None:
tmpl = self.tmpl
path = urlparse(self.del_prefix(req.url)).path
if len(path.split('/')) <= 2:
path = urlparse(storage_url).path
vrs, acc, cont, obj = split_path(path, 1, 4, True)
lang = self.get_lang(req)
base = self.add_prefix(urlparse(storage_url).path)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
resp_headers = {
"X-Container-Object-Count": info["object_count"],
"X-Container-Bytes-Used": info["bytes_used"],
"X-Timestamp": info["created_at"],
"X-PUT-Timestamp": info["put_timestamp"],
}
for key, (value, timestamp) in broker.metadata.iteritems():
if value and (key.lower() in self.save_headers or key.lower().startswith("x-container-meta-")):
resp_headers[key] = value
ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset="utf-8")
container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path)
if out_content_type == "application/json":
ret.body = json.dumps([self.update_data_record(record) for record in container_list])
elif out_content_type.endswith("/xml"):
doc = Element("container", name=container.decode("utf-8"))
for obj in container_list:
record = self.update_data_record(obj)
if "subdir" in record:
name = record["subdir"].decode("utf-8")
sub = SubElement(doc, "subdir", name=name)
SubElement(sub, "name").text = name
else:
obj_element = SubElement(doc, "object")
for field in ["name", "hash", "bytes", "content_type", "last_modified"]:
SubElement(obj_element, field).text = str(record.pop(field)).decode("utf-8")
for field in sorted(record.keys()):
SubElement(obj_element, field).text = str(record[field]).decode("utf-8")
ret.body = tostring(doc, encoding="UTF-8")
