def create_listing(self, req, out_content_type, resp_headers, result, container, **kwargs): container_list = result['objects'] for p in result.get('prefixes', []): record = {'name': p, 'subdir': True} container_list.append(record) container_list.sort(key=lambda x: x['name']) ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') versions = kwargs.get('versions', False) slo = kwargs.get('slo', False) if out_content_type == 'application/json': ret.body = json.dumps([ self.update_data_record(r, versions, slo) for r in container_list ]).encode('utf-8') req.environ['swift.format_listing'] = False elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj, versions, slo) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in [ "name", "hash", "bytes", "content_type", "last_modified" ]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) req.environ['swift.format_listing'] = False else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = ('\n'.join(rec['name'] for rec in container_list) + '\n').encode('utf-8') return ret
def create_listing(self, req, out_content_type, info, metadata, container_list, container): list_meta = get_param(req, 'list_meta', 'f').lower() in TRUE_VALUES resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in metadata.iteritems(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record, list_meta) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj, list_meta) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): if list_meta and field == 'metadata': meta = SubElement(obj_element, field) for k, v in record[field].iteritems(): SubElement(meta, k).text = str( v.decode('utf-8')) else: SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' return ret
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None): """ Given a list of responses from several servers, choose the best to return to the API. :param req: swob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :returns: swob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) > len(statuses) / 2: status = max(hstatuses) status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] if etag: resp.headers['etag'] = etag.strip('"') return resp self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'), {'type': server_type, 'statuses': statuses}) resp.status = '503 Internal Server Error' return resp
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None, headers=None): """ Given a list of responses from several servers, choose the best to return to the API. :param req: swob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :param headers: headers of each response :returns: swob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) >= quorum_size(len(statuses)): status = max(hstatuses) status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] if headers: update_headers(resp, headers[status_index]) if etag: resp.headers['etag'] = etag.strip('"') return resp self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'), {'type': server_type, 'statuses': statuses}) resp.status = '503 Internal Server Error' return resp
def zip_get_content_response(fp, files, boundary=_boundary): def part_with_attachment(fn): part = Response(headers={ 'Content-Type': 'application/octet-stream', 'Content-Disposition': 'attachment; filename=%s' % fn.encode('utf-8'), }) part.body = zip_file.read(fn) return part with zipfile.ZipFile(fp) as zip_file: try: total_size = sum(zip_file.getinfo(fn).file_size for fn in files) except KeyError: raise HTTPNotFound(body='File not found in the zip\r\n') if total_size > MAX_CONTENT_SIZE: raise HTTPRequestEntityTooLarge() if len(files) == 1: resp = part_with_attachment(files[0]) else: resp = Response( content_type='multipart/mixed; boundary=%s' % boundary) body = io.BytesIO() for fn in files: part_resp = part_with_attachment(fn) body.write('\r\n--%s\r\n' % boundary) body.write(dump_response(part_resp)) body.write('\r\n--%s--\r\n' % boundary) resp.body = body.getvalue() return resp
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None): """ Given a list of responses from several servers, choose the best to return to the API. :param req: swob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :returns: swob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) > len(statuses) / 2: status = max(hstatuses) status_index = statuses.index(status) resp.status = "%s %s" % (status, reasons[status_index]) resp.body = bodies[status_index] if etag: resp.headers["etag"] = etag.strip('"') return resp self.app.logger.error(_("%(type)s returning 503 for %(statuses)s"), {"type": server_type, "statuses": statuses}) resp.status = "503 Internal Server Error" return resp
def part_with_attachment(fn): part = Response(headers={ 'Content-Type': 'application/octet-stream', 'Content-Disposition': 'attachment; filename=%s' % fn.encode('utf-8'), }) part.body = zip_file.read(fn) return part
def get_err_response(self, msg='Unable to process requested file'): self.logger.error(msg) resp = Response(content_type='text/xml') resp.status = HTTP_BAD_REQUEST resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \ '<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \ % (HTTP_BAD_REQUEST, msg) return resp
def get_err_response(self, msg="Unable to process requested file"): self.logger.error(msg) resp = Response(content_type="text/xml") resp.status = HTTP_BAD_REQUEST resp.body = ( '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' "<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n" % (HTTP_BAD_REQUEST, msg) ) return resp
def get_err_response(code): """ Given an HTTP response code, create a properly formatted xml error response :param code: error code :returns: webob.response object """ error_table = { 'AccessDenied': (HTTP_FORBIDDEN, 'Access denied'), 'BucketAlreadyExists': (HTTP_CONFLICT, 'The requested bucket name is not available'), 'BucketNotEmpty': (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), 'InvalidArgument': (HTTP_BAD_REQUEST, 'Invalid Argument'), 'InvalidBucketName': (HTTP_BAD_REQUEST, 'The specified bucket is not valid'), 'InvalidURI': (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), 'InvalidDigest': (HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'), 'BadDigest': (HTTP_BAD_REQUEST, 'The Content-Length you specified was invalid'), 'NoSuchBucket': (HTTP_NOT_FOUND, 'The specified bucket does not exist'), 'SignatureDoesNotMatch': (HTTP_FORBIDDEN, 'The calculated request signature does not ' 'match your provided one'), 'RequestTimeTooSkewed': (HTTP_FORBIDDEN, 'The difference between the request time and the' ' current time is too large'), 'NoSuchKey': (HTTP_NOT_FOUND, 'The resource you requested does not exist'), 'Unsupported': (HTTP_NOT_IMPLEMENTED, 'The feature you requested is not yet' ' implemented'), 'MissingContentLength': (HTTP_LENGTH_REQUIRED, 'Length Required'), 'ServiceUnavailable': (HTTP_SERVICE_UNAVAILABLE, 'Please reduce your request rate'), 'IllegalVersioningConfigurationException': (HTTP_BAD_REQUEST, 'The specified versioning configuration invalid'), 'MalformedACLError': (HTTP_BAD_REQUEST, 'The XML you provided was not well-formed or did ' 'not validate against our published schema') } resp = Response(content_type='text/xml') resp.status = error_table[code][0] resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \ '<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \ % (code, error_table[code][1]) return resp
def create_listing(self, req, out_content_type, resp_headers, metadata, result_list, container): container_list = result_list['objects'] for p in result_list.get('prefixes', []): record = {'name': p, 'subdir': True} container_list.append(record) for (k, v) in metadata.iteritems(): if v and (k.lower() in self.pass_through_headers or is_sys_or_user_meta('container', k)): resp_headers[k] = v ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec['name'] for rec in container_list) + '\n' return ret
def create_listing(self, req, out_content_type, info, resp_headers, metadata, container_list, container): for key, (value, timestamp) in metadata.items(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps( [self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in [ "name", "hash", "bytes", "content_type", "last_modified" ]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) return ret
def get_err_response(self, code): """ Given an HTTP response code, create a properly formatted xml error response :param code: error code :returns: webob.response object """ error_table = { 'AccessDenied': (HTTP_FORBIDDEN, 'Access denied'), 'BucketAlreadyExists': (HTTP_CONFLICT, 'The requested bucket name is not available'), 'BucketNotEmpty': (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), 'InvalidArgument': (HTTP_BAD_REQUEST, 'Invalid Argument'), 'InvalidBucketName': (HTTP_BAD_REQUEST, 'The specified bucket is not valid'), 'InvalidURI': (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), 'InvalidDigest': (HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'), 'BadDigest': (HTTP_BAD_REQUEST, 'The Content-Length you specified was invalid'), 'NoSuchBucket': (HTTP_NOT_FOUND, 'The specified bucket does not exist'), 'SignatureDoesNotMatch': (HTTP_FORBIDDEN, 'The calculated request signature does not ' 'match your provided one'), 'RequestTimeTooSkewed': (HTTP_FORBIDDEN, 'The difference between the request time and the' ' current time is too large'), 'NoSuchKey': (HTTP_NOT_FOUND, 'The resource you requested does not exist'), 'NotSuchPolicy': (HTTP_NOT_FOUND, 'The Policy you requested does not exist'), 'NotSuchWebsite': (HTTP_NOT_FOUND, 'The Website you requested does not exist'), 'Unsupported': (HTTP_NOT_IMPLEMENTED, 'The feature you requested is not yet' ' implemented'), 'MissingContentLength': (HTTP_LENGTH_REQUIRED, 'Length Required'), 'ServiceUnavailable': (HTTP_SERVICE_UNAVAILABLE, 'Please reduce your request rate') } resp = Response(content_type='text/xml') resp.status = error_table[code][0] resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' \ '<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n' \ % (code, error_table[code][1]) return resp
def create_listing(self, req, out_content_type, info, resp_headers, metadata, container_list, container): for key, (value, timestamp) in metadata.items(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) return ret
def _deny_request(self, code): error_table = { 'AccessDenied': (401, 'Access denied'), 'InvalidURI': (400, 'Could not parse the specified URI'), } resp = Response(content_type='text/xml') resp.status = error_table[code][0] error_msg = ('<?xml version="1.0" encoding="UTF-8"?>\r\n' '<Error>\r\n <Code>%s</Code>\r\n ' '<Message>%s</Message>\r\n</Error>\r\n' % (code, error_table[code][1])) if six.PY3: error_msg = error_msg.encode() resp.body = error_msg return resp
def get_err_response(code): """ Given an HTTP response code, create a properly formatted xml error response :param code: error code :returns: webob.response object """ error_table = { 'AccessDenied': (HTTP_FORBIDDEN, 'Access denied'), 'BucketAlreadyExists': ( HTTP_CONFLICT, 'The requested bucket name is not available'), 'BucketNotEmpty': ( HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), 'InvalidArgument': (HTTP_BAD_REQUEST, 'Invalid Argument'), 'InvalidBucketName': ( HTTP_BAD_REQUEST, 'The specified bucket is not valid'), 'InvalidURI': (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), 'InvalidDigest': ( HTTP_BAD_REQUEST, 'The Content-MD5 you specified was invalid'), 'BadDigest': ( HTTP_BAD_REQUEST, 'The Content-Length you specified was invalid'), 'EntityTooLarge': ( HTTP_BAD_REQUEST, 'Your proposed upload exceeds the maximum ' 'allowed object size.'), 'NoSuchBucket': ( HTTP_NOT_FOUND, 'The specified bucket does not exist'), 'SignatureDoesNotMatch': ( HTTP_FORBIDDEN, 'The calculated request signature does not match ' 'your provided one'), 'RequestTimeTooSkewed': ( HTTP_FORBIDDEN, 'The difference between the request time and the ' 'current time is too large'), 'NoSuchKey': ( HTTP_NOT_FOUND, 'The resource you requested does not exist'), 'Unsupported': ( HTTP_NOT_IMPLEMENTED, 'The feature you requested is not yet ' 'implemented'), 'MissingContentLength': (HTTP_LENGTH_REQUIRED, 'Length Required'), 'ServiceUnavailable': ( HTTP_SERVICE_UNAVAILABLE, 'Please reduce your request rate')} resp = Response(content_type='text/xml') resp.status = error_table[code][0] resp.body = """%s <Error><Code>%s</Code><Message>%s</Message></Error> """ % (XML_HEADER, code, error_table[code][1]) return resp
def GET(self, req): """ Load ring data from .gz files. """ self._reload() ring_info = self._to_dict() gz_file_to_ring_info = json.dumps(ring_info, sort_keys=True) ring_info_json_md5 = self._ringinfo_md5(gz_file_to_ring_info) resp = Response(request=req) resp.etag = ring_info_json_md5 resp.content_length = len(gz_file_to_ring_info) resp.body = gz_file_to_ring_info resp.status_init = 200 resp.content_type = 'application/json; charset=UTF-8' return resp
def get_err_response(code): """ Given an HTTP response code, create a properly formatted xml error response :param code: error code :returns: webob.response object """ error_table = { "AccessDenied": (HTTP_FORBIDDEN, "Access denied"), "BucketAlreadyExists": (HTTP_CONFLICT, "The requested bucket name is not available"), "BucketNotEmpty": (HTTP_CONFLICT, "The bucket you tried to delete is not empty"), "InvalidArgument": (HTTP_BAD_REQUEST, "Invalid Argument"), "InvalidBucketName": (HTTP_BAD_REQUEST, "The specified bucket is not valid"), "InvalidURI": (HTTP_BAD_REQUEST, "Could not parse the specified URI"), "InvalidDigest": (HTTP_BAD_REQUEST, "The Content-MD5 you specified was invalid"), "BadDigest": (HTTP_BAD_REQUEST, "The Content-Length you specified was invalid"), "NoSuchObject": (HTTP_NOT_FOUND, "The specified object does not exist"), "NoSuchBucket": (HTTP_NOT_FOUND, "The specified bucket does not exist"), "NoSuchUpload": (HTTP_NOT_FOUND, "The specified bucket does not exist"), "SignatureDoesNotMatch": ( HTTP_FORBIDDEN, "The calculated request signature does not " "match your provided one", ), "RequestTimeTooSkewed": ( HTTP_FORBIDDEN, "The difference between the request time and the" " current time is too large", ), "NoSuchKey": (HTTP_NOT_FOUND, "The resource you requested does not exist"), "Unsupported": (HTTP_NOT_IMPLEMENTED, "The feature you requested is not yet" " implemented"), "MissingContentLength": (HTTP_LENGTH_REQUIRED, "Length Required"), "ServiceUnavailable": (HTTP_SERVICE_UNAVAILABLE, "Please reduce your request rate"), } resp = Response(content_type="text/xml") resp.status = error_table[code][0] resp.body = ( '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n ' "<Code>%s</Code>\r\n <Message>%s</Message>\r\n</Error>\r\n" % (code, error_table[code][1]) ) return resp
(obj_status, objct) = get_object(storage_url, token, cont, obj) except ClientException, e: resp = Response(charset='utf8') resp.status = e.http_status return resp except err: pass resp = Response() resp.set_cookie('_token', token, path=self.page_path, max_age=self.cookie_max_age, secure=self.secure) resp.status = HTTP_OK resp.headers = obj_status resp.body = objct self.token_bank[token].update({'msg': ''}) self.memcache_update(token) return resp return HTTPFound(location=self.add_prefix(storage_url)) def page_cont_list(self, req, storage_url, token, template=None): """ """ if template is None: tmpl = self.tmpl path = urlparse(self.del_prefix(req.url)).path if len(path.split('/')) <= 2: path = urlparse(storage_url).path vrs, acc, cont, obj = split_path(path, 1, 4, True) lang = self.get_lang(req) base = self.add_prefix(urlparse(storage_url).path)
'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or key.lower().startswith('x-container-meta-')): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == 'application/json': ret.body = json.dumps( [self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in [ "name", "hash", "bytes", "content_type", "last_modified" ]: SubElement(obj_element, field).text = str(
def GET(self, req): """Handle HTTP GET request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed(request=req, body='Maximum limit is %d' % CONTAINER_LISTING_LIMIT) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == 'application/json': ret.body = json.dumps( [self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in [ "name", "hash", "bytes", "content_type", "last_modified" ]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' return ret
def _GET_using_cache(self, req): # It may be possible to fulfil the request from cache: we only reach # here if request record_type is 'shard' or 'auto', so if the container # state is 'sharded' then look for cached shard ranges. However, if # X-Newest is true then we always fetch from the backend servers. get_newest = config_true_value(req.headers.get('x-newest', False)) if get_newest: self.app.logger.debug( 'Skipping shard cache lookup (x-newest) for %s', req.path_qs) info = None else: info = _get_info_from_caches(self.app, req.environ, self.account_name, self.container_name) if (info and is_success(info['status']) and info.get('sharding_state') == 'sharded'): # container is sharded so we may have the shard ranges cached headers = headers_from_container_info(info) if headers: # only use cached values if all required headers available infocache = req.environ.setdefault('swift.infocache', {}) memcache = cache_from_env(req.environ, True) cache_key = get_cache_key(self.account_name, self.container_name, shard='listing') cached_ranges = infocache.get(cache_key) if cached_ranges is None and memcache: cached_ranges = memcache.get(cache_key) if cached_ranges is not None: infocache[cache_key] = tuple(cached_ranges) # shard ranges can be returned from cache self.app.logger.debug('Found %d shards in cache for %s', len(cached_ranges), req.path_qs) headers.update({ 'x-backend-record-type': 'shard', 'x-backend-cached-results': 'true' }) shard_range_body = self._filter_resp_shard_ranges( req, cached_ranges) # mimic GetOrHeadHandler.get_working_response... # note: server sets charset with content_type but proxy # GETorHEAD_base does not, so don't set it here either resp = Response(request=req, body=shard_range_body) update_headers(resp, headers) resp.last_modified = math.ceil( float(headers['x-put-timestamp'])) resp.environ['swift_x_timestamp'] = headers.get( 'x-timestamp') resp.accept_ranges = 'bytes' resp.content_type = 'application/json' return resp # The request was not fulfilled from cache so send to the backend # server, but instruct the backend server to ignore name constraints in # request params if returning shard ranges so that the response can # potentially be cached. Only do this if the container state is # 'sharded'. We don't attempt to cache shard ranges for a 'sharding' # container as they may include the container itself as a 'gap filler' # for shard ranges that have not yet cleaved; listings from 'gap # filler' shard ranges are likely to become stale as the container # continues to cleave objects to its shards and caching them is # therefore more likely to result in stale or incomplete listings on # subsequent container GETs. req.headers['x-backend-override-shard-name-filter'] = 'sharded' resp = self._GETorHEAD_from_backend(req) sharding_state = resp.headers.get('x-backend-sharding-state', '').lower() resp_record_type = resp.headers.get('x-backend-record-type', '').lower() complete_listing = config_true_value( resp.headers.pop('x-backend-override-shard-name-filter', False)) # given that we sent 'x-backend-override-shard-name-filter=sharded' we # should only receive back 'x-backend-override-shard-name-filter=true' # if the sharding state is 'sharded', but check them both anyway... if (resp_record_type == 'shard' and sharding_state == 'sharded' and complete_listing): # backend returned unfiltered listing state shard ranges so parse # them and replace response body with filtered listing cache_key = get_cache_key(self.account_name, self.container_name, shard='listing') data = self._parse_listing_response(req, resp) backend_shard_ranges = self._parse_shard_ranges(req, data, resp) if backend_shard_ranges is not None: cached_ranges = [dict(sr) for sr in backend_shard_ranges] if resp.headers.get('x-backend-sharding-state') == 'sharded': # cache in infocache even if no shard ranges returned; this # is unexpected but use that result for this request infocache = req.environ.setdefault('swift.infocache', {}) infocache[cache_key] = tuple(cached_ranges) memcache = cache_from_env(req.environ, True) if memcache and cached_ranges: # cache in memcache only if shard ranges as expected self.app.logger.debug('Caching %d shards for %s', len(cached_ranges), req.path_qs) memcache.set( cache_key, cached_ranges, time=self.app.recheck_listing_shard_ranges) # filter returned shard ranges according to request constraints resp.body = self._filter_resp_shard_ranges(req, cached_ranges) return resp
resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or key.lower().startswith('x-container-meta-')): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record.keys()):
def GET(self, req): """Handle HTTP GET request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = constraints.CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > constraints.CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % constraints.CONTAINER_LISTING_LIMIT) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' return ret
if path_type == 4: # object try: (obj_status, objct) = get_object(storage_url, token, cont, obj) except ClientException, e: resp = Response(charset='utf8') resp.status = e.http_status return resp except err: pass resp = Response() resp.set_cookie('_token', token, path=self.page_path, max_age=self.cookie_max_age, secure=self.secure) resp.status = HTTP_OK resp.headers = obj_status resp.body = objct self.token_bank[token].update({'msg': ''}) self.memcache_update(token) return resp return HTTPFound(location=self.add_prefix(storage_url)) def page_cont_list(self, req, storage_url, token, template=None): """ """ if template is None: tmpl = self.tmpl path = urlparse(self.del_prefix(req.url)).path if len(path.split('/')) <= 2: path = urlparse(storage_url).path vrs, acc, cont, obj = split_path(path, 1, 4, True) lang = self.get_lang(req) base = self.add_prefix(urlparse(storage_url).path)
if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() resp_headers = { "X-Container-Object-Count": info["object_count"], "X-Container-Bytes-Used": info["bytes_used"], "X-Timestamp": info["created_at"], "X-PUT-Timestamp": info["put_timestamp"], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or key.lower().startswith("x-container-meta-")): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset="utf-8") container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == "application/json": ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith("/xml"): doc = Element("container", name=container.decode("utf-8")) for obj in container_list: record = self.update_data_record(obj) if "subdir" in record: name = record["subdir"].decode("utf-8") sub = SubElement(doc, "subdir", name=name) SubElement(sub, "name").text = name else: obj_element = SubElement(doc, "object") for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str(record.pop(field)).decode("utf-8") for field in sorted(record.keys()): SubElement(obj_element, field).text = str(record[field]).decode("utf-8") ret.body = tostring(doc, encoding="UTF-8")