def test_valid_api_version(self): version = 'v1' self.assertTrue(constraints.valid_api_version(version)) version = 'v1.0' self.assertTrue(constraints.valid_api_version(version)) version = 'v2' self.assertFalse(constraints.valid_api_version(version))
def __call__(self, env, start_response): """ WSGI entry point. Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env) if not self.memcache_client: self.logger.warning( _('Warning: Cannot ratelimit without a memcached client')) return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if not valid_api_version(version): return self.app(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: return self.app(env, start_response) else: return ratelimit_resp(env, start_response)
def get_controller(self, req): """ Get the controller to handle a request. :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ print 'req.path',req.path if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) print 'd',d return InfoController, d version, account, container, obj = split_path(req.path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) print 'd',d #print 'valid_api_version(version)',valid_api_version(version) if account and not valid_api_version(version): raise APIVersionError('Invalid path') if obj and container and account: info = get_container_info(req.environ, self) print 'info of obj,Acc,Con',info policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) print 'policy_index',policy_index policy = POLICIES.get_by_index(policy_index) print 'policy',policy if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') return self.obj_controller_router[policy], d elif container and account: print 'container & account, returning containercontroller',container,account return ContainerController, d elif account and not container and not obj: print 'account, returning accountcontroller',account return AccountController, d return None, d
def get_controller(self, req): """ Get the controller to handle a request. :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d #分割路径信息 version, account, container, obj = split_path(req.path, 1, 4, True) #生成包含version、account、container、object的路径字典,用于返回 d = dict(version=version, account_name=account, container_name=container, object_name=obj) if account and not valid_api_version(version): raise APIVersionError('Invalid path') #如果是对象操作 if obj and container and account: #获取container信息 info = get_container_info(req.environ, self) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) #通过index获取存储策略对象 policy = POLICIES.get_by_index(policy_index) if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') #返回对象操作的控制器对象,以及路径字典 return self.obj_controller_router[policy], d #如果是container操作,返回container控制器,以及路径字典 elif container and account: return ContainerController, d #如果是account操作,返回account控制器,以及路径字典 elif account and not container and not obj: return AccountController, d return None, d
def get_controller(self, req): """ Get the controller to handle a request. :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d version, account, container, obj = split_path(req.path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) if account and not valid_api_version(version): raise APIVersionError('Invalid path') if obj and container and account: info = get_container_info(req.environ, self) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) policy = POLICIES.get_by_index(policy_index) if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') return self.obj_controller_router[policy], d elif container and account: return ContainerController, d elif account and not container and not obj: return AccountController, d return None, d
def is_proxy_fs(self, env): if 'pfs.is_bimodal' in env: return env['pfs.is_bimodal'] try: # Need at least an account vers, a, c, o = utils.split_path(env['PATH_INFO'], 2, 4, True) if not constraints.valid_api_version(vers): raise ValueError except ValueError: # If it's not a swift request, it can't be a ProxyFS request return False account_info = get_account_info(env, self.base_app, swift_source='') # ignore status; rely on something elsewhere in the pipeline # to propagate the error return utils.config_true_value( account_info["sysmeta"].get('proxyfs-bimodal'))
def __call__(self, req): vrs, acc, con, obj = utils.parse_path(req.path) if not acc or not constraints.valid_api_version(vrs): # could be a GET /info request or something made up by some # other middleware; get out of the way. return self.app try: is_bimodal, owner_addrinfo = self._fetch_owning_proxyfs(req, acc) except (utils.RpcError, utils.RpcTimeout) as err: return swob.HTTPServiceUnavailable( request=req, headers={'Content-Type': 'text/plain'}, body=str(err)) # Other middlewares will find and act on this req.environ[utils.ENV_IS_BIMODAL] = is_bimodal req.environ[utils.ENV_OWNING_PROXYFS] = owner_addrinfo req.environ[utils.ENV_BIMODAL_CHECKER] = self return self.app
def __call__(self, env, start_response): req = Request(env) try: (api_version, account, container, obj) = req.split_path(2, 4, True) bad_path = False except ValueError: bad_path = True # use of bad_path bool is to avoid recursive tracebacks if bad_path or not valid_api_version(api_version): return self.app(env, start_response) try: if not container: return self.account_request(req, api_version, account, start_response) if container and not obj: return self.container_request(req, start_response) else: return self.object_request(req, api_version, account, container, obj)(env, start_response) except HTTPException as error_response: return error_response(env, start_response)
def __call__(self, req): try: (version, account, container, obj) = \ split_path(req.path_info, 4, 4, True) except ValueError: return req.get_response(self.app) # Only worry about data fetches, not uploads. if not valid_api_version(version) or req.method not in ('GET', 'HEAD'): return req.get_response(self.app) # Get all roles that apply to the user making the request roles = set() if (req.environ.get('HTTP_X_IDENTITY_STATUS') == 'Confirmed' or \ req.environ.get('HTTP_X_SERVICE_IDENTITY_STATUS') in \ (None, "Confirmed")): roles = set(list_from_csv(req.environ.get('HTTP_X_ROLES', ''))) # If we have one of the "nostrip" roles, then don't do any stripping if roles.intersection(self.nostrip_roles): return req.get_response(self.app) # Perform the request and grab a response object that we can work # with resp = req.get_response(self.app) # Check that the requested object is actually a CAIDA avro file conttype = resp.headers.get("Content-Type", None) if conttype is None: return resp if not conttype.startswith("application/vnd.caida."): return resp if not conttype.endswith(".avro"): return resp dtype = conttype.replace("application/vnd.caida.", "", 1)[:-5] if dtype not in self.defaultstrip: return resp # Start by planning to strip all fields for this datatype that have # been explicitly appeared in the config file. Then for each role that # the user has, remove any fields from the strip set that should be # retained for that role. tostrip = self.defaultstrip[dtype] for r in roles: if r not in self.dontstrip: # No specified config for this role, so leave strip set as is continue if dtype not in self.dontstrip[r]: continue tostrip = tostrip - self.dontstrip[r][dtype] # Remove the Etag because otherwise swift clients get very upset # about the md5sum of the response body not matching the md5sum # in the Etag header :/ if 'Etag' in resp.headers: del (resp.headers['Etag']) # If we are going to be stripping fields, replace our response # iterable with one that will parse the received Avro and remove # the desired fields. The swift proxy should handle the rest. x = GenericStrippingAvroParser(resp.app_iter, resp.body, tostrip) resp.app_iter = x return resp
def __call__(self, env, start_response): req = Request(env) try: # account and container only version, acct, cont = req.split_path(2, 3) except ValueError: return self.app(env, start_response) if not valid_api_version(version) or req.method not in ('GET', 'HEAD'): return self.app(env, start_response) # OK, definitely have an account/container request. # Get the desired content-type, then force it to a JSON request. try: out_content_type = get_listing_content_type(req) except HTTPException as err: return err(env, start_response) params = req.params params['format'] = 'json' req.params = params status, headers, resp_iter = req.call_application(self.app) header_to_index = {} resp_content_type = resp_length = None for i, (header, value) in enumerate(headers): header = header.lower() if header == 'content-type': header_to_index[header] = i resp_content_type = value.partition(';')[0] elif header == 'content-length': header_to_index[header] = i resp_length = int(value) if not status.startswith('200 '): start_response(status, headers) return resp_iter if resp_content_type != 'application/json': start_response(status, headers) return resp_iter if resp_length is None or \ resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: start_response(status, headers) return resp_iter def set_header(header, value): if value is None: del headers[header_to_index[header]] else: headers[header_to_index[header]] = ( headers[header_to_index[header]][0], str(value)) if req.method == 'HEAD': set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', None) # don't know, can't determine start_response(status, headers) return resp_iter body = b''.join(resp_iter) try: listing = json.loads(body) # Do a couple sanity checks if not isinstance(listing, list): raise ValueError if not all(isinstance(item, dict) for item in listing): raise ValueError except ValueError: # Static web listing that's returning invalid JSON? # Just pass it straight through; that's about all we *can* do. start_response(status, headers) return [body] try: if out_content_type.endswith('/xml'): if cont: body = container_to_xml(listing, cont) else: body = account_to_xml(listing, acct) elif out_content_type == 'text/plain': body = listing_to_text(listing) # else, json -- we continue down here to be sure we set charset except KeyError: # listing was in a bad format -- funky static web listing?? start_response(status, headers) return [body] if not body: status = '%s %s' % (HTTP_NO_CONTENT, RESPONSE_REASONS[HTTP_NO_CONTENT][0]) set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', len(body)) start_response(status, headers) return [body]
def __call__(self, env, start_response): req = Request(env) try: # account and container only version, acct, cont = req.split_path(2, 3) except ValueError: is_account_or_container_req = False else: is_account_or_container_req = True if not is_account_or_container_req: return self.app(env, start_response) if not valid_api_version(version) or req.method not in ('GET', 'HEAD'): return self.app(env, start_response) # OK, definitely have an account/container request. # Get the desired content-type, then force it to a JSON request. try: out_content_type = get_listing_content_type(req) except HTTPException as err: return err(env, start_response) params = req.params can_vary = 'format' not in params params['format'] = 'json' req.params = params # Give other middlewares a chance to be in charge env.setdefault('swift.format_listing', True) status, headers, resp_iter = req.call_application(self.app) if not env.get('swift.format_listing'): start_response(status, headers) return resp_iter header_to_index = {} resp_content_type = resp_length = None for i, (header, value) in enumerate(headers): header = header.lower() if header == 'content-type': header_to_index[header] = i resp_content_type = value.partition(';')[0] elif header == 'content-length': header_to_index[header] = i resp_length = int(value) elif header == 'vary': header_to_index[header] = i if not status.startswith(('200 ', '204 ')): start_response(status, headers) return resp_iter if can_vary: if 'vary' in header_to_index: value = headers[header_to_index['vary']][1] if 'accept' not in list_from_csv(value.lower()): headers[header_to_index['vary']] = ('Vary', value + ', Accept') else: headers.append(('Vary', 'Accept')) if resp_content_type != 'application/json': start_response(status, headers) return resp_iter if resp_length is None or \ resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: start_response(status, headers) return resp_iter def set_header(header, value): if value is None: del headers[header_to_index[header]] else: headers[header_to_index[header]] = ( headers[header_to_index[header]][0], str(value)) if req.method == 'HEAD': set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', None) # don't know, can't determine start_response(status, headers) return resp_iter body = b''.join(resp_iter) try: listing = json.loads(body) # Do a couple sanity checks if not isinstance(listing, list): raise ValueError if not all(isinstance(item, dict) for item in listing): raise ValueError except ValueError: # Static web listing that's returning invalid JSON? # Just pass it straight through; that's about all we *can* do. start_response(status, headers) return [body] if not req.allow_reserved_names: listing = self.filter_reserved(listing, acct, cont) try: if out_content_type.endswith('/xml'): if cont: body = container_to_xml( listing, wsgi_to_bytes(cont).decode('utf-8')) else: body = account_to_xml(listing, wsgi_to_bytes(acct).decode('utf-8')) elif out_content_type == 'text/plain': body = listing_to_text(listing) else: body = json.dumps(listing).encode('ascii') except KeyError: # listing was in a bad format -- funky static web listing?? start_response(status, headers) return [body] if not body: status = '%s %s' % (HTTP_NO_CONTENT, RESPONSE_REASONS[HTTP_NO_CONTENT][0]) set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', len(body)) start_response(status, headers) return [body]
def __call__(self, env, start_response): if time() > self._rtime: self._reload() req = swob.Request(env) try: vers, acct, cont, obj = req.split_path(2, 4, True) except ValueError: return self.app(env, start_response) if req.headers.get(SHUNT_BYPASS_HEADER, ''): self.logger.debug('Bypassing shunt (%s header) for %r', SHUNT_BYPASS_HEADER, req.path_info) return self.app(env, start_response) if not constraints.valid_api_version(vers): return self.app(env, start_response) if not cont: sync_profile = self.sync_profiles.get((acct, '/*')) if req.method == 'GET' and sync_profile and\ sync_profile.get('migration'): # TODO: make the container an optional parameter profile, _ = maybe_munge_profile_for_all_containers( sync_profile, '.stub-container') return self.handle_account(req, start_response, profile, acct) return self.app(env, start_response) sync_profile = next( (self.sync_profiles[(acct, c)] for c in (cont, '/*') if (acct, c) in self.sync_profiles), None) if sync_profile is None: return self.app(env, start_response) sync_profile, per_account = maybe_munge_profile_for_all_containers( sync_profile, cont) if req.method == 'DELETE' and sync_profile.get('migration'): return self.handle_delete(req, start_response, sync_profile, obj, per_account) if not obj: if req.method == 'GET': return self.handle_listing(req, start_response, sync_profile, cont, per_account) if req.method == 'HEAD' and sync_profile.get('migration'): return self.handle_container_head(req, start_response, sync_profile, cont, per_account) if obj and req.method in ('GET', 'HEAD'): # TODO: think about what to do for POST, COPY return self.handle_object(req, start_response, sync_profile, obj, per_account) if req.method == 'POST' and sync_profile.get('migration'): return self.handle_post(req, start_response, sync_profile, obj, per_account) if obj and req.method == 'PUT' and sync_profile.get('migration'): return self.handle_object_put(req, start_response, sync_profile, per_account) return self.app(env, start_response)
def __call__(self, env, start_response): # a lot of this is cribbed from listing_formats / swob.Request if env['REQUEST_METHOD'] != 'GET': # Nothing to translate return self.app(env, start_response) try: v, a, c = split_path(env.get('SCRIPT_NAME', '') + env['PATH_INFO'], 3, 3) if not valid_api_version(v): raise ValueError except ValueError: # not a container request; pass through return self.app(env, start_response) ctx = WSGIContext(self.app) resp_iter = ctx._app_call(env) content_type = content_length = cl_index = None for index, (header, value) in enumerate(ctx._response_headers): header = header.lower() if header == 'content-type': content_type = value.split(';', 1)[0].strip() if content_length: break elif header == 'content-length': cl_index = index try: content_length = int(value) except ValueError: pass # ignore -- we'll bail later if content_type: break if content_type != 'application/json' or content_length is None or \ content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: start_response(ctx._response_status, ctx._response_headers, ctx._response_exc_info) return resp_iter # We've done our sanity checks, slurp the response into memory with closing_if_possible(resp_iter): body = b''.join(resp_iter) try: listing = json.loads(body) for item in listing: if 'subdir' in item: continue value, params = parse_header(item['hash']) if 's3_etag' in params: item['s3_etag'] = '"%s"' % params.pop('s3_etag') item['hash'] = value + ''.join( '; %s=%s' % kv for kv in params.items()) except (TypeError, KeyError, ValueError): # If anything goes wrong above, drop back to original response start_response(ctx._response_status, ctx._response_headers, ctx._response_exc_info) return [body] body = json.dumps(listing).encode('ascii') ctx._response_headers[cl_index] = ( ctx._response_headers[cl_index][0], str(len(body)), ) start_response(ctx._response_status, ctx._response_headers, ctx._response_exc_info) return [body]
def get_controller(self, req): """ Get the controller to handle a request. 获得一个请求的控制器 :param req: the request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ #如果请求为/info{?swiftinfo_sig,swiftinfo_expires}这种格式 #表示请求的是一些info,所以会调用InfoController if req.path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d #获得具体信息 version, account, container, obj = split_path(req.path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) #account存在但是version不对,目前version只能是v1或者是v1.0 if account and not valid_api_version(version): raise APIVersionError('Invalid path') #如果path中account, container和object都存在,返回的是object的controller if obj and container and account: info = get_container_info(req.environ, self) #获得container的存储策略,共有三种策略,在配置文件swift.conf中有,可以去查看 policy_index = req.headers.get('X-Backend-Storage-Policy-Index', info['storage_policy']) policy = POLICIES.get_by_index(policy_index) if not policy: # This indicates that a new policy has been created, # with rings, deployed, released (i.e. deprecated = # False), used by a client to create a container via # another proxy that was restarted after the policy # was released, and is now cached - all before this # worker was HUPed to stop accepting new # connections. There should never be an "unknown" # index - but when there is - it's probably operator # error and hopefully temporary. raise HTTPServiceUnavailable('Unknown Storage Policy') return self.obj_controller_router[policy], d #如果path中只包含account和container,则返回container的controller elif container and account: return ContainerController, d #如果path只存在account,则返回account的controller elif account and not container and not obj: return AccountController, d #都没有,返回None return None, d
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path( 2, 4, rest_with_last=True) is_swifty_request = valid_api_version(version) except ValueError: is_swifty_request = False if not is_swifty_request: return self.app(env, start_response) if not obj: typ = 'Container' if container else 'Account' client_header = 'X-%s-Rfc-Compliant-Etags' % typ sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ if client_header in req.headers: if req.headers[client_header]: req.headers[sysmeta_header] = config_true_value( req.headers[client_header]) else: req.headers[sysmeta_header] = '' if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)): req.headers[sysmeta_header] = '' def translating_start_response(status, headers, exc_info=None): return start_response( status, [(client_header if h.title() == sysmeta_header else h, v) for h, v in headers], exc_info) return self.app(env, translating_start_response) container_info = get_container_info(env, self.app, 'EQ') if not container_info or not is_success(container_info['status']): return self.app(env, start_response) flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags') if flag is None: account_info = get_account_info(env, self.app, 'EQ') if not account_info or not is_success(account_info['status']): return self.app(env, start_response) flag = account_info.get('sysmeta', {}).get('rfc-compliant-etags') if flag is None: flag = self.conf.get('enable_by_default', 'false') if not config_true_value(flag): return self.app(env, start_response) status, headers, resp_iter = req.call_application(self.app) for i, (header, value) in enumerate(headers): if header.lower() == 'etag': if not value.startswith(('"', 'W/"')) or \ not value.endswith('"'): headers[i] = (header, '"%s"' % value) start_response(status, headers) return resp_iter
def __call__(self, req): if req.path == '/info': # Ensure /info requests get the freshest results self.register_info() return self.app try: (version, acc, cont, obj) = req.split_path(3, 4, True) bad_path = False except ValueError: bad_path = True # use of bad_path bool is to avoid recursive tracebacks if bad_path or not valid_api_version(version): return self.app # validate container-sync metdata update info = get_container_info(req.environ, self.app, swift_source='CS') sync_to = req.headers.get('x-container-sync-to') if req.method in ('PUT', 'POST') and cont and not obj: versions_cont = info.get('sysmeta', {}).get('versions-container') if sync_to and versions_cont: raise HTTPBadRequest( 'Cannot configure container sync on a container ' 'with object versioning configured.', request=req) if not self.allow_full_urls: if sync_to and not sync_to.startswith('//'): raise HTTPBadRequest( body='Full URLs are not allowed for X-Container-Sync-To ' 'values. Only realm values of the format ' '//realm/cluster/account/container are allowed.\n', request=req) auth = req.headers.get('x-container-sync-auth') if auth: valid = False auth = auth.split() if len(auth) != 3: req.environ.setdefault('swift.log_info', []).append('cs:not-3-args') else: realm, nonce, sig = auth realm_key = self.realms_conf.key(realm) realm_key2 = self.realms_conf.key2(realm) if not realm_key: req.environ.setdefault('swift.log_info', []).append('cs:no-local-realm-key') else: user_key = info.get('sync_key') if not user_key: req.environ.setdefault( 'swift.log_info', []).append('cs:no-local-user-key') else: # x-timestamp headers get shunted by gatekeeper if 'x-backend-inbound-x-timestamp' in req.headers: req.headers['x-timestamp'] = req.headers.pop( 'x-backend-inbound-x-timestamp') expected = self.realms_conf.get_sig( req.method, req.path, req.headers.get('x-timestamp', '0'), nonce, realm_key, user_key) expected2 = self.realms_conf.get_sig( req.method, req.path, req.headers.get('x-timestamp', '0'), nonce, realm_key2, user_key) if realm_key2 else expected if not streq_const_time(sig, expected) and \ not streq_const_time(sig, expected2): req.environ.setdefault('swift.log_info', []).append('cs:invalid-sig') else: req.environ.setdefault('swift.log_info', []).append('cs:valid') valid = True if not valid: exc = HTTPUnauthorized( body='X-Container-Sync-Auth header not valid; ' 'contact cluster operator for support.', headers={'content-type': 'text/plain'}, request=req) exc.headers['www-authenticate'] = ' '.join([ 'SwiftContainerSync', exc.www_authenticate().split(None, 1)[1] ]) raise exc else: req.environ['swift.authorize_override'] = True # An SLO manifest will already be in the internal manifest # syntax and might be synced before its segments, so stop SLO # middleware from performing the usual manifest validation. req.environ['swift.slo_override'] = True # Similar arguments for static symlinks req.environ['swift.symlink_override'] = True return self.app
def __call__(self, env, start_response): # a lot of this is cribbed from listing_formats / swob.Request if env['REQUEST_METHOD'] != 'GET': # Nothing to translate return self.app(env, start_response) try: v, a, c = split_path( env.get('SCRIPT_NAME', '') + env['PATH_INFO'], 3, 3) if not valid_api_version(v): raise ValueError except ValueError: # not a container request; pass through return self.app(env, start_response) ctx = WSGIContext(self.app) resp_iter = ctx._app_call(env) content_type = content_length = cl_index = None for index, (header, value) in enumerate(ctx._response_headers): header = header.lower() if header == 'content-type': content_type = value.split(';', 1)[0].strip() if content_length: break elif header == 'content-length': cl_index = index try: content_length = int(value) except ValueError: pass # ignore -- we'll bail later if content_type: break if content_type != 'application/json' or content_length is None or \ content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: start_response(ctx._response_status, ctx._response_headers, ctx._response_exc_info) return resp_iter # We've done our sanity checks, slurp the response into memory with closing_if_possible(resp_iter): body = b''.join(resp_iter) try: listing = json.loads(body) for item in listing: if 'subdir' in item: continue value, params = parse_header(item['hash']) if 's3_etag' in params: item['s3_etag'] = '"%s"' % params.pop('s3_etag') item['hash'] = value + ''.join('; %s=%s' % kv for kv in params.items()) except (TypeError, KeyError, ValueError): # If anything goes wrong above, drop back to original response start_response(ctx._response_status, ctx._response_headers, ctx._response_exc_info) return [body] body = json.dumps(listing) ctx._response_headers[cl_index] = ( ctx._response_headers[cl_index][0], str(len(body)), ) start_response(ctx._response_status, ctx._response_headers, ctx._response_exc_info) return [body]