def __call__(self, env, start_response): # If override is set in env, then just pass along if config_true_value(env.get('swift.crypto.override')): return self.app(env, start_response) req = Request(env) if self.disable_encryption and req.method in ('PUT', 'POST'): return self.app(env, start_response) try: req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) if req.method in ('GET', 'HEAD'): handler = EncrypterObjContext(self, self.logger).handle_get_or_head elif req.method == 'PUT': handler = EncrypterObjContext(self, self.logger).handle_put elif req.method == 'POST': handler = EncrypterObjContext(self, self.logger).handle_post else: # anything else return self.app(env, start_response) try: return handler(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response)
def __call__(self, env, start_response): if config_true_value(env.get('swift.crypto.override')): return self.app(env, start_response) req = Request(env) if self.disable_encryption and req.method in ('PUT', 'POST'): return self.app(env, start_response) try: req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) fetch_crypto_keys = env.get(CRYPTO_KEY_CALLBACK) if fetch_crypto_keys is not None: try: fetch_crypto_keys() except HTTPException as exc: if MISSING_KEY_MSG in exc.body: if req.method in ('PUT', 'POST'): # No key, just upload without encryption env['swift.crypto.override'] = True # else: # let the thing fail later, # if a key is required for decoding else: raise except Exception: # Let the parent class handle other exceptions pass res = super(Encrypter, self).__call__(env, start_response) return res
def __call__(self, env, start_response): self.logger.debug('Initialising gate middleware') req = Request(env) try: version, account = req.split_path(1, 3, True) except ValueError: return HttpNotFound(request=req) if account is 'gate': # Handles direct calls to gate return HttpOk if 'X-Gate-Verify' in env: verify = env['X-Gate-Verify'] self.logger.debug('Verification request: %s algorithms: %s' % (req.path, verify)) try: version, account, container, obj = req.split_path(4, 4, True) except ValueError: return HTTPBadRequest(request=req) algorithms = verify.split(',') for algo in algorithms: metakey = 'X-Object-Meta-Gate-%s' % algo.upper() if metakey not in env: self.logger.debug('Invalid verification request, object missing: %s' % (metakey)) return HTTPBadRequest(request=req) if publish_verify(req.path, algorithms): for algo in algorithms: statuskey = 'X-Object-Meta-Gate-Verify-%s-Status' % algo.upper() env[statuskey] = 'Queued' env['X-Object-Meta-Gate-Verify'] = verify if 'X-Gate-Process' in env: module = env['X-Gate-Process'] self.logger.debug('Process request: %s module: %s' % (req.path, module)) try: version, case, container, obj = req.split_path(4, 4, True) except ValueError: return HTTPBadRequest(request=req) if publish_process(req.path, algorithms): for algo in algorithms: env['X-Object-Meta-Gate-Process'] = module env['X-Object-Meta-Gate-Process-Status'] = 'Queued' # TODO: Get reponse to see if a fake object reponse = self.app(env, start_response) return reponse
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env["swift.copy_hook"] = self.copy_hook( env.get("swift.copy_hook", lambda src_req, src_resp, sink_req: src_resp) ) try: if req.method == "PUT" and req.params.get("multipart-manifest") == "put": return self.handle_multipart_put(req, start_response) if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete": return self.handle_multipart_delete(req)(env, start_response) if req.method == "GET" or req.method == "HEAD": return self.handle_multipart_get_or_head(req, start_response) if "X-Static-Large-Object" in req.headers: raise HTTPBadRequest( request=req, body="X-Static-Large-Object is a reserved header. " "To create a static large object add query param " "multipart-manifest=put.", ) except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): # making a duplicate, because if this is a COPY request, we will # modify the PATH_INFO to find out if the 'Destination' is in a # versioned container req = Request(env.copy()) try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) # In case allow_versioned_writes is set in the filter configuration, # the middleware becomes the authority on whether object # versioning is enabled or not. In case it is not set, then # the option in the container configuration is still checked # for backwards compatibility # For a container request, first just check if option is set, # can be either true or false. # If set, check if enabled when actually trying to set container # header. If not set, let request be handled by container server # for backwards compatibility. # For an object request, also check if option is set (either T or F). # If set, check if enabled when checking versions container in # sysmeta property. If it is not set check 'versions' property in # container_info allow_versioned_writes = self.conf.get('allow_versioned_writes') if allow_versioned_writes and container and not obj: return self.container_request(req, start_response, allow_versioned_writes) elif obj and req.method in ('PUT', 'COPY', 'DELETE'): return self.object_request( req, version, account, container, obj, allow_versioned_writes)(env, start_response) else: return self.app(env, start_response)
def get_creds(self, environ): req = Request(environ) try: parts = req.split_path(1, 4, True) _, account, _, _ = parts except ValueError: account = None env_identity = self._integral_keystone_identity(environ) if not env_identity: # user identity is not confirmed. (anonymous?) creds = { 'identity': None, 'is_authoritative': (account and account.startswith(self.reseller_prefix)) } return creds tenant_id, tenant_name = env_identity['tenant'] user_id, user_name = env_identity['user'] roles = [r.strip() for r in env_identity.get('roles', [])] account = self._get_account_for_tenant(tenant_id) is_admin = (tenant_name == user_name) creds = { "identity": env_identity, "roles": roles, "account": account, "tenant_id": tenant_id, "tenant_name": tenant_name, "user_id": user_id, "user_name": user_name, "is_admin": is_admin } return creds
def __call__(self, env, start_response): req = Request(env) # We want to check POST here # it can possibly have content_length > 0 if not self.enforce_quota or req.method not in ("POST", "PUT", "COPY"): return self.app(env, start_response) account_info = get_account_info(req.environ, self.app, swift_source='litequota') if not account_info: return self.app(env, start_response) service_plan = assemble_from_partial(self.metadata_key, account_info['meta']) try: ver, account, container, obj = \ req.split_path(2, 4, rest_with_last=True) except ValueError: return self.app(env, start_response) if not service_plan and req.method == 'PUT' and not obj: service_plan = self.set_serviceplan(req, account) if not service_plan: return self.app(env, start_response) try: service_plan = json.loads(service_plan) except ValueError: return self.app(env, start_response) if service_plan.get('storage', None): resp = self.apply_storage_quota(req, service_plan['storage'], account_info, ver, account, container, obj) if resp: return resp(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env['swift.copy_hook'] = self.copy_hook( env.get('swift.copy_hook', lambda src_req, src_resp, sink_req: src_resp)) if ((req.method == 'GET' or req.method == 'HEAD') and req.params.get('multipart-manifest') != 'get'): return GetContext(self, self.logger).\ handle_request(req, start_response) elif req.method == 'PUT': error_response = self.validate_x_object_manifest_header( req, start_response) if error_response: return error_response(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point. Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ req = Request(env) if self.memcache_client is None: # 在pipline中memcache middleware应该在前面 # memcache为每一个请求建立一个memcache client,并把这个client放入env中 self.memcache_client = cache_from_env(env) if not self.memcache_client: self.logger.warning( _('Warning: Cannot ratelimit without a memcached client')) return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) # ratelimit_resp 为 None 或者 Response 对象,Response 对象是可调用的。 # ratelimit 操作正常执行,或者不需要执行,返回 None,接着向下调用。 if ratelimit_resp is None: return self.app(env, start_response) else: # ratelimit 执行失败,或者需要返回错误信息,返回 Response 对象。 # 不用向下调用,直接返回。 return ratelimit_resp(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) if DEBUG: print('obj:%s' % obj) except ValueError: return self.app(env, start_response) try: if env['REQUEST_METHOD'] == "PUT": # Read the object content and enc_key in memory # encrypt the object with the enc_key content = env['wsgi.input'].read() if content: return Response(status=403, body="content %s detected" % content, content_type="text/plain")(env, start_response) else: return Response(status=403, body="content not detected", content_type="text/plain")(env, start_response) if DEBUG: print(content) #encryptor = Encryptor() content = content + "yyyyyyyyyyyyyyyyy" # Maybe add the decrytion process later #elif env['REQUEST_METHOD'] == "GET": except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env['swift.copy_hook'] = self.copy_hook( env.get('swift.copy_hook', lambda src_req, src_resp, sink_req: src_resp)) try: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': return self.handle_multipart_put(req, start_response) if req.method == 'DELETE' and \ req.params.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req)(env, start_response) if req.method == 'GET' or req.method == 'HEAD': return self.handle_multipart_get_or_head(req, start_response) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: (version, account, container, obj) = req.split_path(4, 4, True) except ValueError: # If obj component is not present in req, do not proceed further. return self.app(env, start_response) self.account_name = account self.container_name = container self.object_name = obj # Save off original request method (COPY/POST) in case it gets mutated # into PUT during handling. This way logging can display the method # the client actually sent. req.environ['swift.orig_req_method'] = req.method if req.method == 'PUT' and req.headers.get('X-Copy-From'): return self.handle_PUT(req, start_response) elif req.method == 'COPY': return self.handle_COPY(req, start_response) elif req.method == 'POST' and self.object_post_as_copy: return self.handle_object_post_as_copy(req, start_response) elif req.method == 'OPTIONS': # Does not interfere with OPTIONS response from (account,container) # servers and /info response. return self.handle_OPTIONS(req, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: parts = req.split_path(2, 4, True) except ValueError: return self.app(env, start_response) if req.method in ('PUT', 'POST', 'GET', 'HEAD'): # handle only those request methods that may require keys km_context = KeyMasterContext(self, req, *parts[1:]) try: return km_context.handle_request(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response) except KeyError as err: if 'object' in err.args: self.app.logger.debug( 'Missing encryption key, cannot handle request') raise HTTPBadRequest(MISSING_KEY_MSG) else: raise # anything else return self.app(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point. Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env) if not self.memcache_client: self.logger.warning( _('Warning: Cannot ratelimit without a memcached client')) return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if not valid_api_version(version): return self.app(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: return self.app(env, start_response) else: return ratelimit_resp(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) try: if obj: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': return self.handle_multipart_put(req, start_response) if req.method == 'DELETE' and \ req.params.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req)(env, start_response) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: (version, account, container, obj) = req.split_path(4, 4, True) except ValueError: # If obj component is not present in req, do not proceed further. return self.app(env, start_response) try: # In some cases, save off original request method since it gets # mutated into PUT during handling. This way logging can display # the method the client actually sent. if req.method == 'PUT' and req.headers.get('X-Copy-From'): return self.handle_PUT(req, start_response) elif req.method == 'COPY': req.environ['swift.orig_req_method'] = req.method return self.handle_COPY(req, start_response, account, container, obj) elif req.method == 'OPTIONS': # Does not interfere with OPTIONS response from # (account,container) servers and /info response. return self.handle_OPTIONS(req, start_response) except HTTPException as e: return e(req.environ, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): self.logger.debug('Calling Utilization Middleware') req = Request(env) if self.check_api_call(env): return self.GET(req)(env, start_response) try: version, account, container, obj = req.split_path(2, 4, True) except ValueError: return self.app(env, start_response) remote_user = env.get('REMOTE_USER') if not remote_user or (isinstance(remote_user, basestring) and remote_user.startswith('.wsgi')): self.logger.debug('### SKIP: REMOTE_USER is %s' % remote_user) return self.app(env, start_response) start_response_args = [None] input_proxy = InputProxy(env['wsgi.input']) env['wsgi.input'] = input_proxy def my_start_response(status, headers, exc_info=None): start_response_args[0] = (status, list(headers), exc_info) def iter_response(iterable): iterator = iter(iterable) try: chunk = next(iterator) while not chunk: chunk = next(iterator) except StopIteration: chunk = '' if start_response_args[0]: start_response(*start_response_args[0]) bytes_sent = 0 try: while chunk: bytes_sent += len(chunk) yield chunk chunk = next(iterator) finally: try: self.publish_sample(env, account, input_proxy.bytes_received, bytes_sent) except Exception: self.logger.exception('Failed to publish samples') try: iterable = self.app(env, my_start_response) except Exception: self.publish_sample(env, account, input_proxy.bytes_received, 0) raise else: return iter_response(iterable)
def __call__(self, env, start_response): req = Request(env) try: (version, account, container, obj) = req.split_path(4, 4, True) except ValueError: # If obj component is not present in req, do not proceed further. return self.app(env, start_response) self.version = version self.account_name = account try: # Check if fastcopy is possible # only plain object as source and destination # FIXME: handle COPY method (with Destination-* headers) check, source_resp = self.fast_copy_allowed(req) if check: self.logger.debug("COPY: fast copy allowed") env['HTTP_OIO_COPY_FROM'] = unquote(env['HTTP_X_COPY_FROM']) del env['HTTP_X_COPY_FROM'] # handle metadata if not config_true_value(req.headers.get('x-fresh-metadata', 'false')): # cf copy middlware exclude_headers = ('x-static-large-object', 'etag', 'x-object-manifest', 'content-type', 'x-timestamp', 'x-backend-timestamp') # copy original headers but don't overwrite source headers copy_header_subset( source_resp, req, lambda k: k.lower() not in exclude_headers and k.lower() not in req.headers) def _start_response(status, headers, exc_info=None): headers.append(('X-Copied-From-Account', env.get('HTTP_X_COPY_FROM_ACCOUNT', self.account_name))) headers.append(('X-Copied-From', quote(env['HTTP_OIO_COPY_FROM']))) start_response(status, headers, exc_info) return self.app(env, _start_response) except HTTPException as exc: return exc(req.environ, start_response) return super(OioServerSideCopyMiddleware, self).__call__( env, start_response)
def __call__(self, env, start_response): req = Request(env) try: parts = req.split_path(2, 4, True) except ValueError: return self.app(env, start_response) if req.method in ('PUT', 'POST', 'GET', 'HEAD'): # handle only those request methods that may require keys km_context = KeyMasterContext(self, *parts[1:]) try: return km_context.handle_request(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response) # anything else return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: version, acc, cont, obj = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) try: if obj: # object context context = SymlinkObjectContext(self.app, self.logger, self.symloop_max) return context.handle_object(req, start_response) else: # container context context = SymlinkContainerContext(self.app, self.logger) return context.handle_container(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response)
def __call__(self, env, start_response): if env['REQUEST_METHOD'] != 'GET': return self.app(env, start_response) req = Request(env) version, account, container, obj = req.split_path(1, 4, True) if None in [version, account, container, obj]: return self.app(env, start_response) if random.randint(1, self.max_range) > self.count_threshold: return self.app(env, start_response) return_status = [None, None] start_t = time.time() def start_response_wc(status, headers): return_status[0] = status content_len = 0 for item in headers: if item[0] == "Content-Length": content_len = long(item[1]) headers.append(('objslo_start_time', start_t)) headers.append(('objslo_end_time', time.time())) return_status[1] = content_len return start_response(status, headers) if env['REQUEST_METHOD'] == 'GET': server_ip = env['SERVER_NAME'] server_port = str(env['SERVER_PORT']) device = ":".join([server_ip, server_port]) self.redis_client.hincrby(self.incomingreqkey, device, 1 * self.sample_ratio) data = self.app(env, start_response_wc) duration = time.time() - start_t if self.durationswitch: self.redis_client.lpush("%s_duration" % (device), duration) if duration > self.slolatency: self.redis_client.hincrby(self.sloviolatecounter, device, 1 * self.sample_ratio) else: self.redis_client.hincrby(self.slomeetcounter, device, 1 * self.sample_ratio) content_len = return_status[1] self.redis_client.hincrby(self.returnreqkey, device, 1 * self.sample_ratio) self.redis_client.hincrby(self.returnreqsizekey, device, content_len * self.sample_ratio) self.redis_client.hincrby(self.returnreqlatencykey, device, int(duration*1000) * self.sample_ratio) self.redis_client.hincrby(self.readcountskey, device, (int(content_len/self.blocksize) + 1) * self.sample_ratio) return data
def __call__(self, env, start_response): req = Request(env) try: parts = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) if parts[3] and req.method in ('GET', 'HEAD'): handler = DecrypterObjContext(self, self.logger).handle elif parts[2] and req.method == 'GET': handler = DecrypterContContext(self, self.logger).handle else: # url and/or request verb is not handled by decrypter return self.app(env, start_response) try: return handler(req, start_response) except HTTPException as err_resp: return err_resp(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) if ((req.method == 'GET' or req.method == 'HEAD') and req.params.get('multipart-manifest') != 'get'): return GetContext(self, self.logger).\ handle_request(req, start_response) elif req.method == 'PUT': error_response = self._validate_x_object_manifest_header(req) if error_response: return error_response(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): request = Request(env) if request.method == 'PUT': try: version, account, container, obj = \ request.split_path(1, 4, True) except ValueError: return self.app(env, start_response) # check container creation request if account and container and not obj: policy_name = request.headers.get('X-Storage-Policy', '') default_policy = POLICIES.default.name if (policy_name in self.policies) or \ (policy_name == '' and default_policy in self.policies): container = unquote(container) if len(container) > constraints. \ SOF_MAX_CONTAINER_NAME_LENGTH: resp = HTTPBadRequest(request=request) resp.body = \ 'Container name length of %d longer than %d' % \ (len(container), constraints.SOF_MAX_CONTAINER_NAME_LENGTH) return resp(env, start_response) elif account and container and obj: # check object creation request obj = unquote(obj) container_info = get_container_info( env, self.app) policy = POLICIES.get_by_index( container_info['storage_policy']) if policy.name in self.policies: error_response = sof_check_object_creation(request, obj) if error_response: self.logger.warn("returning error: %s", error_response) return error_response(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): request = Request(env) if not request.path.startswith(self.endpoint_path): return self.app(env, start_response) if request.method != 'GET': resp = HTTPMethodNotAllowed(req=request, headers={"Allow": "GET"}) return resp(env, start_response) try: self.config.read(self.conf["__file__"]) except (IOError, KeyError): resp = HTTPServerError(req=request, body="An error occurred", content_type="text/plain") return resp(env, start_response) config_dict = self._config_parser_to_nested_dict() try: endpoint, section, option = request.split_path(1, 3, False) except ValueError: resp = HTTPBadRequest(req=request, headers={}) return resp(env, start_response) try: if section and option: config_dict = {section: {option: config_dict[section][option]}} elif section and not option: config_dict = {section: config_dict[section]} except KeyError: resp = HTTPNotFound(req=request, body="Requested values aren't available", content_type="text/plain") return resp(env, start_response) resp = Response(req=request, body=json.dumps(config_dict), content_type="application/json") return resp(env, start_response)
def __call__(self, env, start_response): req = Request(env) if req.method not in self.write_methods: return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if req.method == 'COPY' and 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') account = check_account_format(req, dest_account) if self.account_read_only(req, account): msg = 'Writes are disabled for this account.' return HTTPMethodNotAllowed(body=msg)(env, start_response) return self.app(env, start_response)
def get_target(self, environ): req = Request(environ) try: parts = req.split_path(1, 4, True) version, account, container, obj = parts except ValueError: version = account = container = obj = None referrers, acls = swift_acl.parse_acl(getattr(req, 'acl', None)) target = { "req": req, "method": req.method.lower(), "version": version, "account": account, "container": container, "object": obj, "acls": acls, "referrers": referrers } return target
def __call__(self, env, start_response): req = Request(env) new_service = env.get('liteauth.new_service', None) if new_service: account_id = env.get('REMOTE_USER', '') if not account_id: return HTTPInternalServerError() if not self.activate_service(account_id, new_service, req.environ): return HTTPInternalServerError() if req.method in ['PUT', 'POST' ] and not 'x-zerovm-execute' in req.headers: account_info = get_account_info(req.environ, self.app, swift_source='litequota') service_plan = assemble_from_partial(self.metadata_key, account_info['meta']) if service_plan: try: service_plan = json.loads(service_plan) path_parts = req.split_path(2, 4, rest_with_last=True) except ValueError: return self.app(env, start_response) if len(path_parts) == 3: quota = service_plan['storage']['containers'] new_size = int(account_info['container_count']) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge( body='Over quota: containers')(env, start_response) else: new_size = int(account_info['bytes']) + (req.content_length or 0) quota = service_plan['storage']['bytes'] if 0 <= quota < new_size: return HTTPRequestEntityTooLarge( body='Over quota: bytes')(env, start_response) quota = service_plan['storage']['objects'] new_size = int(account_info['total_object_count']) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge( body='Over quota: objects')(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): request = Request(env) if request.method == 'PUT': try: version, account, container, obj = \ request.split_path(1, 4, True) except ValueError: return self.app(env, start_response) # check container creation request if account and container and not obj: policy_name = request.headers.get('X-Storage-Policy', '') default_policy = POLICIES.default.name if (policy_name in self.policies) or \ (policy_name == '' and default_policy in self.policies): container = unquote(container) if len(container) > constraints. \ SOF_MAX_CONTAINER_NAME_LENGTH: resp = HTTPBadRequest(request=request) resp.body = \ 'Container name length of %d longer than %d' % \ (len(container), constraints.SOF_MAX_CONTAINER_NAME_LENGTH) return resp(env, start_response) elif account and container and obj: # check object creation request obj = unquote(obj) container_info = get_container_info(env, self.app) policy = POLICIES.get_by_index( container_info['storage_policy']) if policy.name in self.policies: error_response = sof_check_object_creation(request, obj) if error_response: self.logger.warn("returning error: %s", error_response) return error_response(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): # making a duplicate, because if this is a COPY request, we will # modify the PATH_INFO to find out if the 'Destination' is in a # versioned container req = Request(env.copy()) try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) # In case allow_versioned_writes is set in the filter configuration, # the middleware becomes the authority on whether object # versioning is enabled or not. In case it is not set, then # the option in the container configuration is still checked # for backwards compatibility # For a container request, first just check if option is set, # can be either true or false. # If set, check if enabled when actually trying to set container # header. If not set, let request be handled by container server # for backwards compatibility. # For an object request, also check if option is set (either T or F). # If set, check if enabled when checking versions container in # sysmeta property. If it is not set check 'versions' property in # container_info allow_versioned_writes = self.conf.get('allow_versioned_writes') if allow_versioned_writes and container and not obj: try: return self.container_request(req, start_response, allow_versioned_writes) except HTTPException as error_response: return error_response(env, start_response) elif obj and req.method in ('PUT', 'COPY', 'DELETE'): try: return self.object_request(req, version, account, container, obj, allow_versioned_writes)( env, start_response) except HTTPException as error_response: return error_response(env, start_response) else: return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: (api_version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) # In case allow_versioned_writes is set in the filter configuration, # the middleware becomes the authority on whether object # versioning is enabled or not. In case it is not set, then # the option in the container configuration is still checked # for backwards compatibility # For a container request, first just check if option is set, # can be either true or false. # If set, check if enabled when actually trying to set container # header. If not set, let request be handled by container server # for backwards compatibility. # For an object request, also check if option is set (either T or F). # If set, check if enabled when checking versions container in # sysmeta property. If it is not set check 'versions' property in # container_info allow_versioned_writes = self.conf.get('allow_versioned_writes') if allow_versioned_writes and container and not obj: try: return self.container_request(req, start_response, allow_versioned_writes) except HTTPException as error_response: return error_response(env, start_response) elif (obj and (req.method in ('PUT', 'DELETE') and not req.environ.get('swift.post_as_copy') or req.method in ('HEAD', 'GET'))): try: return self.object_request( req, api_version, account, container, obj, allow_versioned_writes)(env, start_response) except HTTPException as error_response: return error_response(env, start_response) else: return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: (api_version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) # In case allow_versioned_writes is set in the filter configuration, # the middleware becomes the authority on whether object # versioning is enabled or not. In case it is not set, then # the option in the container configuration is still checked # for backwards compatibility # For a container request, first just check if option is set, # can be either true or false. # If set, check if enabled when actually trying to set container # header. If not set, let request be handled by container server # for backwards compatibility. # For an object request, also check if option is set (either T or F). # If set, check if enabled when checking versions container in # sysmeta property. If it is not set check 'versions' property in # container_info allow_versioned_writes = self.conf.get('allow_versioned_writes') if allow_versioned_writes and container and not obj: try: return self.container_request(req, start_response, allow_versioned_writes) except HTTPException as error_response: return error_response(env, start_response) elif (obj and (req.method in ('PUT', 'DELETE') and not req.environ.get('swift.post_as_copy') or req.method in ('HEAD', 'GET', 'POST'))): try: return self.object_request( req, api_version, account, container, obj, allow_versioned_writes)(env, start_response) except HTTPException as error_response: return error_response(env, start_response) else: return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) new_service = env.get('liteauth.new_service', None) if new_service: account_id = env.get('REMOTE_USER', '') if not account_id: return HTTPInternalServerError() if not self.activate_service(account_id, new_service, req.environ): return HTTPInternalServerError() if req.method in ['PUT', 'POST'] and not 'x-zerovm-execute' in req.headers: account_info = get_account_info(req.environ, self.app, swift_source='litequota') service_plan = assemble_from_partial(self.metadata_key, account_info['meta']) if service_plan: try: service_plan = json.loads(service_plan) path_parts = req.split_path(2, 4, rest_with_last=True) except ValueError: return self.app(env, start_response) if len(path_parts) == 3: quota = service_plan['storage']['containers'] new_size = int(account_info['container_count']) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge( body='Over quota: containers')(env, start_response) else: new_size = int(account_info['bytes']) + (req.content_length or 0) quota = service_plan['storage']['bytes'] if 0 <= quota < new_size: return HTTPRequestEntityTooLarge( body='Over quota: bytes')(env, start_response) quota = service_plan['storage']['objects'] new_size = int(account_info['total_object_count']) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge( body='Over quota: objects')(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): request = Request(env.copy()) if request.method != "GET": return self.app(env, start_response) (version, acc, con, obj) = request.split_path(1, 4, False) if not obj: return self.app(env, start_response) memcache_key = 'RequestCache/%s/%s/%s' % (acc, con, obj) cached_content = self.memcache.get(memcache_key) if cached_content: response = Response(request=request, body=cached_content) response.headers['X-RequestCache'] = 'True' return response(env, start_response) sub_req = wsgi.make_subrequest(env) sub_resp = sub_req.get_response(self.app) self.memcache.set(memcache_key, sub_resp.body, time=86400.0) return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: (api_version, account, container, obj) = req.split_path(2, 4, True) bad_path = False except ValueError: bad_path = True # use of bad_path bool is to avoid recursive tracebacks if bad_path or not valid_api_version(api_version): return self.app(env, start_response) try: if not container: return self.account_request(req, api_version, account, start_response) if container and not obj: return self.container_request(req, start_response) else: return self.object_request(req, api_version, account, container, obj)(env, start_response) except HTTPException as error_response: return error_response(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ if env.get('swift.slo_override'): return self.app(env, start_response) req = Request(env) try: vrs, account, container, obj = req.split_path(3, 4, True) except ValueError: return self.app(env, start_response) if not obj: if req.method == 'GET': return self.handle_container_listing(req, start_response) return self.app(env, start_response) try: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': return self.handle_multipart_put(req, start_response) if req.method == 'DELETE' and \ req.params.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req)(env, start_response) if req.method == 'GET' or req.method == 'HEAD': return self.handle_multipart_get_or_head(req, start_response) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point. Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env) if not self.memcache_client: self.logger.warning( _('Warning: Cannot ratelimit without a memcached client')) return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: return self.app(env, start_response) else: return ratelimit_resp(env, start_response)
def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env['swift.copy_hook'] = self.copy_hook( env.get('swift.copy_hook', lambda src_req, src_resp, sink_req: src_resp)) if ((req.method == 'GET' or req.method == 'HEAD') and req.params.get('multipart-manifest') != 'get'): return GetContext(self, self.logger).\ handle_request(req, start_response) elif req.method == 'PUT': error_response = self._validate_x_object_manifest_header(req) if error_response: return error_response(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) # Split request path to determine version, account, container, object try: (version, account, container, obj) = req.split_path(2, 4, True) except ValueError: self.logger.debug('split_path exception') return self.app(env, start_response) self.logger.debug(':%s:%s:%s:%s:', version, account, container, obj) # If request is not HLM request or not a GET, it is not processed # by this middleware method = req.method query = req.query_string or '' if not (method == 'POST' and ('MIGRATE' in query or 'RECALL' in query) or method == 'GET'): return self.app(env, start_response) # Process GET object data request, if object is migrated return error # code 412 'Precondition Failed' (consider using 455 'Method Not Valid # in This State') - the error code is returned if any object replica is # migrated. # TODO: provide option to return error code only if all replicas are # migrated, and redirect get request to one of non-migrated replicas if req.method == "GET" and obj and 'STATUS' not in query: # check status and either let GET proceed or return error code rc, out, replicas_status = self.get_object_replicas_status( req, account, container, obj) if rc == REMOTE_STATUS: #send the replica status to requester node return Response(status=HTTP_OK, body=out, content_type="text/plain")(env, start_response) self.logger.debug('replicas_status %s', str(replicas_status)) ret_error = False for replica_status in replicas_status: status = literal_eval(replica_status)['status'] if status not in ['resident', 'premigrated']: ret_error = True if ret_error: return Response(status=HTTP_PRECONDITION_FAILED, body="Object %s needs to be RECALL-ed before " "it can be accessed.\n" % literal_eval(replicas_status[0])['object'], content_type="text/plain")(env, start_response) return self.app(env, start_response) # Process POST request to migrate/recall object elif method == 'POST' and obj: if 'MIGRATE' in query or 'RECALL' in query: if 'MIGRATE' in query: hlm_req = 'MIGRATE' hlm_backend = self.migrate_backend elif 'RECALL' in query: hlm_req = 'RECALL' hlm_backend = self.recall_backend # submit hlm request for object replicas status, out = self.submit_object_replicas_migration_recall( req, account, container, obj, hlm_req, hlm_backend) self.logger.debug('submit_object_replicas_migration_recall()') if status == SUBMITTED_FORWARDED_REQUEST: self.logger.debug('SUBMITTED_FORWARDED_REQUEST') return Response(status=HTTP_OK, body='Accepted remote replica HLM request', content_type="text/plain")(env, start_response) elif status == FAILED_SUBMITTING_REQUEST: self.logger.debug('FAILED_SUBMITTING_REQUEST') return Response(status=HTTP_INTERNAL_SERVER_ERROR, body=out, content_type="text/plain")(env, start_response) elif status == SUBMITTED_REQUESTS: self.logger.debug('SUBMITTED_REQUESTS') return Response(status=HTTP_OK, body='Accepted %s request.\n' % hlm_req, content_type="text/plain")(env, start_response) else: # invalid case self.logger.debug('INVALID_CASE') return Response(status=HTTP_INTERNAL_SERVER_ERROR, body=out, content_type="text/plain")(env, start_response) # Process GET object status request elif req.method == "GET" and obj: if 'STATUS' in query: # Get status of each replica rc, out, replicas_status = self.get_object_replicas_status( req, account, container, obj) if rc == REMOTE_STATUS: # send the replica status to requester node return Response(status=HTTP_OK, body=out, content_type="text/plain")(env, start_response) # Prepare/format object status info to report # (json is default format) out = self.format_object_status_info_for_reporting( req, replicas_status) + '\n' # Report object status return Response(status=HTTP_OK, body=out, content_type="text/plain")(env, start_response) # Process container request if (container and not obj and ((method == 'POST' and ('MIGRATE' in query or 'RECALL' in query)) or method == 'GET' and 'STATUS' in query)): self.logger.debug('Process container request') # Get list of objects list_url = 'http://%(ip)s:8080%(url)s' list_req = list_url % {'ip': self.ip, 'url': req.path} self.logger.debug('list_req: %s', list_req) self.logger.debug('req.headers: %s', str(req.headers)) token = req.headers['X-Storage-Token'] self.logger.debug('token: %s', token) headers = {'X-Storage-Token': token} response = requests.get(list_req, headers=headers) self.logger.debug('response.headers: %s', str(response.headers)) self.logger.debug('list: %s', str(response.content)) objects = response.content.strip().split('\n') # Submit migration or recall if method == 'POST': if 'MIGRATE' in query: hlm_req = 'MIGRATE' hlm_backend = self.migrate_backend elif 'RECALL' in query: hlm_req = 'RECALL' hlm_backend = self.recall_backend # submit hlm requests success = 0 failure = 0 for obj in objects: self.logger.debug('obj: %s', obj) status, out = self.submit_object_replicas_migration_recall( req, account, container, obj, hlm_req, hlm_backend) self.logger.debug('submit_object_replicas_migr.._recall()') if status == SUBMITTED_FORWARDED_REQUEST: self.logger.debug('SUBMITTED_FORWARDED_REQUEST') return Response(status=HTTP_OK, body='Accepted remote replica' 'HLM request', content_type="text/plain")( env, start_response) elif status == FAILED_SUBMITTING_REQUEST: self.logger.debug('FAILED_SUBMITTING_REQUEST') failure += 1 elif status == SUBMITTED_REQUESTS: self.logger.debug('SUBMITTED_REQUESTS') success += 1 if failure == 0: return Response(status=HTTP_OK, body='Submitted %s requests.\n' % hlm_req, content_type="text/plain")(env, start_response) elif success == 0: return Response(status=HTTP_INTERNAL_SERVER_ERROR, body='Failed to submit %s requests.\n' % hlm_req, content_type="text/plain")(env, start_response) else: return Response(status=HTTP_OK, body="Submitting %s requests" " is only partially" " successful.\n" % hlm_req, content_type="text/plain")(env, start_response) elif method == 'GET': # submit hlm requests accumulated_out = "[" for obj in objects: self.logger.debug('obj: %s', obj) # Get status of each replica # rewrite req.path to point to object instead of container req_orig_path = req.environ['PATH_INFO'] req.environ['PATH_INFO'] += "/" + obj rc, out, replicas_status = self.get_object_replicas_status( req, account, container, obj) # Prepare/format object status info to report # (json is default format) accumulated_out += self.format_object_status_info_for_reporting( req, replicas_status) + ',' req.environ['PATH_INFO'] = req_orig_path # Report accumulated object status accumulated_out = accumulated_out[:-1] + ']' return Response(status=HTTP_OK, body=accumulated_out, content_type="text/plain")(env, start_response) return self.app(env, start_response)
class AutosyncMiddleware(object): def __init__(self, app, conf): self.app = app self.conf = conf self.req_timeout = 2 self.conn_timeout = 10 self.keychars = string.ascii_letters + string.digits self.logger = get_logger(self.conf, log_route='nemo') self.override_auth = \ config_true_value(conf.get('override_auth', False)) self.default_my_cluster = conf.get('autosync_my_cluster', None) self.default_placement = conf.get('autosync_placement', [self.default_my_cluster]) if self.default_placement: self.default_placement = self.default_placement.split(',') #def redirect(self): # peer = choice(self.placement) # resp = HTTPMovedPermanently(location=(peer + self.env['PATH_INFO'])) # return resp(self.env, self.start_response) def send_to_peer(self, peer, sync_to_peer, key): peer = peer.lower() ssl = False if peer.startswith('https://'): ssl = True peer = peer[8:] if peer.startswith('http://'): peer = peer[7:] try: with Timeout(self.conn_timeout): if ssl: #print 'HTTPS %s ' % peer conn = HTTPSConnection(peer) else: #print 'HTTP %s ' % peer conn = HTTPConnection(peer) conn.putrequest(self.req.method, self.req.path_qs) conn.putheader('X-Orig-Cluster', self.my_cluster) conn.putheader('X-Account-Meta-Orig-Cluster', self.my_cluster) conn.putheader('X-Container-Meta-Orig-Cluster', self.my_cluster) if key: sync_to = sync_to_peer + self.env['PATH_INFO'] conn.putheader('X-Container-Sync-To', sync_to) for header, value in self.req.headers.iteritems(): if header != 'X-Container-Sync-To': conn.putheader(header, value) conn.endheaders(message_body=None) with Timeout(self.req_timeout): resp = conn.getresponse() status = resp.status return (status, resp.getheaders(), resp.read()) except (Exception, Timeout) as e: # Print error log print >> sys.stderr, peer + ': Exception, Timeout error: %s' % e print '<<<<<<<< HTTP_SERVICE_UNAVAILABLE' #return HTTP_SERVICE_UNAVAILABLE, None, None def send_to_peers(self, peers, key): pile = GreenPile(len(peers)) # Have the first peer to sync to the local cluster sync_to_peer = self.my_cluster for peer in peers: # create thread per peer and send a request pile.spawn(self.send_to_peer, peer, sync_to_peer, key) # Have the next peer to sync to the present peer sync_to_peer = peer # collect the results, if anyone failed.... response = [resp for resp in pile if resp] while len(response) < len(peers): response.append((HTTP_SERVICE_UNAVAILABLE, None, None)) return response def highest_response(self, resps, swap={}): highest_resp = None highest_status = -1 for resp in resps: status = resp[0] if status in swap: status = swap[status] status = int(status) if status > highest_status: highest_status = status highest_resp = resp if highest_resp: return Response(body=highest_resp[2], status=highest_resp[0], headers=highest_resp[1]) return HTTPServiceUnavailable(request=self.req) def all_success(self, resps): for resp in resps: if not is_success(resp[0]): return False return True def __call__(self, env, start_response): def my_start_response(status, headers, exc_info=None): self.status = status self.headers = list(headers) self.exc_info = exc_info self.env = env self.start_response = start_response # If request was already processed by autosync # (here or at the original cluster where it first hit) if 'HTTP_X_ORIG_CLUSTER' in env: print >> sys.stderr, 'HTTP_X_ORIG_CLUSTER found!' if self.override_auth: env['swift_owner'] = True return self.app(env, start_response) # If it is a local call or a tempurl object call if 'swift.authorize_override' in env: return self.app(env, start_response) # Get Placement parameters if 'swift.my_cluster' in env: self.my_cluster = env['swift.my_cluster'] else: self.my_cluster = self.default_my_cluster if 'swift.placement' in env: placement = env['swift.placement'] else: placement = self.default_placement or self.my_cluster if not self.my_cluster or not placement: return self.app(env, start_response) self.req = Request(env) # For now we support only placement here and in one other place if self.my_cluster not in placement: return HTTPInternalServerError(request=self.req) # return self.redirect() peers = [p for p in placement if p != self.my_cluster] if len(peers) != 1: return HTTPInternalServerError(request=self.req) # This request needs to be handled localy try: (version, account, container, obj) = \ self.req.split_path(2, 4, True) except ValueError: return self.app(env, start_response) if obj or self.req.method in ('OPTIONS', 'GET', 'HEAD'): # business as usual - I will serve the request locally and be done # TBD, in case of 404 returned from GET object, try a remote copy? return self.app(env, start_response) # Lets see, its either PUT, POST or DELETE account/container # Otherwise said - 'we need to change the account/container' # both here and with peers... # As part of any container creation/modification (PUT/POST): # Create a new key to protect the container communication from now # and until the next time the container is updated. # Note that race may occur with container-sync daemons resulting in # container-sync failing due to misaligned keys. # Changing the keys per update help support changes in the placement # and can serve as a simple mechanism for replacing conatienr sync keys # If this turns out to be an issue, we may extract and reuse the same # key for the duration of the container existance. if container and self.req.method in ['POST', 'PUT']: key = ''.join(choice(self.keychars) for x in range(64)) # Add the key to the env when calling the local cluster env['HTTP_X_CONTAINER_SYNC_KEY'] = key # Set the container replica of the local cluster to sync to the # last cluster in the list of peers sync_to_peer = peers[-1] # Sync to the prev peer sync_to = sync_to_peer + self.env['PATH_INFO'] env['HTTP_X_CONTAINER_SYNC_TO'] = sync_to else: key = None # Signals that there are no Container-Sync headers # Try localy, if we fail and not DELETE respond with a faliure. resp_data = self.app(self.env, my_start_response) data = ''.join(iter(resp_data)) if hasattr(resp_data, 'close'): resp_data.close() resp_status_int = int(self.status[:3]) # Faliure at local cluster during anything but DELETE... abandon ship if not is_success(resp_status_int) and self.req.method != 'DELETE': # Dont even try the peers start_response(self.status, self.headers, self.exc_info) return data # Call peers and accomulate responses try: # Note that key is None if not during container PUT/POST resps = self.send_to_peers(peers, key) # Append the local cluster response resps.append((resp_status_int, self.headers, data)) except: return HTTPServiceUnavailable(request=self.req) resp = None if self.req.method == 'DELETE': # Special treatment to DELETE - respond with the best we have resp = self.highest_response(resps, swap={'404': '1'}) else: # PUT/POST - respond only if all success if self.all_success(resps): resp = self.highest_response(resps) else: # PUT/POST with local success and remote faliure resp = HTTPServiceUnavailable(request=self.req) return resp(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path( 2, 4, rest_with_last=True) is_swifty_request = valid_api_version(version) except ValueError: is_swifty_request = False if not is_swifty_request: return self.app(env, start_response) if not obj: typ = 'Container' if container else 'Account' client_header = 'X-%s-Rfc-Compliant-Etags' % typ sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ if client_header in req.headers: if req.headers[client_header]: req.headers[sysmeta_header] = config_true_value( req.headers[client_header]) else: req.headers[sysmeta_header] = '' if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)): req.headers[sysmeta_header] = '' def translating_start_response(status, headers, exc_info=None): return start_response( status, [(client_header if h.title() == sysmeta_header else h, v) for h, v in headers], exc_info) return self.app(env, translating_start_response) container_info = get_container_info(env, self.app, 'EQ') if not container_info or not is_success(container_info['status']): return self.app(env, start_response) flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags') if flag is None: account_info = get_account_info(env, self.app, 'EQ') if not account_info or not is_success(account_info['status']): return self.app(env, start_response) flag = account_info.get('sysmeta', {}).get('rfc-compliant-etags') if flag is None: flag = self.conf.get('enable_by_default', 'false') if not config_true_value(flag): return self.app(env, start_response) status, headers, resp_iter = req.call_application(self.app) for i, (header, value) in enumerate(headers): if header.lower() == 'etag': if not value.startswith(('"', 'W/"')) or \ not value.endswith('"'): headers[i] = (header, '"%s"' % value) start_response(status, headers) return resp_iter
def __call__(self, env, start_response): req = Request(env) try: (version, account, container, obj) = req.split_path(4, 4, True) except ValueError: # If obj component is not present in req, do not proceed further. return self.app(env, start_response) self.account_name = account self.container_name = container self.object_name = obj # Check if fastcopy is possible # (only plain object at this time) if req.headers.get('X-Copy-From') and not ( req.headers.get('Range') or req.headers.get('X-Amz-Copy-Source-Range')): # TODO: it could be also a copy from MPU to full object # but I pretty sure that this case is explained on AWS doc req.headers['Oio-Copy-From'] = req.headers.get('X-Copy-From') del req.headers['X-Copy-From'] return self.app(env, start_response) """ TODO (how access proxy/obj.py instance to checking SLO ?) # compare range and segments ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) == 1: ranges = ranges[0] # check that ORG object is SLO container, obj = req.headers['X-Copy-From'].split('/', 1) storage = self.app.storage props = storage.object_get_properties( self.account_name, container, obj) if props['properties'].get(SLO, None): manifest = json.loads("".join(data)) offset = 0 # identify segment to copy for entry in manifest: if ranges[0] == offset and \ ranges[1] + 1 == offset + entry['bytes']: _, container, obj = entry['name'].split('/', 2) checksum = entry['hash'] self.app.logger.info( "LINK SLO (%s,%s,%s) TO (%s,%s,%s)", self.account_name, self.container_name, self.object_name, self.account_name, container, obj) return self.app(env, start_response) break offset += entry['bytes'] else: print("NO SEGMENT FOUND, FALLACK") """ try: # In some cases, save off original request method since it gets # mutated into PUT during handling. This way logging can display # the method the client actually sent. if req.method == 'PUT' and req.headers.get('X-Copy-From'): return self.handle_PUT(req, start_response) elif req.method == 'COPY': req.environ['swift.orig_req_method'] = req.method return self.handle_COPY(req, start_response) elif req.method == 'POST' and self.object_post_as_copy: req.environ['swift.orig_req_method'] = req.method return self.handle_object_post_as_copy(req, start_response) elif req.method == 'OPTIONS': # Does not interfere with OPTIONS response from # (account,container) servers and /info response. return self.handle_OPTIONS(req, start_response) except HTTPException as e: return e(req.environ, start_response) return self.app(env, start_response)
def handle_request(self, env, start_response): request = Request(env) method = request.method if method not in ('POST', 'PUT', 'COPY', 'DELETE'): return self.app(env, start_response) # Get the response from the rest of the pipeline before we # start doing anything; this means that whatever is being created # or deleted will have been done before we start constructing # the notification payload response = self._app_call(env) status_code = self._get_status_int() try: ver, account, container, obj = request.split_path( 2, 4, rest_with_last=True) except ValueError: start_response(self._response_status, self._response_headers, self._response_exc_info) return response event_methods = { 'DELETE': 'ObjectRemoved:Delete', 'COPY': 'ObjectCreated:Copy', 'PUT': 'ObjectCreated:Put', 'POST': 'ObjectModified' } event_object = ('object' if obj else 'container' if container else 'account') event_type = '%s' % (event_methods[method]) if status_code in (200, 201, 202, 204): request_headers = request.headers payload = self._get_request_auth_info(request_headers) payload['object_type'] = event_object payload['account'] = account if container: payload['container'] = container if obj: payload['object'] = obj if method != 'DELETE': head_headers = self._make_head_request(env).headers copy_from = request_headers.get('X-Copy-From') if method == 'PUT' and copy_from: # Copies are turned into PUTs with an X-Copy-From # in the object middleware # though we don't need to handle them differently event_type = event_methods['COPY'] if copy_from[0] == '/': copy_from = copy_from[1:] copy_from_container, copy_from_object = copy_from.split( '/', 1) payload['copy-from-container'] = copy_from_container payload['copy-from-object'] = copy_from_object if request_headers.get('X-Fresh-Metadata', None): payload['copy-fresh-metadata'] = bool( request_headers.get('X-Fresh-Metadata')) payload.update( self._get_account_metadata(request_headers, self._response_headers)) if container: payload.update( self._get_container_metadata(request_headers, self._response_headers)) if obj: payload.update( self._get_object_metadata(request_headers, self._response_headers)) modified_timestamp = head_headers.get('X-Timestamp') if modified_timestamp: modified_datetime = datetime.fromtimestamp( float(modified_timestamp)) payload['updated_at'] = modified_datetime.strftime( '%Y-%m-%dT%H:%M:%S.%f') payload['x-timestamp'] = modified_timestamp def set_field_if_exists(source, dest): value = head_headers.get(source) if value: payload[dest] = value set_field_if_exists('Last-Modified', 'last-modified') if obj: for field in (('Content-Length', 'content-length'), ('Content-Type', 'content-type')): set_field_if_exists(*field) self._notifier.notify({}, event_type, payload) # We don't want to tamper with the response start_response(self._response_status, self._response_headers, self._response_exc_info) return response
def __call__(self, env, start_response): req = Request(env) try: # account and container only version, acct, cont = req.split_path(2, 3) except ValueError: is_account_or_container_req = False else: is_account_or_container_req = True if not is_account_or_container_req: return self.app(env, start_response) if not valid_api_version(version) or req.method not in ('GET', 'HEAD'): return self.app(env, start_response) # OK, definitely have an account/container request. # Get the desired content-type, then force it to a JSON request. try: out_content_type = get_listing_content_type(req) except HTTPException as err: return err(env, start_response) params = req.params can_vary = 'format' not in params params['format'] = 'json' req.params = params # Give other middlewares a chance to be in charge env.setdefault('swift.format_listing', True) status, headers, resp_iter = req.call_application(self.app) if not env.get('swift.format_listing'): start_response(status, headers) return resp_iter header_to_index = {} resp_content_type = resp_length = None for i, (header, value) in enumerate(headers): header = header.lower() if header == 'content-type': header_to_index[header] = i resp_content_type = value.partition(';')[0] elif header == 'content-length': header_to_index[header] = i resp_length = int(value) elif header == 'vary': header_to_index[header] = i if not status.startswith(('200 ', '204 ')): start_response(status, headers) return resp_iter if can_vary: if 'vary' in header_to_index: value = headers[header_to_index['vary']][1] if 'accept' not in list_from_csv(value.lower()): headers[header_to_index['vary']] = ('Vary', value + ', Accept') else: headers.append(('Vary', 'Accept')) if resp_content_type != 'application/json': start_response(status, headers) return resp_iter if resp_length is None or \ resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: start_response(status, headers) return resp_iter def set_header(header, value): if value is None: del headers[header_to_index[header]] else: headers[header_to_index[header]] = ( headers[header_to_index[header]][0], str(value)) if req.method == 'HEAD': set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', None) # don't know, can't determine start_response(status, headers) return resp_iter body = b''.join(resp_iter) try: listing = json.loads(body) # Do a couple sanity checks if not isinstance(listing, list): raise ValueError if not all(isinstance(item, dict) for item in listing): raise ValueError except ValueError: # Static web listing that's returning invalid JSON? # Just pass it straight through; that's about all we *can* do. start_response(status, headers) return [body] if not req.allow_reserved_names: listing = self.filter_reserved(listing, acct, cont) try: if out_content_type.endswith('/xml'): if cont: body = container_to_xml( listing, wsgi_to_bytes(cont).decode('utf-8')) else: body = account_to_xml(listing, wsgi_to_bytes(acct).decode('utf-8')) elif out_content_type == 'text/plain': body = listing_to_text(listing) else: body = json.dumps(listing).encode('ascii') except KeyError: # listing was in a bad format -- funky static web listing?? start_response(status, headers) return [body] if not body: status = '%s %s' % (HTTP_NO_CONTENT, RESPONSE_REASONS[HTTP_NO_CONTENT][0]) set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', len(body)) start_response(status, headers) return [body]
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if account is None: return self.app(env, start_response) if env.get('swift.authorize_override', False): return self.app(env, start_response) # First, restrict modification of auth meta-data to only users with # the admin role (or roles that have been specially enabled in # the swift config). if req.method == "POST": # following code to get roles is borrowed from keystoneauth roles = set() if (env.get('HTTP_X_IDENTITY_STATUS') == 'Confirmed' or env.get('HTTP_X_SERVICE_IDENTITY_STATUS') in (None, 'Confirmed')): roles = set(list_from_csv(env.get('HTTP_X_ROLES', ''))) if not roles.intersection(self.allowed_meta_write_roles): for k, v in req.headers.iteritems(): if k.startswith('X-Container-Meta-Allowed-Iprange') \ or k.startswith('X-Account-Meta-Allowed-Iprange'): return Response(status=403, body=deny_meta_change, request=req)(env, start_response) # Grab the metadata for the account and container if container is not None and container != "": try: container_info = \ get_container_info(req.environ, self.app, swift_source='IPRangeACLMiddleware') except ValueError: # if we can't get container info, then we deny the request return Response(status=403, body="Invalid container (%s)" % container, request=req)(env, start_response) else: container_info = None try: acc_info = get_account_info(req.environ, self.app, swift_source='IPRangeACLMiddleware') except ValueError: # if we can't get account info, then we deny the request return Response(status=403, body="Invalid account (%s)" % account, request=req)(env, start_response) remote_ip = get_remote_client(req) allowed = set() default = "denied" # Read any account-level ACLs meta = acc_info['meta'] for k, v in meta.iteritems(): if k.startswith("allowed-iprange") and len(v) > 0: allowed.add(v) # This key is used to set the default access policy in # cases where no ACLs are present in the meta-data. if k == "ipacl-default": default = v # If the request is for a container or object, check for any # container-level ACLs if container_info is not None: meta = container_info['meta'] for k, v in meta.iteritems(): # Each allowed range must have a unique meta-data key, but # the key must begin with 'allowed-iprange-' if k.startswith('allowed-iprange-') and len(v) > 0: allowed.add(v) # This key is used to set the default access policy in # cases where no ACLs are present in the meta-data. # NOTE: Container-level default behaviour will override # account-level defaults. if k == "ipacl-default": default = v # XXX Could probably condense this into one tree, but not sure # whether Pytricia is OK with mixing IPv4 and IPv6 prefixes. self.pyt = pytricia.PyTricia(32) self.pyt6 = pytricia.PyTricia(128) # If there are no IP range ACLs in the meta-data and the # default policy is "allowed", then we can grant access. if len(allowed) == 0 and default == "allowed": return self.app(env, start_response) else: # Build the patricia tree of allowed IP prefixes for pref in allowed: if ':' in pref: try: addrcheck = ipaddress.IPv6Network(unicode(pref), False) except ipaddress.AddressValueError: self.logger.debug( "iprange_acl -- skipping invalid IP prefix: %(pref)s", {'pref': pref}) continue self.pyt6[pref] = "allowed" else: try: addrcheck = ipaddress.IPv4Network(unicode(pref), False) except ipaddress.AddressValueError: self.logger.debug( "iprange_acl -- skipping invalid IP prefix: %(pref)s", {'pref': pref}) continue self.pyt[pref] = "allowed" # Always allow our own IP, otherwise we could lock ourselves out from # the container! if ':' in self.local_ip: self.pyt6[self.local_ip] = "allowed" else: self.pyt[self.local_ip] = "allowed" # Add our default allowed IP ranges to the patricia tree for default_range in self.default_ranges: if ':' in default_range: try: addrcheck = ipaddress.IPv6Network(unicode(default_range), \ False) except ipaddress.AddressValueError: self.logger.debug("Invalid always_allow prefix for IPv6: %s" \ % (default_range)) else: self.pyt6[default_range] = "allowed" else: try: addrcheck = ipaddress.IPv4Network(unicode(default_range), \ False) except ipaddress.AddressValueError: self.logger.debug("Invalid always_allow prefix for IPv4: %s" \ % (default_range)) else: self.pyt[default_range] = "allowed" # Look up the address of the client in the patricia tree if ':' in remote_ip: status = self.pyt6.get(remote_ip) else: status = self.pyt.get(remote_ip) if status == "allowed": return self.app(env, start_response) return Response(status=403, body=self.deny_message, request=req)(env, start_response)
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) container_info = get_container_info( req.environ, self.app, swift_source='ImageScalerMiddleware') # parse query string if req.query_string: qs = parse_qs(req.query_string) if 'size' in qs: req_size = qs['size'] else: self.logger.debug("image-scaler: No image scaling requested.") return self.app(env, start_response) else: # nothing for us to do, no scaling requested self.logger.debug("image-scaler: No image scaling requested.") return self.app(env, start_response) # check container whether scaling is allowed meta = container_info['meta'] if not meta.has_key('image-scaling') or \ meta.has_key('image-scaling') and \ not meta['image-scaling'].lower() in ['true', '1']: # nothing for us to do self.logger.debug("image-scaler: Image scaling not " "allowed. Nothing for us to do.") return self.app(env, start_response) # default allowed extensions allowed_exts = self.conf.get('formats', 'jpg;png;gif') allowed_exts = allowed_exts.lower() allowed_exts = allowed_exts.split(';') # check whether file has the allowed ending if meta.has_key('image-scaling-extensions'): allowed_exts = meta['image-scaling-extensions'].split(',') requested_ext = req.path.rsplit('.', 1)[-1] if not requested_ext.lower() in map(lambda x: x.lower(), allowed_exts): self.logger.info("image-scaler: extension %s not allowed" " for image scaling" % requested_ext) return self.app(env, start_response) # get maxsize from config, otherwise 20 MB max_size = self.conf.get('maxsize', '20971520') try: max_size = int(max_size) except ValueError: max_size = 20971520 self.logger.error( "wrong format for max_size from configuration file, using 20 MB" ) if meta.has_key('image-scaling-max-size'): max_size = int(meta['image-scaling-max-size']) obj_info = get_object_info(req.environ, self.app, swift_source="ImageScalerMiddleware") if int(obj_info['length']) > max_size: self.logger.info("image-scaler: object too large") return self.app(env, start_response) response = ImageScalerResponse(start_response, req_size, self.logger) app_iter = self.app(env, response.scaler_start_response) if app_iter is not None: response.finish_response(app_iter) return response.write()
def __call__(self, env, start_response): req = Request(env) try: # account and container only version, acct, cont = req.split_path(2, 3) except ValueError: return self.app(env, start_response) if not valid_api_version(version) or req.method not in ('GET', 'HEAD'): return self.app(env, start_response) # OK, definitely have an account/container request. # Get the desired content-type, then force it to a JSON request. try: out_content_type = get_listing_content_type(req) except HTTPException as err: return err(env, start_response) params = req.params params['format'] = 'json' req.params = params status, headers, resp_iter = req.call_application(self.app) header_to_index = {} resp_content_type = resp_length = None for i, (header, value) in enumerate(headers): header = header.lower() if header == 'content-type': header_to_index[header] = i resp_content_type = value.partition(';')[0] elif header == 'content-length': header_to_index[header] = i resp_length = int(value) if not status.startswith('200 '): start_response(status, headers) return resp_iter if resp_content_type != 'application/json': start_response(status, headers) return resp_iter if resp_length is None or \ resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: start_response(status, headers) return resp_iter def set_header(header, value): if value is None: del headers[header_to_index[header]] else: headers[header_to_index[header]] = ( headers[header_to_index[header]][0], str(value)) if req.method == 'HEAD': set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', None) # don't know, can't determine start_response(status, headers) return resp_iter body = b''.join(resp_iter) try: listing = json.loads(body) # Do a couple sanity checks if not isinstance(listing, list): raise ValueError if not all(isinstance(item, dict) for item in listing): raise ValueError except ValueError: # Static web listing that's returning invalid JSON? # Just pass it straight through; that's about all we *can* do. start_response(status, headers) return [body] try: if out_content_type.endswith('/xml'): if cont: body = container_to_xml(listing, cont) else: body = account_to_xml(listing, acct) elif out_content_type == 'text/plain': body = listing_to_text(listing) # else, json -- we continue down here to be sure we set charset except KeyError: # listing was in a bad format -- funky static web listing?? start_response(status, headers) return [body] if not body: status = '%s %s' % (HTTP_NO_CONTENT, RESPONSE_REASONS[HTTP_NO_CONTENT][0]) set_header('content-type', out_content_type + '; charset=utf-8') set_header('content-length', len(body)) start_response(status, headers) return [body]