Ejemplo n.º 1
0
    def __call__(self, env, start_response):
        # If override is set in env, then just pass along
        if config_true_value(env.get('swift.crypto.override')):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ('PUT', 'POST'):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('GET', 'HEAD'):
            handler = EncrypterObjContext(self, self.logger).handle_get_or_head
        elif req.method == 'PUT':
            handler = EncrypterObjContext(self, self.logger).handle_put
        elif req.method == 'POST':
            handler = EncrypterObjContext(self, self.logger).handle_post
        else:
            # anything else
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Ejemplo n.º 2
0
    def __call__(self, env, start_response):
        if config_true_value(env.get('swift.crypto.override')):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ('PUT', 'POST'):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        fetch_crypto_keys = env.get(CRYPTO_KEY_CALLBACK)
        if fetch_crypto_keys is not None:
            try:
                fetch_crypto_keys()
            except HTTPException as exc:
                if MISSING_KEY_MSG in exc.body:
                    if req.method in ('PUT', 'POST'):
                        # No key, just upload without encryption
                        env['swift.crypto.override'] = True
                    # else:
                    #   let the thing fail later,
                    #   if a key is required for decoding
                else:
                    raise
            except Exception:
                # Let the parent class handle other exceptions
                pass
        res = super(Encrypter, self).__call__(env, start_response)
        return res
Ejemplo n.º 3
0
    def __call__(self, env, start_response):
        self.logger.debug('Initialising gate middleware')

        req = Request(env)
        try:
            version, account = req.split_path(1, 3, True)
        except ValueError:
            return HttpNotFound(request=req)
        
        if account is 'gate':
            # Handles direct calls to gate
            return HttpOk

        if 'X-Gate-Verify' in env:
            verify = env['X-Gate-Verify']
            self.logger.debug('Verification request: %s algorithms: %s' % (req.path, verify))

            try:
                version, account, container, obj = req.split_path(4, 4, True)
            except ValueError:
                return HTTPBadRequest(request=req)

            algorithms = verify.split(',')
            for algo in algorithms:
                metakey = 'X-Object-Meta-Gate-%s' % algo.upper()
                if metakey not in env:
                    self.logger.debug('Invalid verification request, object missing: %s' % (metakey))
                    return HTTPBadRequest(request=req)

            if publish_verify(req.path, algorithms):
                for algo in algorithms:
                    statuskey = 'X-Object-Meta-Gate-Verify-%s-Status' % algo.upper()
                    env[statuskey] = 'Queued'
                    env['X-Object-Meta-Gate-Verify'] = verify

        if 'X-Gate-Process' in env:
            module = env['X-Gate-Process']
            self.logger.debug('Process request: %s module: %s' % (req.path, module))

            try:
                version, case, container, obj = req.split_path(4, 4, True)
            except ValueError:
                return HTTPBadRequest(request=req)

            if publish_process(req.path, algorithms):
                for algo in algorithms:
                    env['X-Object-Meta-Gate-Process'] = module
                    env['X-Object-Meta-Gate-Process-Status'] = 'Queued'

        # TODO: Get reponse to see if a fake object
        reponse = self.app(env, start_response)
        return reponse
Ejemplo n.º 4
0
Archivo: slo.py Proyecto: pchng/swift
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env["swift.copy_hook"] = self.copy_hook(
            env.get("swift.copy_hook", lambda src_req, src_resp, sink_req: src_resp)
        )

        try:
            if req.method == "PUT" and req.params.get("multipart-manifest") == "put":
                return self.handle_multipart_put(req, start_response)
            if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete":
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == "GET" or req.method == "HEAD":
                return self.handle_multipart_get_or_head(req, start_response)
            if "X-Static-Large-Object" in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body="X-Static-Large-Object is a reserved header. "
                    "To create a static large object add query param "
                    "multipart-manifest=put.",
                )
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 5
0
    def __call__(self, env, start_response):
        # making a duplicate, because if this is a COPY request, we will
        # modify the PATH_INFO to find out if the 'Destination' is in a
        # versioned container
        req = Request(env.copy())
        try:
            (version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get('allow_versioned_writes')
        if allow_versioned_writes and container and not obj:
            return self.container_request(req, start_response,
                                          allow_versioned_writes)
        elif obj and req.method in ('PUT', 'COPY', 'DELETE'):
            return self.object_request(
                req, version, account, container, obj,
                allow_versioned_writes)(env, start_response)
        else:
            return self.app(env, start_response)
Ejemplo n.º 6
0
    def get_creds(self, environ):
        req = Request(environ)
        try:
            parts = req.split_path(1, 4, True)
            _, account, _, _ = parts
        except ValueError:
            account = None

        env_identity = self._integral_keystone_identity(environ)
        if not env_identity:
            # user identity is not confirmed. (anonymous?)
            creds = {
                'identity': None,
                'is_authoritative': (account and
                                     account.startswith(self.reseller_prefix))
            }
            return creds

        tenant_id, tenant_name = env_identity['tenant']
        user_id, user_name = env_identity['user']
        roles = [r.strip() for r in env_identity.get('roles', [])]
        account = self._get_account_for_tenant(tenant_id)
        is_admin = (tenant_name == user_name)

        creds = {
            "identity": env_identity,
            "roles": roles,
            "account": account,
            "tenant_id": tenant_id,
            "tenant_name": tenant_name,
            "user_id": user_id,
            "user_name": user_name,
            "is_admin": is_admin
        }
        return creds
Ejemplo n.º 7
0
    def __call__(self, env, start_response):
        req = Request(env)
        # We want to check POST here
        # it can possibly have content_length > 0
        if not self.enforce_quota or req.method not in ("POST", "PUT", "COPY"):
            return self.app(env, start_response)
        account_info = get_account_info(req.environ, self.app,
                                        swift_source='litequota')
        if not account_info:
            return self.app(env, start_response)
        service_plan = assemble_from_partial(self.metadata_key,
                                             account_info['meta'])
        try:
            ver, account, container, obj = \
                req.split_path(2, 4, rest_with_last=True)
        except ValueError:
            return self.app(env, start_response)
        if not service_plan and req.method == 'PUT' and not obj:
            service_plan = self.set_serviceplan(req, account)
        if not service_plan:
            return self.app(env, start_response)
        try:
            service_plan = json.loads(service_plan)
        except ValueError:
            return self.app(env, start_response)

        if service_plan.get('storage', None):
            resp = self.apply_storage_quota(req, service_plan['storage'],
                                            account_info,
                                            ver, account, container, obj)
            if resp:
                return resp(env, start_response)
        return self.app(env, start_response)
Ejemplo n.º 8
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        if ((req.method == 'GET' or req.method == 'HEAD') and
                req.params.get('multipart-manifest') != 'get'):
            return GetContext(self, self.logger).\
                handle_request(req, start_response)
        elif req.method == 'PUT':
            error_response = self.validate_x_object_manifest_header(
                req, start_response)
            if error_response:
                return error_response(env, start_response)
        return self.app(env, start_response)
Ejemplo n.º 9
0
    def __call__(self, env, start_response):
        """
        WSGI entry point.
        Wraps env in swob.Request object and passes it down.

        :param env: WSGI environment dictionary
        :param start_response: WSGI callable
        """
        req = Request(env)
        if self.memcache_client is None:
            # 在pipline中memcache middleware应该在前面
            # memcache为每一个请求建立一个memcache client,并把这个client放入env中
            self.memcache_client = cache_from_env(env)
        if not self.memcache_client:
            self.logger.warning(
                _('Warning: Cannot ratelimit without a memcached client'))
            return self.app(env, start_response)
        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
        # ratelimit_resp 为 None 或者 Response 对象,Response 对象是可调用的。
        # ratelimit 操作正常执行,或者不需要执行,返回 None,接着向下调用。
        if ratelimit_resp is None:
            return self.app(env, start_response)
        else:
            # ratelimit 执行失败,或者需要返回错误信息,返回 Response 对象。
            # 不用向下调用,直接返回。
            return ratelimit_resp(env, start_response)
Ejemplo n.º 10
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
            if DEBUG:
                print('obj:%s' % obj)
        except ValueError:
            return self.app(env, start_response)
        try:
            if env['REQUEST_METHOD'] == "PUT":
                # Read the object content and enc_key in memory 
                # encrypt the object with the enc_key
                content = env['wsgi.input'].read()
                if content:
                    return Response(status=403,
                                body="content %s detected" % content,
                                content_type="text/plain")(env, start_response)
                else:
                    return Response(status=403,
                                body="content not detected",
                                content_type="text/plain")(env, start_response)
                if DEBUG:
                    print(content)
                #encryptor = Encryptor()
                content = content + "yyyyyyyyyyyyyyyyy"
            # Maybe add the decrytion process later 
            #elif env['REQUEST_METHOD'] == "GET":
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 11
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        try:
            if req.method == 'PUT' and \
                    req.params.get('multipart-manifest') == 'put':
                return self.handle_multipart_put(req, start_response)
            if req.method == 'DELETE' and \
                    req.params.get('multipart-manifest') == 'delete':
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == 'GET' or req.method == 'HEAD':
                return self.handle_multipart_get_or_head(req, start_response)
            if 'X-Static-Large-Object' in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body='X-Static-Large-Object is a reserved header. '
                    'To create a static large object add query param '
                    'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 12
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.account_name = account
        self.container_name = container
        self.object_name = obj

        # Save off original request method (COPY/POST) in case it gets mutated
        # into PUT during handling. This way logging can display the method
        # the client actually sent.
        req.environ['swift.orig_req_method'] = req.method

        if req.method == 'PUT' and req.headers.get('X-Copy-From'):
            return self.handle_PUT(req, start_response)
        elif req.method == 'COPY':
            return self.handle_COPY(req, start_response)
        elif req.method == 'POST' and self.object_post_as_copy:
            return self.handle_object_post_as_copy(req, start_response)
        elif req.method == 'OPTIONS':
            # Does not interfere with OPTIONS response from (account,container)
            # servers and /info response.
            return self.handle_OPTIONS(req, start_response)

        return self.app(env, start_response)
Ejemplo n.º 13
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            parts = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('PUT', 'POST', 'GET', 'HEAD'):
            # handle only those request methods that may require keys
            km_context = KeyMasterContext(self, req, *parts[1:])
            try:
                return km_context.handle_request(req, start_response)
            except HTTPException as err_resp:
                return err_resp(env, start_response)
            except KeyError as err:
                if 'object' in err.args:
                    self.app.logger.debug(
                        'Missing encryption key, cannot handle request')
                    raise HTTPBadRequest(MISSING_KEY_MSG)
                else:
                    raise

        # anything else
        return self.app(env, start_response)
Ejemplo n.º 14
0
    def __call__(self, env, start_response):
        """
        WSGI entry point.
        Wraps env in swob.Request object and passes it down.

        :param env: WSGI environment dictionary
        :param start_response: WSGI callable
        """
        req = Request(env)
        if self.memcache_client is None:
            self.memcache_client = cache_from_env(env)
        if not self.memcache_client:
            self.logger.warning(
                _('Warning: Cannot ratelimit without a memcached client'))
            return self.app(env, start_response)
        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        if not valid_api_version(version):
            return self.app(env, start_response)
        ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
        if ratelimit_resp is None:
            return self.app(env, start_response)
        else:
            return ratelimit_resp(env, start_response)
Ejemplo n.º 15
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        try:
            if obj:
                if req.method == 'PUT' and \
                        req.params.get('multipart-manifest') == 'put':
                    return self.handle_multipart_put(req, start_response)
                if req.method == 'DELETE' and \
                        req.params.get('multipart-manifest') == 'delete':
                    return self.handle_multipart_delete(req)(env,
                                                             start_response)
                if 'X-Static-Large-Object' in req.headers:
                    raise HTTPBadRequest(
                        request=req,
                        body='X-Static-Large-Object is a reserved header. '
                        'To create a static large object add query param '
                        'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 16
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        try:
            # In some cases, save off original request method since it gets
            # mutated into PUT during handling. This way logging can display
            # the method the client actually sent.
            if req.method == 'PUT' and req.headers.get('X-Copy-From'):
                return self.handle_PUT(req, start_response)
            elif req.method == 'COPY':
                req.environ['swift.orig_req_method'] = req.method
                return self.handle_COPY(req, start_response,
                                        account, container, obj)
            elif req.method == 'OPTIONS':
                # Does not interfere with OPTIONS response from
                # (account,container) servers and /info response.
                return self.handle_OPTIONS(req, start_response)

        except HTTPException as e:
            return e(req.environ, start_response)

        return self.app(env, start_response)
    def __call__(self, env, start_response):
        self.logger.debug('Calling Utilization Middleware')

        req = Request(env)
        if self.check_api_call(env):
            return self.GET(req)(env, start_response)

        try:
            version, account, container, obj = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        remote_user = env.get('REMOTE_USER')
        if not remote_user or (isinstance(remote_user, basestring) and
                               remote_user.startswith('.wsgi')):
            self.logger.debug('### SKIP: REMOTE_USER is %s' % remote_user)
            return self.app(env, start_response)

        start_response_args = [None]
        input_proxy = InputProxy(env['wsgi.input'])
        env['wsgi.input'] = input_proxy

        def my_start_response(status, headers, exc_info=None):
            start_response_args[0] = (status, list(headers), exc_info)

        def iter_response(iterable):
            iterator = iter(iterable)
            try:
                chunk = next(iterator)
                while not chunk:
                    chunk = next(iterator)
            except StopIteration:
                chunk = ''

            if start_response_args[0]:
                start_response(*start_response_args[0])

            bytes_sent = 0
            try:
                while chunk:
                    bytes_sent += len(chunk)
                    yield chunk
                    chunk = next(iterator)
            finally:
                try:
                    self.publish_sample(env, account,
                                        input_proxy.bytes_received,
                                        bytes_sent)
                except Exception:
                    self.logger.exception('Failed to publish samples')

        try:
            iterable = self.app(env, my_start_response)
        except Exception:
            self.publish_sample(env, account, input_proxy.bytes_received, 0)
            raise
        else:
            return iter_response(iterable)
Ejemplo n.º 18
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.version = version
        self.account_name = account

        try:
            # Check if fastcopy is possible
            # only plain object as source and destination
            # FIXME: handle COPY method (with Destination-* headers)
            check, source_resp = self.fast_copy_allowed(req)
            if check:
                self.logger.debug("COPY: fast copy allowed")
                env['HTTP_OIO_COPY_FROM'] = unquote(env['HTTP_X_COPY_FROM'])
                del env['HTTP_X_COPY_FROM']
                # handle metadata
                if not config_true_value(req.headers.get('x-fresh-metadata',
                                                         'false')):

                    # cf copy middlware
                    exclude_headers = ('x-static-large-object', 'etag',
                                       'x-object-manifest', 'content-type',
                                       'x-timestamp', 'x-backend-timestamp')
                    # copy original headers but don't overwrite source headers
                    copy_header_subset(
                        source_resp, req,
                        lambda k: k.lower() not in exclude_headers
                        and k.lower() not in req.headers)

                def _start_response(status, headers, exc_info=None):
                    headers.append(('X-Copied-From-Account',
                                    env.get('HTTP_X_COPY_FROM_ACCOUNT',
                                            self.account_name)))
                    headers.append(('X-Copied-From',
                                    quote(env['HTTP_OIO_COPY_FROM'])))
                    start_response(status, headers, exc_info)

                return self.app(env, _start_response)
        except HTTPException as exc:
            return exc(req.environ, start_response)

        return super(OioServerSideCopyMiddleware, self).__call__(
            env, start_response)
Ejemplo n.º 19
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            parts = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('PUT', 'POST', 'GET', 'HEAD'):
            # handle only those request methods that may require keys
            km_context = KeyMasterContext(self, *parts[1:])
            try:
                return km_context.handle_request(req, start_response)
            except HTTPException as err_resp:
                return err_resp(env, start_response)

        # anything else
        return self.app(env, start_response)
Ejemplo n.º 20
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            parts = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('PUT', 'POST', 'GET', 'HEAD'):
            # handle only those request methods that may require keys
            km_context = KeyMasterContext(self, *parts[1:])
            try:
                return km_context.handle_request(req, start_response)
            except HTTPException as err_resp:
                return err_resp(env, start_response)

        # anything else
        return self.app(env, start_response)
Ejemplo n.º 21
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            version, acc, cont, obj = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        try:
            if obj:
                # object context
                context = SymlinkObjectContext(self.app, self.logger,
                                               self.symloop_max)
                return context.handle_object(req, start_response)
            else:
                # container context
                context = SymlinkContainerContext(self.app, self.logger)
                return context.handle_container(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Ejemplo n.º 22
0
    def __call__(self, env, start_response):
        if env['REQUEST_METHOD'] != 'GET':
            return self.app(env, start_response)
        req = Request(env)
        version, account, container, obj = req.split_path(1, 4, True)
        if None in [version, account, container, obj]:
            return self.app(env, start_response)
        if random.randint(1, self.max_range) > self.count_threshold:
            return self.app(env, start_response)

        return_status = [None, None]
        start_t = time.time()

        def start_response_wc(status, headers):
            return_status[0] = status
            content_len = 0
            for item in headers:
                if item[0] == "Content-Length":
                    content_len = long(item[1])
            headers.append(('objslo_start_time', start_t))
            headers.append(('objslo_end_time', time.time()))
            return_status[1] = content_len
            return start_response(status, headers)

        if env['REQUEST_METHOD'] == 'GET':
            server_ip = env['SERVER_NAME']
            server_port = str(env['SERVER_PORT'])
            device = ":".join([server_ip, server_port])
            self.redis_client.hincrby(self.incomingreqkey, device, 1 * self.sample_ratio)
            data = self.app(env, start_response_wc)
            duration = time.time() - start_t
            if self.durationswitch:
                self.redis_client.lpush("%s_duration" % (device), duration)
            if duration > self.slolatency:
                self.redis_client.hincrby(self.sloviolatecounter, device, 1 * self.sample_ratio)
            else:
                self.redis_client.hincrby(self.slomeetcounter, device, 1 * self.sample_ratio)
            content_len = return_status[1]
            self.redis_client.hincrby(self.returnreqkey, device, 1 * self.sample_ratio)
            self.redis_client.hincrby(self.returnreqsizekey, device, content_len * self.sample_ratio)
            self.redis_client.hincrby(self.returnreqlatencykey, device, int(duration*1000) * self.sample_ratio)
            self.redis_client.hincrby(self.readcountskey, device, (int(content_len/self.blocksize) + 1) * self.sample_ratio)
            return data
Ejemplo n.º 23
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            parts = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if parts[3] and req.method in ('GET', 'HEAD'):
            handler = DecrypterObjContext(self, self.logger).handle
        elif parts[2] and req.method == 'GET':
            handler = DecrypterContContext(self, self.logger).handle
        else:
            # url and/or request verb is not handled by decrypter
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Ejemplo n.º 24
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if ((req.method == 'GET' or req.method == 'HEAD')
                and req.params.get('multipart-manifest') != 'get'):
            return GetContext(self, self.logger).\
                handle_request(req, start_response)
        elif req.method == 'PUT':
            error_response = self._validate_x_object_manifest_header(req)
            if error_response:
                return error_response(env, start_response)
        return self.app(env, start_response)
Ejemplo n.º 25
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            parts = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if parts[3] and req.method in ('GET', 'HEAD'):
            handler = DecrypterObjContext(self, self.logger).handle
        elif parts[2] and req.method == 'GET':
            handler = DecrypterContContext(self, self.logger).handle
        else:
            # url and/or request verb is not handled by decrypter
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Ejemplo n.º 26
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if ((req.method == 'GET' or req.method == 'HEAD') and
                req.params.get('multipart-manifest') != 'get'):
            return GetContext(self, self.logger).\
                handle_request(req, start_response)
        elif req.method == 'PUT':
            error_response = self._validate_x_object_manifest_header(req)
            if error_response:
                return error_response(env, start_response)
        return self.app(env, start_response)
Ejemplo n.º 27
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            version, acc, cont, obj = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        try:
            if obj:
                # object context
                context = SymlinkObjectContext(self.app, self.logger,
                                               self.symloop_max)
                return context.handle_object(req, start_response)
            else:
                # container context
                context = SymlinkContainerContext(self.app, self.logger)
                return context.handle_container(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
    def __call__(self, env, start_response):
        request = Request(env)

        if request.method == 'PUT':
            try:
                version, account, container, obj = \
                    request.split_path(1, 4, True)
            except ValueError:
                return self.app(env, start_response)

            # check container creation request
            if account and container and not obj:
                policy_name = request.headers.get('X-Storage-Policy', '')
                default_policy = POLICIES.default.name
                if (policy_name in self.policies) or \
                   (policy_name == '' and default_policy in self.policies):

                    container = unquote(container)
                    if len(container) > constraints. \
                            SOF_MAX_CONTAINER_NAME_LENGTH:
                        resp = HTTPBadRequest(request=request)
                        resp.body = \
                            'Container name length of %d longer than %d' % \
                            (len(container),
                                constraints.SOF_MAX_CONTAINER_NAME_LENGTH)
                        return resp(env, start_response)
            elif account and container and obj:
                # check object creation request
                obj = unquote(obj)

                container_info = get_container_info(
                    env, self.app)
                policy = POLICIES.get_by_index(
                    container_info['storage_policy'])

                if policy.name in self.policies:
                    error_response = sof_check_object_creation(request, obj)
                    if error_response:
                        self.logger.warn("returning error: %s", error_response)
                        return error_response(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 29
0
    def __call__(self, env, start_response):
        request = Request(env)

        if not request.path.startswith(self.endpoint_path):
            return self.app(env, start_response)

        if request.method != 'GET':
            resp = HTTPMethodNotAllowed(req=request, headers={"Allow": "GET"})
            return resp(env, start_response)

        try:
            self.config.read(self.conf["__file__"])
        except (IOError, KeyError):
            resp = HTTPServerError(req=request,
                                   body="An error occurred",
                                   content_type="text/plain")
            return resp(env, start_response)

        config_dict = self._config_parser_to_nested_dict()

        try:
            endpoint, section, option = request.split_path(1, 3, False)
        except ValueError:
            resp = HTTPBadRequest(req=request, headers={})
            return resp(env, start_response)

        try:
            if section and option:
                config_dict = {section: {option: config_dict[section][option]}}
            elif section and not option:
                config_dict = {section: config_dict[section]}
        except KeyError:
            resp = HTTPNotFound(req=request,
                                body="Requested values aren't available",
                                content_type="text/plain")
            return resp(env, start_response)

        resp = Response(req=request,
                        body=json.dumps(config_dict),
                        content_type="application/json")

        return resp(env, start_response)
Ejemplo n.º 30
0
    def __call__(self, env, start_response):
        req = Request(env)

        if req.method not in self.write_methods:
            return self.app(env, start_response)

        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method == 'COPY' and 'Destination-Account' in req.headers:
            dest_account = req.headers.get('Destination-Account')
            account = check_account_format(req, dest_account)

        if self.account_read_only(req, account):
            msg = 'Writes are disabled for this account.'
            return HTTPMethodNotAllowed(body=msg)(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 31
0
    def __call__(self, env, start_response):
        req = Request(env)

        if req.method not in self.write_methods:
            return self.app(env, start_response)

        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method == 'COPY' and 'Destination-Account' in req.headers:
            dest_account = req.headers.get('Destination-Account')
            account = check_account_format(req, dest_account)

        if self.account_read_only(req, account):
            msg = 'Writes are disabled for this account.'
            return HTTPMethodNotAllowed(body=msg)(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 32
0
    def get_target(self, environ):
        req = Request(environ)
        try:
            parts = req.split_path(1, 4, True)
            version, account, container, obj = parts
        except ValueError:
            version = account = container = obj = None

        referrers, acls = swift_acl.parse_acl(getattr(req, 'acl', None))
        target = {
            "req": req,
            "method": req.method.lower(),
            "version": version,
            "account": account,
            "container": container,
            "object": obj,
            "acls": acls,
            "referrers": referrers
        }
        return target
Ejemplo n.º 33
0
 def __call__(self, env, start_response):
     req = Request(env)
     new_service = env.get('liteauth.new_service', None)
     if new_service:
         account_id = env.get('REMOTE_USER', '')
         if not account_id:
             return HTTPInternalServerError()
         if not self.activate_service(account_id, new_service, req.environ):
             return HTTPInternalServerError()
     if req.method in ['PUT', 'POST'
                       ] and not 'x-zerovm-execute' in req.headers:
         account_info = get_account_info(req.environ,
                                         self.app,
                                         swift_source='litequota')
         service_plan = assemble_from_partial(self.metadata_key,
                                              account_info['meta'])
         if service_plan:
             try:
                 service_plan = json.loads(service_plan)
                 path_parts = req.split_path(2, 4, rest_with_last=True)
             except ValueError:
                 return self.app(env, start_response)
             if len(path_parts) == 3:
                 quota = service_plan['storage']['containers']
                 new_size = int(account_info['container_count'])
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: containers')(env, start_response)
             else:
                 new_size = int(account_info['bytes']) + (req.content_length
                                                          or 0)
                 quota = service_plan['storage']['bytes']
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: bytes')(env, start_response)
                 quota = service_plan['storage']['objects']
                 new_size = int(account_info['total_object_count'])
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: objects')(env, start_response)
     return self.app(env, start_response)
Ejemplo n.º 34
0
    def __call__(self, env, start_response):
        request = Request(env)

        if request.method == 'PUT':
            try:
                version, account, container, obj = \
                    request.split_path(1, 4, True)
            except ValueError:
                return self.app(env, start_response)

            # check container creation request
            if account and container and not obj:
                policy_name = request.headers.get('X-Storage-Policy', '')
                default_policy = POLICIES.default.name
                if (policy_name in self.policies) or \
                   (policy_name == '' and default_policy in self.policies):

                    container = unquote(container)
                    if len(container) > constraints. \
                            SOF_MAX_CONTAINER_NAME_LENGTH:
                        resp = HTTPBadRequest(request=request)
                        resp.body = \
                            'Container name length of %d longer than %d' % \
                            (len(container),
                                constraints.SOF_MAX_CONTAINER_NAME_LENGTH)
                        return resp(env, start_response)
            elif account and container and obj:
                # check object creation request
                obj = unquote(obj)

                container_info = get_container_info(env, self.app)
                policy = POLICIES.get_by_index(
                    container_info['storage_policy'])

                if policy.name in self.policies:
                    error_response = sof_check_object_creation(request, obj)
                    if error_response:
                        self.logger.warn("returning error: %s", error_response)
                        return error_response(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 35
0
    def __call__(self, env, start_response):
        # making a duplicate, because if this is a COPY request, we will
        # modify the PATH_INFO to find out if the 'Destination' is in a
        # versioned container
        req = Request(env.copy())
        try:
            (version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get('allow_versioned_writes')
        if allow_versioned_writes and container and not obj:
            try:
                return self.container_request(req, start_response,
                                              allow_versioned_writes)
            except HTTPException as error_response:
                return error_response(env, start_response)
        elif obj and req.method in ('PUT', 'COPY', 'DELETE'):
            try:
                return self.object_request(req, version, account, container,
                                           obj, allow_versioned_writes)(
                                               env, start_response)
            except HTTPException as error_response:
                return error_response(env, start_response)
        else:
            return self.app(env, start_response)
Ejemplo n.º 36
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (api_version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get('allow_versioned_writes')
        if allow_versioned_writes and container and not obj:
            try:
                return self.container_request(req, start_response,
                                              allow_versioned_writes)
            except HTTPException as error_response:
                return error_response(env, start_response)
        elif (obj and (req.method in ('PUT', 'DELETE') and
                       not req.environ.get('swift.post_as_copy') or
                       req.method in ('HEAD', 'GET'))):
            try:
                return self.object_request(
                    req, api_version, account, container, obj,
                    allow_versioned_writes)(env, start_response)
            except HTTPException as error_response:
                return error_response(env, start_response)
        else:
            return self.app(env, start_response)
Ejemplo n.º 37
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (api_version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get('allow_versioned_writes')
        if allow_versioned_writes and container and not obj:
            try:
                return self.container_request(req, start_response,
                                              allow_versioned_writes)
            except HTTPException as error_response:
                return error_response(env, start_response)
        elif (obj and (req.method in ('PUT', 'DELETE') and
                       not req.environ.get('swift.post_as_copy') or
                       req.method in ('HEAD', 'GET', 'POST'))):
            try:
                return self.object_request(
                    req, api_version, account, container, obj,
                    allow_versioned_writes)(env, start_response)
            except HTTPException as error_response:
                return error_response(env, start_response)
        else:
            return self.app(env, start_response)
Ejemplo n.º 38
0
 def __call__(self, env, start_response):
     req = Request(env)
     new_service = env.get('liteauth.new_service', None)
     if new_service:
         account_id = env.get('REMOTE_USER', '')
         if not account_id:
             return HTTPInternalServerError()
         if not self.activate_service(account_id, new_service, req.environ):
                 return HTTPInternalServerError()
     if req.method in ['PUT', 'POST'] and not 'x-zerovm-execute' in req.headers:
         account_info = get_account_info(req.environ, self.app,
                                         swift_source='litequota')
         service_plan = assemble_from_partial(self.metadata_key,
                                              account_info['meta'])
         if service_plan:
             try:
                 service_plan = json.loads(service_plan)
                 path_parts = req.split_path(2, 4, rest_with_last=True)
             except ValueError:
                 return self.app(env, start_response)
             if len(path_parts) == 3:
                 quota = service_plan['storage']['containers']
                 new_size = int(account_info['container_count'])
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: containers')(env, start_response)
             else:
                 new_size = int(account_info['bytes']) + (req.content_length or 0)
                 quota = service_plan['storage']['bytes']
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: bytes')(env, start_response)
                 quota = service_plan['storage']['objects']
                 new_size = int(account_info['total_object_count'])
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: objects')(env, start_response)
     return self.app(env, start_response)
Ejemplo n.º 39
0
    def __call__(self, env, start_response):

        request = Request(env.copy())
        if request.method != "GET":
            return self.app(env, start_response)

        (version, acc, con, obj) = request.split_path(1, 4, False)
        if not obj:
            return self.app(env, start_response)

        memcache_key = 'RequestCache/%s/%s/%s' % (acc, con, obj)

        cached_content = self.memcache.get(memcache_key)
        if cached_content:
            response = Response(request=request, body=cached_content)
            response.headers['X-RequestCache'] = 'True'
            return response(env, start_response)

        sub_req = wsgi.make_subrequest(env)
        sub_resp = sub_req.get_response(self.app)
        self.memcache.set(memcache_key, sub_resp.body, time=86400.0)

        return self.app(env, start_response)
Ejemplo n.º 40
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (api_version, account, container, obj) = req.split_path(2, 4, True)
            bad_path = False
        except ValueError:
            bad_path = True

        # use of bad_path bool is to avoid recursive tracebacks
        if bad_path or not valid_api_version(api_version):
            return self.app(env, start_response)

        try:
            if not container:
                return self.account_request(req, api_version, account,
                                            start_response)
            if container and not obj:
                return self.container_request(req, start_response)
            else:
                return self.object_request(req, api_version, account,
                                           container, obj)(env, start_response)
        except HTTPException as error_response:
            return error_response(env, start_response)
Ejemplo n.º 41
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        if env.get('swift.slo_override'):
            return self.app(env, start_response)

        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if not obj:
            if req.method == 'GET':
                return self.handle_container_listing(req, start_response)
            return self.app(env, start_response)

        try:
            if req.method == 'PUT' and \
                    req.params.get('multipart-manifest') == 'put':
                return self.handle_multipart_put(req, start_response)
            if req.method == 'DELETE' and \
                    req.params.get('multipart-manifest') == 'delete':
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == 'GET' or req.method == 'HEAD':
                return self.handle_multipart_get_or_head(req, start_response)
            if 'X-Static-Large-Object' in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body='X-Static-Large-Object is a reserved header. '
                    'To create a static large object add query param '
                    'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 42
0
    def __call__(self, env, start_response):
        """
        WSGI entry point.
        Wraps env in swob.Request object and passes it down.

        :param env: WSGI environment dictionary
        :param start_response: WSGI callable
        """
        req = Request(env)
        if self.memcache_client is None:
            self.memcache_client = cache_from_env(env)
        if not self.memcache_client:
            self.logger.warning(
                _('Warning: Cannot ratelimit without a memcached client'))
            return self.app(env, start_response)
        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
        if ratelimit_resp is None:
            return self.app(env, start_response)
        else:
            return ratelimit_resp(env, start_response)
Ejemplo n.º 43
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        if ((req.method == 'GET' or req.method == 'HEAD')
                and req.params.get('multipart-manifest') != 'get'):
            return GetContext(self, self.logger).\
                handle_request(req, start_response)
        elif req.method == 'PUT':
            error_response = self._validate_x_object_manifest_header(req)
            if error_response:
                return error_response(env, start_response)
        return self.app(env, start_response)
Ejemplo n.º 44
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
            if DEBUG:
                print('obj:%s' % obj)
        except ValueError:
            return self.app(env, start_response)
        try:
            if env['REQUEST_METHOD'] == "PUT":
                # Read the object content and enc_key in memory
                # encrypt the object with the enc_key
                content = env['wsgi.input'].read()
                if content:
                    return Response(status=403,
                                    body="content %s detected" % content,
                                    content_type="text/plain")(env,
                                                               start_response)
                else:
                    return Response(status=403,
                                    body="content not detected",
                                    content_type="text/plain")(env,
                                                               start_response)
                if DEBUG:
                    print(content)
                #encryptor = Encryptor()
                content = content + "yyyyyyyyyyyyyyyyy"
            # Maybe add the decrytion process later
            #elif env['REQUEST_METHOD'] == "GET":
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 45
0
    def __call__(self, env, start_response):
        req = Request(env)

        # Split request path to determine version, account, container, object
        try:
            (version, account, container, obj) = req.split_path(2, 4, True)
        except ValueError:
            self.logger.debug('split_path exception')
            return self.app(env, start_response)
        self.logger.debug(':%s:%s:%s:%s:', version, account, container, obj)

        # If request is not HLM request or not a GET, it is not processed
        # by this middleware
        method = req.method
        query = req.query_string or ''
        if not (method == 'POST'
                and ('MIGRATE' in query
                     or 'RECALL' in query)
                or method == 'GET'):
            return self.app(env, start_response)

        # Process GET object data request, if object is migrated return error
        # code 412 'Precondition Failed' (consider using 455 'Method Not Valid
        # in This State') - the error code is returned if any object replica is
        # migrated.
        # TODO: provide option to return error code only if all replicas are
        # migrated, and redirect get request to one of non-migrated replicas
        if req.method == "GET" and obj and 'STATUS' not in query:
            # check status and either let GET proceed or return error code
            rc, out, replicas_status = self.get_object_replicas_status(
                req, account, container, obj)
            if rc == REMOTE_STATUS:
                #send the replica status to requester node
                return Response(status=HTTP_OK,
                                body=out,
                                content_type="text/plain")(env,
                                                           start_response)
            self.logger.debug('replicas_status %s', str(replicas_status))
            ret_error = False
            for replica_status in replicas_status:
                status = literal_eval(replica_status)['status']
                if status not in ['resident', 'premigrated']:
                    ret_error = True
            if ret_error:
                return Response(status=HTTP_PRECONDITION_FAILED,
                                body="Object %s needs to be RECALL-ed before "
                                "it can be accessed.\n" %
                                literal_eval(replicas_status[0])['object'],
                                content_type="text/plain")(env, start_response)

            return self.app(env, start_response)

        # Process POST request to migrate/recall object
        elif method == 'POST' and obj:
            if 'MIGRATE' in query or 'RECALL' in query:
                if 'MIGRATE' in query:
                    hlm_req = 'MIGRATE'
                    hlm_backend = self.migrate_backend
                elif 'RECALL' in query:
                    hlm_req = 'RECALL'
                    hlm_backend = self.recall_backend
                # submit hlm request for object replicas
                status, out = self.submit_object_replicas_migration_recall(
                    req, account, container, obj, hlm_req, hlm_backend)
                self.logger.debug('submit_object_replicas_migration_recall()')
                if status == SUBMITTED_FORWARDED_REQUEST:
                    self.logger.debug('SUBMITTED_FORWARDED_REQUEST')
                    return Response(status=HTTP_OK,
                                    body='Accepted remote replica HLM request',
                                    content_type="text/plain")(env,
                                                               start_response)
                elif status == FAILED_SUBMITTING_REQUEST:
                    self.logger.debug('FAILED_SUBMITTING_REQUEST')
                    return Response(status=HTTP_INTERNAL_SERVER_ERROR,
                                    body=out,
                                    content_type="text/plain")(env,
                                                               start_response)
                elif status == SUBMITTED_REQUESTS:
                    self.logger.debug('SUBMITTED_REQUESTS')
                    return Response(status=HTTP_OK,
                                    body='Accepted %s request.\n' % hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
                else:  # invalid case
                    self.logger.debug('INVALID_CASE')
                    return Response(status=HTTP_INTERNAL_SERVER_ERROR,
                                    body=out,
                                    content_type="text/plain")(env,
                                                               start_response)

        # Process GET object status request
        elif req.method == "GET" and obj:
            if 'STATUS' in query:
                # Get status of each replica
                rc, out, replicas_status = self.get_object_replicas_status(
                    req, account, container, obj)
                if rc == REMOTE_STATUS:
                    # send the replica status to requester node
                    return Response(status=HTTP_OK,
                                    body=out,
                                    content_type="text/plain")(env,
                                                               start_response)
                # Prepare/format object status info to report
                # (json is default format)
                out = self.format_object_status_info_for_reporting(
                    req, replicas_status) + '\n'
                # Report object status
                return Response(status=HTTP_OK,
                                body=out,
                                content_type="text/plain")(env, start_response)

        # Process container request
        if (container and not obj and
           ((method == 'POST' and ('MIGRATE' in query or 'RECALL' in query))
                or method == 'GET' and 'STATUS' in query)):
            self.logger.debug('Process container request')
            # Get list of objects
            list_url = 'http://%(ip)s:8080%(url)s'
            list_req = list_url % {'ip': self.ip,
                                   'url': req.path}
            self.logger.debug('list_req: %s', list_req)
            self.logger.debug('req.headers: %s', str(req.headers))
            token = req.headers['X-Storage-Token']
            self.logger.debug('token: %s', token)
            headers = {'X-Storage-Token': token}
            response = requests.get(list_req, headers=headers)
            self.logger.debug('response.headers: %s', str(response.headers))
            self.logger.debug('list: %s', str(response.content))
            objects = response.content.strip().split('\n')
            # Submit migration or recall
            if method == 'POST':
                if 'MIGRATE' in query:
                    hlm_req = 'MIGRATE'
                    hlm_backend = self.migrate_backend
                elif 'RECALL' in query:
                    hlm_req = 'RECALL'
                    hlm_backend = self.recall_backend
                # submit hlm requests
                success = 0
                failure = 0
                for obj in objects:
                    self.logger.debug('obj: %s', obj)
                    status, out = self.submit_object_replicas_migration_recall(
                        req, account, container, obj, hlm_req, hlm_backend)
                    self.logger.debug('submit_object_replicas_migr.._recall()')
                    if status == SUBMITTED_FORWARDED_REQUEST:
                        self.logger.debug('SUBMITTED_FORWARDED_REQUEST')
                        return Response(status=HTTP_OK,
                                        body='Accepted remote replica'
                                        'HLM request',
                                        content_type="text/plain")(
                            env, start_response)
                    elif status == FAILED_SUBMITTING_REQUEST:
                        self.logger.debug('FAILED_SUBMITTING_REQUEST')
                        failure += 1
                    elif status == SUBMITTED_REQUESTS:
                        self.logger.debug('SUBMITTED_REQUESTS')
                        success += 1
                if failure == 0:
                    return Response(status=HTTP_OK,
                                    body='Submitted %s requests.\n' % hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
                elif success == 0:
                    return Response(status=HTTP_INTERNAL_SERVER_ERROR,
                                    body='Failed to submit %s requests.\n' %
                                    hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
                else:
                    return Response(status=HTTP_OK,
                                    body="Submitting %s requests"
                                    " is only partially"
                                    " successful.\n" % hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
            elif method == 'GET':
                # submit hlm requests
                accumulated_out = "["
                for obj in objects:
                    self.logger.debug('obj: %s', obj)

                    # Get status of each replica
                    # rewrite req.path to point to object instead of container
                    req_orig_path = req.environ['PATH_INFO']
                    req.environ['PATH_INFO'] += "/" + obj
                    rc, out, replicas_status = self.get_object_replicas_status(
                        req, account, container, obj)

                    # Prepare/format object status info to report
                    # (json is default format)
                    accumulated_out += self.format_object_status_info_for_reporting(
                        req, replicas_status) + ','

                    req.environ['PATH_INFO'] = req_orig_path

                # Report accumulated object status
                accumulated_out = accumulated_out[:-1] + ']'
                return Response(status=HTTP_OK,
                                body=accumulated_out,
                                content_type="text/plain")(env, start_response)

        return self.app(env, start_response)
Ejemplo n.º 46
0
class AutosyncMiddleware(object):
    def __init__(self, app, conf):
        self.app = app
        self.conf = conf
        self.req_timeout = 2
        self.conn_timeout = 10
        self.keychars = string.ascii_letters + string.digits
        self.logger = get_logger(self.conf, log_route='nemo')
        self.override_auth = \
            config_true_value(conf.get('override_auth', False))
        self.default_my_cluster = conf.get('autosync_my_cluster', None)
        self.default_placement = conf.get('autosync_placement',
                                          [self.default_my_cluster])
        if self.default_placement:
            self.default_placement = self.default_placement.split(',')

    #def redirect(self):
    #    peer = choice(self.placement)
    #    resp = HTTPMovedPermanently(location=(peer + self.env['PATH_INFO']))
    #    return resp(self.env, self.start_response)

    def send_to_peer(self, peer, sync_to_peer, key):
        peer = peer.lower()
        ssl = False
        if peer.startswith('https://'):
            ssl = True
            peer = peer[8:]
        if peer.startswith('http://'):
            peer = peer[7:]
        try:
            with Timeout(self.conn_timeout):
                if ssl:
                    #print 'HTTPS %s ' % peer
                    conn = HTTPSConnection(peer)
                else:
                    #print 'HTTP %s ' % peer
                    conn = HTTPConnection(peer)
            conn.putrequest(self.req.method, self.req.path_qs)
            conn.putheader('X-Orig-Cluster', self.my_cluster)
            conn.putheader('X-Account-Meta-Orig-Cluster', self.my_cluster)
            conn.putheader('X-Container-Meta-Orig-Cluster', self.my_cluster)
            if key:
                sync_to = sync_to_peer + self.env['PATH_INFO']
                conn.putheader('X-Container-Sync-To', sync_to)
            for header, value in self.req.headers.iteritems():
                if header != 'X-Container-Sync-To':
                    conn.putheader(header, value)
            conn.endheaders(message_body=None)
            with Timeout(self.req_timeout):
                resp = conn.getresponse()
                status = resp.status
                return (status, resp.getheaders(), resp.read())
        except (Exception, Timeout) as e:
            # Print error log
            print >> sys.stderr, peer + ': Exception, Timeout error: %s' % e

        print '<<<<<<<< HTTP_SERVICE_UNAVAILABLE'
        #return HTTP_SERVICE_UNAVAILABLE, None, None

    def send_to_peers(self, peers, key):
        pile = GreenPile(len(peers))
        # Have the first peer to sync to the local cluster
        sync_to_peer = self.my_cluster
        for peer in peers:
            # create thread per peer and send a request
            pile.spawn(self.send_to_peer, peer, sync_to_peer, key)
            # Have the next peer to sync to the present peer
            sync_to_peer = peer
        # collect the results, if anyone failed....
        response = [resp for resp in pile if resp]
        while len(response) < len(peers):
            response.append((HTTP_SERVICE_UNAVAILABLE, None, None))
        return response

    def highest_response(self, resps, swap={}):
        highest_resp = None
        highest_status = -1
        for resp in resps:
            status = resp[0]
            if status in swap:
                status = swap[status]
            status = int(status)
            if status > highest_status:
                highest_status = status
                highest_resp = resp
        if highest_resp:
            return Response(body=highest_resp[2], status=highest_resp[0],
                            headers=highest_resp[1])
        return HTTPServiceUnavailable(request=self.req)

    def all_success(self, resps):
        for resp in resps:
            if not is_success(resp[0]):
                return False
        return True

    def __call__(self, env, start_response):
        def my_start_response(status, headers, exc_info=None):
            self.status = status
            self.headers = list(headers)
            self.exc_info = exc_info
        self.env = env
        self.start_response = start_response

        # If request was already processed by autosync
        # (here or at the original cluster where it first hit)
        if 'HTTP_X_ORIG_CLUSTER' in env:
            print >> sys.stderr, 'HTTP_X_ORIG_CLUSTER found!'
            if self.override_auth:
                env['swift_owner'] = True
            return self.app(env, start_response)

        # If it is a local call or a tempurl object call
        if 'swift.authorize_override' in env:
            return self.app(env, start_response)

        # Get Placement parameters
        if 'swift.my_cluster' in env:
            self.my_cluster = env['swift.my_cluster']
        else:
            self.my_cluster = self.default_my_cluster

        if 'swift.placement' in env:
            placement = env['swift.placement']
        else:
            placement = self.default_placement or self.my_cluster
        
        if not self.my_cluster or not placement:
            return self.app(env, start_response)

        self.req = Request(env)
        # For now we support only placement here and in one other place
        if self.my_cluster not in placement:
            return HTTPInternalServerError(request=self.req)
        #   return self.redirect()

        peers = [p for p in placement if p != self.my_cluster]
        if len(peers) != 1:
            return HTTPInternalServerError(request=self.req) 

        # This request needs to be handled localy
        try:
            (version, account, container, obj) = \
                self.req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)
        if obj or self.req.method in ('OPTIONS', 'GET', 'HEAD'):
            # business as usual - I will serve the request locally and be done
            # TBD, in case of 404 returned from GET object, try a remote copy?
            return self.app(env, start_response)

        # Lets see, its either PUT, POST or DELETE account/container
        # Otherwise said - 'we need to change the account/container'
        # both here and with peers...

        # As part of any container creation/modification (PUT/POST):
        # Create a new key  to protect the container communication from now
        # and until the next time the container is updated.
        # Note that race may occur with container-sync daemons resulting in
        # container-sync failing due to misaligned keys.
        # Changing the keys per update help support changes in the placement
        # and can serve as a simple mechanism for replacing conatienr sync keys
        # If this turns out to be an issue, we may extract and reuse the same
        # key for the duration of the container existance. 
        if container and self.req.method in ['POST', 'PUT']:
            key = ''.join(choice(self.keychars) for x in range(64))
            # Add the key to the env when calling the local cluster 
            env['HTTP_X_CONTAINER_SYNC_KEY'] = key
            # Set the container replica of the local cluster to sync to the
            # last cluster in the list of peers
            sync_to_peer = peers[-1]  # Sync to the prev peer
            sync_to = sync_to_peer + self.env['PATH_INFO']
            env['HTTP_X_CONTAINER_SYNC_TO'] = sync_to
        else:
            key = None  # Signals that there are no Container-Sync headers

        # Try localy, if we fail and not DELETE respond with a faliure.
        resp_data = self.app(self.env, my_start_response)
        data = ''.join(iter(resp_data))
        if hasattr(resp_data, 'close'):
            resp_data.close()
        resp_status_int = int(self.status[:3])


        # Faliure at local cluster during anything but DELETE... abandon ship
        if not is_success(resp_status_int) and self.req.method != 'DELETE':
            # Dont even try the peers
            start_response(self.status, self.headers, self.exc_info)
            return data

        # Call peers and accomulate responses
        try:
            # Note that key is None if not during container PUT/POST
            resps = self.send_to_peers(peers, key)
            # Append the local cluster response
            resps.append((resp_status_int, self.headers, data))
        except:
            return HTTPServiceUnavailable(request=self.req)

        resp = None
        if self.req.method == 'DELETE':
            # Special treatment to DELETE - respond with the best we have
            resp = self.highest_response(resps, swap={'404': '1'})
        else:  # PUT/POST - respond only if all success
            if self.all_success(resps):
                resp = self.highest_response(resps)
            else:
                # PUT/POST with local success and remote faliure
                resp = HTTPServiceUnavailable(request=self.req)
        return resp(env, start_response)
Ejemplo n.º 47
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            version, account, container, obj = req.split_path(
                2, 4, rest_with_last=True)
            is_swifty_request = valid_api_version(version)
        except ValueError:
            is_swifty_request = False

        if not is_swifty_request:
            return self.app(env, start_response)

        if not obj:
            typ = 'Container' if container else 'Account'
            client_header = 'X-%s-Rfc-Compliant-Etags' % typ
            sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ
            if client_header in req.headers:
                if req.headers[client_header]:
                    req.headers[sysmeta_header] = config_true_value(
                        req.headers[client_header])
                else:
                    req.headers[sysmeta_header] = ''
            if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)):
                req.headers[sysmeta_header] = ''

            def translating_start_response(status, headers, exc_info=None):
                return start_response(
                    status,
                    [(client_header if h.title() == sysmeta_header else h, v)
                     for h, v in headers], exc_info)

            return self.app(env, translating_start_response)

        container_info = get_container_info(env, self.app, 'EQ')
        if not container_info or not is_success(container_info['status']):
            return self.app(env, start_response)

        flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags')
        if flag is None:
            account_info = get_account_info(env, self.app, 'EQ')
            if not account_info or not is_success(account_info['status']):
                return self.app(env, start_response)

            flag = account_info.get('sysmeta', {}).get('rfc-compliant-etags')

        if flag is None:
            flag = self.conf.get('enable_by_default', 'false')

        if not config_true_value(flag):
            return self.app(env, start_response)

        status, headers, resp_iter = req.call_application(self.app)

        for i, (header, value) in enumerate(headers):
            if header.lower() == 'etag':
                if not value.startswith(('"', 'W/"')) or \
                        not value.endswith('"'):
                    headers[i] = (header, '"%s"' % value)

        start_response(status, headers)
        return resp_iter
Ejemplo n.º 48
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.account_name = account
        self.container_name = container
        self.object_name = obj

        # Check if fastcopy is possible
        # (only plain object at this time)
        if req.headers.get('X-Copy-From') and not (
                req.headers.get('Range')
                or req.headers.get('X-Amz-Copy-Source-Range')):
            # TODO: it could be also a copy from MPU to full object
            # but I pretty sure that this case is explained on AWS doc
            req.headers['Oio-Copy-From'] = req.headers.get('X-Copy-From')
            del req.headers['X-Copy-From']
            return self.app(env, start_response)
        """ TODO (how access proxy/obj.py instance to checking SLO ?)
            # compare range and segments

            ranges = ranges_from_http_header(req.headers.get('Range'))
            if len(ranges) == 1:
                ranges = ranges[0]

                # check that ORG object is SLO
                container, obj = req.headers['X-Copy-From'].split('/', 1)
                storage = self.app.storage
                props = storage.object_get_properties(
                    self.account_name, container, obj)
                if props['properties'].get(SLO, None):
                    manifest = json.loads("".join(data))
                offset = 0
                # identify segment to copy
                for entry in manifest:
                    if ranges[0] == offset and \
                            ranges[1] + 1 == offset + entry['bytes']:
                        _, container, obj = entry['name'].split('/', 2)
                        checksum = entry['hash']
                        self.app.logger.info(
                            "LINK SLO (%s,%s,%s) TO (%s,%s,%s)",
                            self.account_name, self.container_name,
                            self.object_name,
                            self.account_name, container, obj)
                        return self.app(env, start_response)
                        break
                    offset += entry['bytes']
                else:
                    print("NO SEGMENT FOUND, FALLACK")
        """
        try:
            # In some cases, save off original request method since it gets
            # mutated into PUT during handling. This way logging can display
            # the method the client actually sent.
            if req.method == 'PUT' and req.headers.get('X-Copy-From'):
                return self.handle_PUT(req, start_response)
            elif req.method == 'COPY':
                req.environ['swift.orig_req_method'] = req.method
                return self.handle_COPY(req, start_response)
            elif req.method == 'POST' and self.object_post_as_copy:
                req.environ['swift.orig_req_method'] = req.method
                return self.handle_object_post_as_copy(req, start_response)
            elif req.method == 'OPTIONS':
                # Does not interfere with OPTIONS response from
                # (account,container) servers and /info response.
                return self.handle_OPTIONS(req, start_response)

        except HTTPException as e:
            return e(req.environ, start_response)

        return self.app(env, start_response)
Ejemplo n.º 49
0
    def handle_request(self, env, start_response):
        request = Request(env)
        method = request.method
        if method not in ('POST', 'PUT', 'COPY', 'DELETE'):
            return self.app(env, start_response)

        # Get the response from the rest of the pipeline before we
        # start doing anything; this means that whatever is being created
        # or deleted will have been done before we start constructing
        # the notification payload
        response = self._app_call(env)
        status_code = self._get_status_int()

        try:
            ver, account, container, obj = request.split_path(
                2, 4, rest_with_last=True)
        except ValueError:
            start_response(self._response_status, self._response_headers,
                           self._response_exc_info)
            return response

        event_methods = {
            'DELETE': 'ObjectRemoved:Delete',
            'COPY': 'ObjectCreated:Copy',
            'PUT': 'ObjectCreated:Put',
            'POST': 'ObjectModified'
        }

        event_object = ('object'
                        if obj else 'container' if container else 'account')

        event_type = '%s' % (event_methods[method])

        if status_code in (200, 201, 202, 204):
            request_headers = request.headers

            payload = self._get_request_auth_info(request_headers)
            payload['object_type'] = event_object
            payload['account'] = account
            if container:
                payload['container'] = container
                if obj:
                    payload['object'] = obj

            if method != 'DELETE':
                head_headers = self._make_head_request(env).headers

                copy_from = request_headers.get('X-Copy-From')
                if method == 'PUT' and copy_from:
                    # Copies are turned into PUTs with an X-Copy-From
                    # in the object middleware
                    # though we don't need to handle them differently
                    event_type = event_methods['COPY']
                    if copy_from[0] == '/':
                        copy_from = copy_from[1:]
                    copy_from_container, copy_from_object = copy_from.split(
                        '/', 1)

                    payload['copy-from-container'] = copy_from_container
                    payload['copy-from-object'] = copy_from_object

                    if request_headers.get('X-Fresh-Metadata', None):
                        payload['copy-fresh-metadata'] = bool(
                            request_headers.get('X-Fresh-Metadata'))

                payload.update(
                    self._get_account_metadata(request_headers,
                                               self._response_headers))
                if container:
                    payload.update(
                        self._get_container_metadata(request_headers,
                                                     self._response_headers))

                    if obj:
                        payload.update(
                            self._get_object_metadata(request_headers,
                                                      self._response_headers))

                modified_timestamp = head_headers.get('X-Timestamp')
                if modified_timestamp:
                    modified_datetime = datetime.fromtimestamp(
                        float(modified_timestamp))
                    payload['updated_at'] = modified_datetime.strftime(
                        '%Y-%m-%dT%H:%M:%S.%f')
                    payload['x-timestamp'] = modified_timestamp

                def set_field_if_exists(source, dest):
                    value = head_headers.get(source)
                    if value:
                        payload[dest] = value

                set_field_if_exists('Last-Modified', 'last-modified')

                if obj:
                    for field in (('Content-Length', 'content-length'),
                                  ('Content-Type', 'content-type')):
                        set_field_if_exists(*field)

            self._notifier.notify({}, event_type, payload)

        # We don't want to tamper with the response
        start_response(self._response_status, self._response_headers,
                       self._response_exc_info)

        return response
Ejemplo n.º 50
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            # account and container only
            version, acct, cont = req.split_path(2, 3)
        except ValueError:
            is_account_or_container_req = False
        else:
            is_account_or_container_req = True
        if not is_account_or_container_req:
            return self.app(env, start_response)

        if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
            return self.app(env, start_response)

        # OK, definitely have an account/container request.
        # Get the desired content-type, then force it to a JSON request.
        try:
            out_content_type = get_listing_content_type(req)
        except HTTPException as err:
            return err(env, start_response)

        params = req.params
        can_vary = 'format' not in params
        params['format'] = 'json'
        req.params = params

        # Give other middlewares a chance to be in charge
        env.setdefault('swift.format_listing', True)
        status, headers, resp_iter = req.call_application(self.app)
        if not env.get('swift.format_listing'):
            start_response(status, headers)
            return resp_iter

        header_to_index = {}
        resp_content_type = resp_length = None
        for i, (header, value) in enumerate(headers):
            header = header.lower()
            if header == 'content-type':
                header_to_index[header] = i
                resp_content_type = value.partition(';')[0]
            elif header == 'content-length':
                header_to_index[header] = i
                resp_length = int(value)
            elif header == 'vary':
                header_to_index[header] = i

        if not status.startswith(('200 ', '204 ')):
            start_response(status, headers)
            return resp_iter

        if can_vary:
            if 'vary' in header_to_index:
                value = headers[header_to_index['vary']][1]
                if 'accept' not in list_from_csv(value.lower()):
                    headers[header_to_index['vary']] = ('Vary',
                                                        value + ', Accept')
            else:
                headers.append(('Vary', 'Accept'))

        if resp_content_type != 'application/json':
            start_response(status, headers)
            return resp_iter

        if resp_length is None or \
                resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
            start_response(status, headers)
            return resp_iter

        def set_header(header, value):
            if value is None:
                del headers[header_to_index[header]]
            else:
                headers[header_to_index[header]] = (
                    headers[header_to_index[header]][0], str(value))

        if req.method == 'HEAD':
            set_header('content-type', out_content_type + '; charset=utf-8')
            set_header('content-length', None)  # don't know, can't determine
            start_response(status, headers)
            return resp_iter

        body = b''.join(resp_iter)
        try:
            listing = json.loads(body)
            # Do a couple sanity checks
            if not isinstance(listing, list):
                raise ValueError
            if not all(isinstance(item, dict) for item in listing):
                raise ValueError
        except ValueError:
            # Static web listing that's returning invalid JSON?
            # Just pass it straight through; that's about all we *can* do.
            start_response(status, headers)
            return [body]

        if not req.allow_reserved_names:
            listing = self.filter_reserved(listing, acct, cont)

        try:
            if out_content_type.endswith('/xml'):
                if cont:
                    body = container_to_xml(
                        listing,
                        wsgi_to_bytes(cont).decode('utf-8'))
                else:
                    body = account_to_xml(listing,
                                          wsgi_to_bytes(acct).decode('utf-8'))
            elif out_content_type == 'text/plain':
                body = listing_to_text(listing)
            else:
                body = json.dumps(listing).encode('ascii')
        except KeyError:
            # listing was in a bad format -- funky static web listing??
            start_response(status, headers)
            return [body]

        if not body:
            status = '%s %s' % (HTTP_NO_CONTENT,
                                RESPONSE_REASONS[HTTP_NO_CONTENT][0])

        set_header('content-type', out_content_type + '; charset=utf-8')
        set_header('content-length', len(body))
        start_response(status, headers)
        return [body]
Ejemplo n.º 51
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if account is None:
            return self.app(env, start_response)

        if env.get('swift.authorize_override', False):
            return self.app(env, start_response)

        # First, restrict modification of auth meta-data to only users with
        # the admin role (or roles that have been specially enabled in
        # the swift config).
        if req.method == "POST":
            # following code to get roles is borrowed from keystoneauth
            roles = set()
            if (env.get('HTTP_X_IDENTITY_STATUS') == 'Confirmed'
                    or env.get('HTTP_X_SERVICE_IDENTITY_STATUS')
                    in (None, 'Confirmed')):
                roles = set(list_from_csv(env.get('HTTP_X_ROLES', '')))

            if not roles.intersection(self.allowed_meta_write_roles):
                for k, v in req.headers.iteritems():
                    if k.startswith('X-Container-Meta-Allowed-Iprange') \
                            or k.startswith('X-Account-Meta-Allowed-Iprange'):
                        return Response(status=403,
                                        body=deny_meta_change,
                                        request=req)(env, start_response)

        # Grab the metadata for the account and container
        if container is not None and container != "":
            try:
                container_info = \
                    get_container_info(req.environ, self.app,
                                       swift_source='IPRangeACLMiddleware')
            except ValueError:
                # if we can't get container info, then we deny the request
                return Response(status=403,
                                body="Invalid container (%s)" % container,
                                request=req)(env, start_response)
        else:
            container_info = None

        try:
            acc_info = get_account_info(req.environ,
                                        self.app,
                                        swift_source='IPRangeACLMiddleware')
        except ValueError:
            # if we can't get account info, then we deny the request
            return Response(status=403,
                            body="Invalid account (%s)" % account,
                            request=req)(env, start_response)

        remote_ip = get_remote_client(req)

        allowed = set()
        default = "denied"

        # Read any account-level ACLs
        meta = acc_info['meta']
        for k, v in meta.iteritems():
            if k.startswith("allowed-iprange") and len(v) > 0:
                allowed.add(v)

            # This key is used to set the default access policy in
            # cases where no ACLs are present in the meta-data.
            if k == "ipacl-default":
                default = v

        # If the request is for a container or object, check for any
        # container-level ACLs
        if container_info is not None:
            meta = container_info['meta']
            for k, v in meta.iteritems():
                # Each allowed range must have a unique meta-data key, but
                # the key must begin with 'allowed-iprange-'
                if k.startswith('allowed-iprange-') and len(v) > 0:
                    allowed.add(v)

                # This key is used to set the default access policy in
                # cases where no ACLs are present in the meta-data.

                # NOTE: Container-level default behaviour will override
                # account-level defaults.
                if k == "ipacl-default":
                    default = v

        # XXX Could probably condense this into one tree, but not sure
        # whether Pytricia is OK with mixing IPv4 and IPv6 prefixes.
        self.pyt = pytricia.PyTricia(32)
        self.pyt6 = pytricia.PyTricia(128)

        # If there are no IP range ACLs in the meta-data and the
        # default policy is "allowed", then we can grant access.
        if len(allowed) == 0 and default == "allowed":
            return self.app(env, start_response)
        else:
            # Build the patricia tree of allowed IP prefixes
            for pref in allowed:

                if ':' in pref:
                    try:
                        addrcheck = ipaddress.IPv6Network(unicode(pref), False)
                    except ipaddress.AddressValueError:
                        self.logger.debug(
                            "iprange_acl -- skipping invalid IP prefix: %(pref)s",
                            {'pref': pref})
                        continue
                    self.pyt6[pref] = "allowed"
                else:
                    try:
                        addrcheck = ipaddress.IPv4Network(unicode(pref), False)
                    except ipaddress.AddressValueError:
                        self.logger.debug(
                            "iprange_acl -- skipping invalid IP prefix: %(pref)s",
                            {'pref': pref})
                        continue

                    self.pyt[pref] = "allowed"

        # Always allow our own IP, otherwise we could lock ourselves out from
        # the container!
        if ':' in self.local_ip:
            self.pyt6[self.local_ip] = "allowed"
        else:
            self.pyt[self.local_ip] = "allowed"

        # Add our default allowed IP ranges to the patricia tree
        for default_range in self.default_ranges:
            if ':' in default_range:
                try:
                    addrcheck = ipaddress.IPv6Network(unicode(default_range), \
                            False)
                except ipaddress.AddressValueError:
                    self.logger.debug("Invalid always_allow prefix for IPv6: %s" \
                            % (default_range))
                else:
                    self.pyt6[default_range] = "allowed"
            else:
                try:
                    addrcheck = ipaddress.IPv4Network(unicode(default_range), \
                            False)
                except ipaddress.AddressValueError:
                    self.logger.debug("Invalid always_allow prefix for IPv4: %s" \
                            % (default_range))
                else:
                    self.pyt[default_range] = "allowed"

        # Look up the address of the client in the patricia tree
        if ':' in remote_ip:
            status = self.pyt6.get(remote_ip)
        else:
            status = self.pyt.get(remote_ip)

        if status == "allowed":
            return self.app(env, start_response)

        return Response(status=403, body=self.deny_message,
                        request=req)(env, start_response)
Ejemplo n.º 52
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            version, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        container_info = get_container_info(
            req.environ, self.app, swift_source='ImageScalerMiddleware')

        # parse query string
        if req.query_string:
            qs = parse_qs(req.query_string)
            if 'size' in qs:
                req_size = qs['size']
            else:
                self.logger.debug("image-scaler: No image scaling requested.")
                return self.app(env, start_response)
        else:
            # nothing for us to do, no scaling requested
            self.logger.debug("image-scaler: No image scaling requested.")
            return self.app(env, start_response)

        # check container whether scaling is allowed
        meta = container_info['meta']
        if not meta.has_key('image-scaling') or \
                meta.has_key('image-scaling') and \
                not meta['image-scaling'].lower() in ['true', '1']:
            # nothing for us to do
            self.logger.debug("image-scaler: Image scaling not "
                              "allowed. Nothing for us to do.")
            return self.app(env, start_response)

        # default allowed extensions
        allowed_exts = self.conf.get('formats', 'jpg;png;gif')
        allowed_exts = allowed_exts.lower()
        allowed_exts = allowed_exts.split(';')
        # check whether file has the allowed ending
        if meta.has_key('image-scaling-extensions'):
            allowed_exts = meta['image-scaling-extensions'].split(',')

        requested_ext = req.path.rsplit('.', 1)[-1]
        if not requested_ext.lower() in map(lambda x: x.lower(), allowed_exts):
            self.logger.info("image-scaler: extension %s not allowed"
                             " for image scaling" % requested_ext)
            return self.app(env, start_response)

        # get maxsize from config, otherwise 20 MB
        max_size = self.conf.get('maxsize', '20971520')
        try:
            max_size = int(max_size)
        except ValueError:
            max_size = 20971520
            self.logger.error(
                "wrong format for max_size from configuration file, using 20 MB"
            )

        if meta.has_key('image-scaling-max-size'):
            max_size = int(meta['image-scaling-max-size'])
        obj_info = get_object_info(req.environ,
                                   self.app,
                                   swift_source="ImageScalerMiddleware")
        if int(obj_info['length']) > max_size:
            self.logger.info("image-scaler: object too large")
            return self.app(env, start_response)

        response = ImageScalerResponse(start_response, req_size, self.logger)
        app_iter = self.app(env, response.scaler_start_response)

        if app_iter is not None:
            response.finish_response(app_iter)

        return response.write()
Ejemplo n.º 53
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            # account and container only
            version, acct, cont = req.split_path(2, 3)
        except ValueError:
            return self.app(env, start_response)

        if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
            return self.app(env, start_response)

        # OK, definitely have an account/container request.
        # Get the desired content-type, then force it to a JSON request.
        try:
            out_content_type = get_listing_content_type(req)
        except HTTPException as err:
            return err(env, start_response)

        params = req.params
        params['format'] = 'json'
        req.params = params

        status, headers, resp_iter = req.call_application(self.app)

        header_to_index = {}
        resp_content_type = resp_length = None
        for i, (header, value) in enumerate(headers):
            header = header.lower()
            if header == 'content-type':
                header_to_index[header] = i
                resp_content_type = value.partition(';')[0]
            elif header == 'content-length':
                header_to_index[header] = i
                resp_length = int(value)

        if not status.startswith('200 '):
            start_response(status, headers)
            return resp_iter

        if resp_content_type != 'application/json':
            start_response(status, headers)
            return resp_iter

        if resp_length is None or \
                resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
            start_response(status, headers)
            return resp_iter

        def set_header(header, value):
            if value is None:
                del headers[header_to_index[header]]
            else:
                headers[header_to_index[header]] = (
                    headers[header_to_index[header]][0], str(value))

        if req.method == 'HEAD':
            set_header('content-type', out_content_type + '; charset=utf-8')
            set_header('content-length', None)  # don't know, can't determine
            start_response(status, headers)
            return resp_iter

        body = b''.join(resp_iter)
        try:
            listing = json.loads(body)
            # Do a couple sanity checks
            if not isinstance(listing, list):
                raise ValueError
            if not all(isinstance(item, dict) for item in listing):
                raise ValueError
        except ValueError:
            # Static web listing that's returning invalid JSON?
            # Just pass it straight through; that's about all we *can* do.
            start_response(status, headers)
            return [body]

        try:
            if out_content_type.endswith('/xml'):
                if cont:
                    body = container_to_xml(listing, cont)
                else:
                    body = account_to_xml(listing, acct)
            elif out_content_type == 'text/plain':
                body = listing_to_text(listing)
            # else, json -- we continue down here to be sure we set charset
        except KeyError:
            # listing was in a bad format -- funky static web listing??
            start_response(status, headers)
            return [body]

        if not body:
            status = '%s %s' % (HTTP_NO_CONTENT,
                                RESPONSE_REASONS[HTTP_NO_CONTENT][0])

        set_header('content-type', out_content_type + '; charset=utf-8')
        set_header('content-length', len(body))
        start_response(status, headers)
        return [body]