Exemple #1
0
    def handle_request(self, req, start_response):
        """
        Take a GET or HEAD request, and if it is for a dynamic large object
        manifest, return an appropriate response.

        Otherwise, simply pass it through.
        """
        update_ignore_range_header(req, 'X-Object-Manifest')
        resp_iter = self._app_call(req.environ)

        # make sure this response is for a dynamic large object manifest
        for header, value in self._response_headers:
            if (header.lower() == 'x-object-manifest'):
                content_length = self._response_header_value('content-length')
                if content_length is not None and int(content_length) < 1024:
                    # Go ahead and consume small bodies
                    drain_and_close(resp_iter)
                close_if_possible(resp_iter)
                response = self.get_or_head_response(
                    req, wsgi_to_str(wsgi_unquote(value)))
                return response(req.environ, start_response)
        # Not a dynamic large object manifest; just pass it through.
        start_response(self._response_status,
                       self._response_headers,
                       self._response_exc_info)
        return resp_iter
    def handle_delete(self, req, start_response):
        """
        Handle request to delete a user's container.

        As part of deleting a container, this middleware will also delete
        the hidden container holding object versions.

        Before a user's container can be deleted, swift must check
        if there are still old object versions from that container.
        Only after disabling versioning and deleting *all* object versions
        can a container be deleted.
        """
        container_info = get_container_info(req.environ,
                                            self.app,
                                            swift_source='OV')

        versions_cont = unquote(
            container_info.get('sysmeta', {}).get('versions-container', ''))

        if versions_cont:
            account = req.split_path(3, 3, True)[1]
            # using a HEAD request here as opposed to get_container_info
            # to make sure we get an up-to-date value
            versions_req = make_pre_authed_request(
                req.environ,
                method='HEAD',
                swift_source='OV',
                path=wsgi_quote('/v1/%s/%s' %
                                (account, str_to_wsgi(versions_cont))),
                headers={'X-Backend-Allow-Reserved-Names': 'true'})
            vresp = versions_req.get_response(self.app)
            drain_and_close(vresp)
            if vresp.is_success and int(
                    vresp.headers.get('X-Container-Object-Count', 0)) > 0:
                raise HTTPConflict(
                    'Delete all versions before deleting container.',
                    request=req)
            elif not vresp.is_success and vresp.status_int != 404:
                raise HTTPInternalServerError(
                    'Error deleting versioned container')
            else:
                versions_req.method = 'DELETE'
                resp = versions_req.get_response(self.app)
                drain_and_close(resp)
                if not is_success(resp.status_int) and resp.status_int != 404:
                    raise HTTPInternalServerError(
                        'Error deleting versioned container')

        app_resp = self._app_call(req.environ)

        start_response(self._response_status, self._response_headers,
                       self._response_exc_info)
        return app_resp
Exemple #3
0
    def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
                                          etag):
        if req.environ.get('swift.symlink_override'):
            req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
            req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \
                req.headers[TGT_BYTES_SYMLINK_HDR]
            return

        # next we'll make sure the E-Tag matches a real object
        new_req = make_subrequest(req.environ,
                                  path=wsgi_quote(symlink_target_path),
                                  method='HEAD',
                                  swift_source='SYM')
        if req.allow_reserved_names:
            new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
        self._last_target_path = symlink_target_path
        resp = self._recursive_get_head(new_req,
                                        target_etag=etag,
                                        follow_softlinks=False)
        if self._get_status_int() == HTTP_NOT_FOUND:
            raise HTTPConflict(body='X-Symlink-Target does not exist',
                               request=req,
                               headers={
                                   'Content-Type': 'text/plain',
                                   'Content-Location': self._last_target_path
                               })
        if not is_success(self._get_status_int()):
            drain_and_close(resp)
            raise status_map[self._get_status_int()](request=req)
        response_headers = HeaderKeyDict(self._response_headers)
        # carry forward any etag update params (e.g. "slo_etag"), we'll append
        # symlink_target_* params to this header after this method returns
        override_header = get_container_update_override_key('etag')
        if override_header in response_headers and \
                override_header not in req.headers:
            sep, params = response_headers[override_header].partition(';')[1:]
            req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params

        # It's troublesome that there's so much leakage with SLO
        if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \
                override_header not in req.headers:
            req.headers[override_header] = '%s; slo_etag=%s' % (
                MD5_OF_EMPTY_STRING,
                response_headers['X-Object-Sysmeta-Slo-Etag'])
        req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = (
            response_headers.get('x-object-sysmeta-slo-size')
            or response_headers['Content-Length'])

        req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag

        if not req.headers.get('Content-Type'):
            req.headers['Content-Type'] = response_headers['Content-Type']
Exemple #4
0
 def _check_response_error(self, req, resp):
     """
     Raise Error Response in case of error
     """
     if is_success(resp.status_int):
         return
     # any error should be short
     drain_and_close(resp)
     if is_client_error(resp.status_int):
         # missing container or bad permissions
         raise HTTPPreconditionFailed(request=req)
     # could not version the data, bail
     raise HTTPServiceUnavailable(request=req)
Exemple #5
0
    def handle_obj_versions_delete_push(self, req, versions_cont, api_version,
                                        account_name, container_name,
                                        object_name):
        """
        Handle DELETE requests when in history mode.

        Copy current version of object to versions_container and write a
        delete marker before proceeding with original request.

        :param req: original request.
        :param versions_cont: container where previous versions of the object
                              are stored.
        :param api_version: api version.
        :param account_name: account name.
        :param object_name: name of object of original request
        """
        self._copy_current(req, versions_cont, api_version, account_name,
                           object_name)

        marker_path = "/%s/%s/%s/%s" % (
            api_version, account_name, versions_cont,
            self._build_versions_object_name(object_name, time.time()))
        marker_headers = {
            # Definitive source of truth is Content-Type, and since we add
            # a swift_* param, we know users haven't set it themselves.
            # This is still open to users POSTing to update the content-type
            # but they're just shooting themselves in the foot then.
            'content-type': DELETE_MARKER_CONTENT_TYPE,
            'content-length': '0',
            'x-auth-token': req.headers.get('x-auth-token')
        }
        marker_req = make_pre_authed_request(req.environ,
                                             path=wsgi_quote(marker_path),
                                             headers=marker_headers,
                                             method='PUT',
                                             swift_source='VW')
        marker_req.environ['swift.content_type_overridden'] = True
        marker_resp = marker_req.get_response(self.app)
        self._check_response_error(req, marker_resp)
        drain_and_close(marker_resp)

        # successfully copied and created delete marker; safe to delete
        return self.app
Exemple #6
0
    def _copy_current(self, req, versions_cont, api_version, account_name,
                      object_name):
        # validate the write access to the versioned container before
        # making any backend requests
        if 'swift.authorize' in req.environ:
            container_info = get_container_info(req.environ,
                                                self.app,
                                                swift_source='VW')
            req.acl = container_info.get('write_acl')
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                raise aresp

        get_resp = self._get_source_object(req, req.path_info)

        if get_resp.status_int == HTTP_NOT_FOUND:
            # nothing to version, proceed with original request
            drain_and_close(get_resp)
            return

        # check for any other errors
        self._check_response_error(req, get_resp)

        # if there's an existing object, then copy it to
        # X-Versions-Location
        ts_source = get_resp.headers.get(
            'x-timestamp',
            calendar.timegm(
                time.strptime(get_resp.headers['last-modified'],
                              '%a, %d %b %Y %H:%M:%S GMT')))
        vers_obj_name = self._build_versions_object_name(
            object_name, ts_source)

        put_path_info = "/%s/%s/%s/%s" % (api_version, account_name,
                                          versions_cont, vers_obj_name)
        req.environ['QUERY_STRING'] = ''
        put_resp = self._put_versioned_obj(req, put_path_info, get_resp)

        self._check_response_error(req, put_resp)
        # successful PUT response should be short
        drain_and_close(put_resp)
Exemple #7
0
    def _restore_data(self, req, versions_cont, api_version, account_name,
                      container_name, object_name, prev_obj_name):
        get_path = "/%s/%s/%s/%s" % (api_version, account_name, versions_cont,
                                     prev_obj_name)

        get_resp = self._get_source_object(req, get_path)

        # if the version isn't there, keep trying with previous version
        if get_resp.status_int == HTTP_NOT_FOUND:
            drain_and_close(get_resp)
            return False

        self._check_response_error(req, get_resp)

        put_path_info = "/%s/%s/%s/%s" % (api_version, account_name,
                                          container_name, object_name)
        put_resp = self._put_versioned_obj(req, put_path_info, get_resp)

        self._check_response_error(req, put_resp)
        drain_and_close(put_resp)
        return get_path
Exemple #8
0
    def handle_obj_versions_delete_pop(self, req, versions_cont, api_version,
                                       account_name, container_name,
                                       object_name):
        """
        Handle DELETE requests when in stack mode.

        Delete current version of object and pop previous version in its place.

        :param req: original request.
        :param versions_cont: container where previous versions of the object
                              are stored.
        :param api_version: api version.
        :param account_name: account name.
        :param container_name: container name.
        :param object_name: object name.
        """
        listing_prefix = self._build_versions_object_prefix(object_name)
        item_iter = self._listing_iter(account_name, versions_cont,
                                       listing_prefix, req)

        auth_token_header = {'X-Auth-Token': req.headers.get('X-Auth-Token')}
        authed = False
        for previous_version in item_iter:
            if not authed:
                # validate the write access to the versioned container before
                # making any backend requests
                if 'swift.authorize' in req.environ:
                    container_info = get_container_info(req.environ,
                                                        self.app,
                                                        swift_source='VW')
                    req.acl = container_info.get('write_acl')
                    aresp = req.environ['swift.authorize'](req)
                    if aresp:
                        return aresp
                    authed = True

            if previous_version['content_type'] == DELETE_MARKER_CONTENT_TYPE:
                # check whether we have data in the versioned container
                obj_head_headers = {'X-Newest': 'True'}
                obj_head_headers.update(auth_token_header)
                head_req = make_pre_authed_request(req.environ,
                                                   path=wsgi_quote(
                                                       req.path_info),
                                                   method='HEAD',
                                                   headers=obj_head_headers,
                                                   swift_source='VW')
                hresp = head_req.get_response(self.app)
                drain_and_close(hresp)

                if hresp.status_int != HTTP_NOT_FOUND:
                    self._check_response_error(req, hresp)
                    # if there's an existing object, then just let the delete
                    # through (i.e., restore to the delete-marker state):
                    break

                # no data currently in the container (delete marker is current)
                for version_to_restore in item_iter:
                    if version_to_restore['content_type'] == \
                            DELETE_MARKER_CONTENT_TYPE:
                        # Nothing to restore
                        break
                    obj_to_restore = bytes_to_wsgi(
                        version_to_restore['name'].encode('utf-8'))
                    req.environ['QUERY_STRING'] = ''
                    restored_path = self._restore_data(
                        req, versions_cont, api_version, account_name,
                        container_name, object_name, obj_to_restore)
                    if not restored_path:
                        continue

                    old_del_req = make_pre_authed_request(
                        req.environ,
                        path=wsgi_quote(restored_path),
                        method='DELETE',
                        headers=auth_token_header,
                        swift_source='VW')
                    del_resp = old_del_req.get_response(self.app)
                    drain_and_close(del_resp)
                    if del_resp.status_int != HTTP_NOT_FOUND:
                        self._check_response_error(req, del_resp)
                        # else, well, it existed long enough to do the
                        # copy; we won't worry too much
                    break
                prev_obj_name = bytes_to_wsgi(
                    previous_version['name'].encode('utf-8'))
                marker_path = "/%s/%s/%s/%s" % (api_version, account_name,
                                                versions_cont, prev_obj_name)
                # done restoring, redirect the delete to the marker
                req = make_pre_authed_request(req.environ,
                                              path=wsgi_quote(marker_path),
                                              method='DELETE',
                                              headers=auth_token_header,
                                              swift_source='VW')
            else:
                # there are older versions so copy the previous version to the
                # current object and delete the previous version
                prev_obj_name = bytes_to_wsgi(
                    previous_version['name'].encode('utf-8'))
                req.environ['QUERY_STRING'] = ''
                restored_path = self._restore_data(req, versions_cont,
                                                   api_version, account_name,
                                                   container_name, object_name,
                                                   prev_obj_name)
                if not restored_path:
                    continue

                # redirect the original DELETE to the source of the reinstated
                # version object - we already auth'd original req so make a
                # pre-authed request
                req = make_pre_authed_request(req.environ,
                                              path=wsgi_quote(restored_path),
                                              method='DELETE',
                                              headers=auth_token_header,
                                              swift_source='VW')

            # remove 'X-If-Delete-At', since it is not for the older copy
            if 'X-If-Delete-At' in req.headers:
                del req.headers['X-If-Delete-At']
            break

        # handle DELETE request here in case it was modified
        return req.get_response(self.app)
Exemple #9
0
    def _listing_pages_iter(self,
                            account_name,
                            lcontainer,
                            lprefix,
                            req,
                            marker='',
                            end_marker='',
                            reverse=True):
        '''Get "pages" worth of objects that start with a prefix.

        The optional keyword arguments ``marker``, ``end_marker``, and
        ``reverse`` are used similar to how they are for containers. We're
        either coming:

           - directly from ``_listing_iter``, in which case none of the
             optional args are specified, or

           - from ``_in_proxy_reverse_listing``, in which case ``reverse``
             is ``False`` and both ``marker`` and ``end_marker`` are specified
             (although they may still be blank).
        '''
        while True:
            lreq = make_pre_authed_request(
                req.environ,
                method='GET',
                swift_source='VW',
                path=wsgi_quote('/v1/%s/%s' % (account_name, lcontainer)))
            lreq.environ['QUERY_STRING'] = \
                'prefix=%s&marker=%s' % (wsgi_quote(lprefix),
                                         wsgi_quote(marker))
            if end_marker:
                lreq.environ['QUERY_STRING'] += '&end_marker=%s' % (
                    wsgi_quote(end_marker))
            if reverse:
                lreq.environ['QUERY_STRING'] += '&reverse=on'
            lresp = lreq.get_response(self.app)
            if not is_success(lresp.status_int):
                # errors should be short
                drain_and_close(lresp)
                if lresp.status_int == HTTP_NOT_FOUND:
                    raise ListingIterNotFound()
                elif is_client_error(lresp.status_int):
                    raise HTTPPreconditionFailed(request=req)
                else:
                    raise ListingIterError()

            if not lresp.body:
                break

            sublisting = json.loads(lresp.body)
            if not sublisting:
                break

            # When using the ``reverse`` param, check that the listing is
            # actually reversed
            first_item = bytes_to_wsgi(sublisting[0]['name'].encode('utf-8'))
            last_item = bytes_to_wsgi(sublisting[-1]['name'].encode('utf-8'))
            page_is_after_marker = marker and first_item > marker
            if reverse and (first_item < last_item or page_is_after_marker):
                # Apparently there's at least one pre-2.6.0 container server
                yield self._in_proxy_reverse_listing(account_name, lcontainer,
                                                     lprefix, req, marker,
                                                     sublisting)
                return

            marker = last_item
            yield sublisting
    def enable_versioning(self, req, start_response):
        container_info = get_container_info(req.environ,
                                            self.app,
                                            swift_source='OV')

        # if container is already configured to use old style versioning,
        # we don't allow user to enable object versioning here. They must
        # choose which middleware to use, only one style of versioning
        # is supported for a given container
        versions_cont = container_info.get('sysmeta',
                                           {}).get('versions-location')
        legacy_versions_cont = container_info.get('versions')
        if versions_cont or legacy_versions_cont:
            raise HTTPBadRequest(
                'Cannot enable object versioning on a container '
                'that is already using the legacy versioned writes '
                'feature.',
                request=req)

        # versioning and container-sync do not yet work well together
        # container-sync needs to be enhanced to sync previous versions
        sync_to = container_info.get('sync_to')
        if sync_to:
            raise HTTPBadRequest(
                'Cannot enable object versioning on a container '
                'configured as source of container syncing.',
                request=req)

        versions_cont = container_info.get('sysmeta',
                                           {}).get('versions-container')
        is_enabled = config_true_value(req.headers[CLIENT_VERSIONS_ENABLED])

        req.headers[SYSMETA_VERSIONS_ENABLED] = is_enabled

        # TODO: a POST request to a primary container that doesn't exist
        # will fail, so we will create and delete the versions container
        # for no reason
        if config_true_value(is_enabled):
            (version, account, container, _) = req.split_path(3, 4, True)

            # Attempt to use same policy as primary container, otherwise
            # use default policy
            if is_success(container_info['status']):
                primary_policy_idx = container_info['storage_policy']
                if POLICIES[primary_policy_idx].is_deprecated:
                    # Do an auth check now, so we don't leak information
                    # about the container
                    aresp = req.environ['swift.authorize'](req)
                    if aresp:
                        raise aresp

                    # Proxy controller would catch the deprecated policy, too,
                    # but waiting until then would mean the error message
                    # would be a generic "Error enabling object versioning".
                    raise HTTPBadRequest(
                        'Cannot enable object versioning on a container '
                        'that uses a deprecated storage policy.',
                        request=req)
                hdrs = {'X-Storage-Policy': POLICIES[primary_policy_idx].name}
            else:
                if req.method == 'PUT' and \
                        'X-Storage-Policy' in req.headers:
                    hdrs = {
                        'X-Storage-Policy': req.headers['X-Storage-Policy']
                    }
                else:
                    hdrs = {}
            hdrs['X-Backend-Allow-Reserved-Names'] = 'true'

            versions_cont = self._build_versions_container_name(container)
            versions_cont_path = "/%s/%s/%s" % (version, account,
                                                versions_cont)
            ver_cont_req = make_pre_authed_request(
                req.environ,
                path=wsgi_quote(versions_cont_path),
                method='PUT',
                headers=hdrs,
                swift_source='OV')
            resp = ver_cont_req.get_response(self.app)
            # Should always be short; consume the body
            drain_and_close(resp)
            if is_success(resp.status_int) or resp.status_int == HTTP_CONFLICT:
                req.headers[SYSMETA_VERSIONS_CONT] = wsgi_quote(versions_cont)
            else:
                raise HTTPInternalServerError(
                    'Error enabling object versioning')

        # make original request
        app_resp = self._app_call(req.environ)

        # if we just created a versions container but the original
        # request failed, delete the versions container
        # and let user retry later
        if not is_success(self._get_status_int()) and \
                SYSMETA_VERSIONS_CONT in req.headers:
            versions_cont_path = "/%s/%s/%s" % (version, account,
                                                versions_cont)
            ver_cont_req = make_pre_authed_request(
                req.environ,
                path=wsgi_quote(versions_cont_path),
                method='DELETE',
                headers=hdrs,
                swift_source='OV')

            # TODO: what if this one fails??
            resp = ver_cont_req.get_response(self.app)
            drain_and_close(resp)

        if self._response_headers is None:
            self._response_headers = []
        for key, val in self._response_headers:
            if key.lower() == SYSMETA_VERSIONS_ENABLED:
                self._response_headers.extend([
                    (CLIENT_VERSIONS_ENABLED.title(), val)
                ])

        start_response(self._response_status, self._response_headers,
                       self._response_exc_info)
        return app_resp
    def handle_request(self, req, start_response):
        """
        Handle request for container resource.

        On PUT, POST set version location and enabled flag sysmeta.
        For container listings of a versioned container, update the object's
        bytes and etag to use the target's instead of using the symlink info.
        """
        # FIXME(FVE): this request is not needed when handling
        # list-object-versions requests.
        app_resp = self._app_call(req.environ)
        _, account, container, _ = req.split_path(3, 4, True)
        location = ''
        curr_bytes = 0
        bytes_idx = -1
        for i, (header, value) in enumerate(self._response_headers):
            if header == 'X-Container-Bytes-Used':
                curr_bytes = value
                bytes_idx = i
            if header.lower() == SYSMETA_VERSIONS_CONT:
                location = value
            if header.lower() == SYSMETA_VERSIONS_ENABLED:
                self._response_headers.extend([
                    (CLIENT_VERSIONS_ENABLED.title(), value)
                ])

        if location:
            location = wsgi_unquote(location)

            # update bytes header
            if bytes_idx > -1:
                head_req = make_pre_authed_request(
                    req.environ,
                    method='HEAD',
                    swift_source='OV',
                    path=wsgi_quote('/v1/%s/%s' % (account, location)),
                    headers={'X-Backend-Allow-Reserved-Names': 'true'})
                vresp = head_req.get_response(self.app)
                if vresp.is_success:
                    ver_bytes = vresp.headers.get('X-Container-Bytes-Used', 0)
                    self._response_headers[bytes_idx] = (
                        'X-Container-Bytes-Used',
                        str(int(curr_bytes) + int(ver_bytes)))
                drain_and_close(vresp)
        elif is_success(self._get_status_int()):
            # If client is doing a version-aware listing for a container that
            # (as best we could tell) has never had versioning enabled,
            # err on the side of there being data anyway -- the metadata we
            # found may not be the most up-to-date.

            # Note that any extra listing request we make will likely 404.
            try:
                location = self._build_versions_container_name(container)
            except ValueError:
                # may be internal listing to a reserved namespace container
                pass
        # else, we won't need location anyway

        if req.method == 'GET' and 'versions' in req.params:
            return self._list_versions(req, start_response, container)

        start_response(self._response_status, self._response_headers,
                       self._response_exc_info)
        return app_resp
    def handle_versioned_request(self, req, versions_cont, api_version,
                                 account, container, obj, is_enabled, version):
        """
        Handle 'version-id' request for object resource. When a request
        contains a ``version-id=<id>`` parameter, the request is acted upon
        the actual version of that object. Version-aware operations
        require that the container is versioned, but do not require that
        the versioning is currently enabled. Users should be able to
        operate on older versions of an object even if versioning is
        currently suspended.

        PUT and POST requests are not allowed as that would overwrite
        the contents of the versioned object.

        :param req: The original request
        :param versions_cont: container holding versions of the requested obj
        :param api_version: should be v1 unless swift bumps api version
        :param account: account name string
        :param container: container name string
        :param object: object name string
        :param is_enabled: is versioning currently enabled
        :param version: version of the object to act on
        """
        if not versions_cont and version != 'null':
            raise HTTPBadRequest(
                'version-aware operations require that the container is '
                'versioned',
                request=req)
        req.environ['oio.query'] = {'version': version}
        if version != 'null':
            try:
                int(version)
            except ValueError:
                raise HTTPBadRequest('Invalid version parameter', request=req)

        if req.method == 'DELETE':
            return self.handle_delete_version(req, versions_cont, api_version,
                                              account, container, obj,
                                              is_enabled, version)
        elif req.method == 'PUT':
            return self.handle_put_version(req, versions_cont, api_version,
                                           account, container, obj, is_enabled,
                                           version)
        if version == 'null':
            resp = req.get_response(self.app)
            if resp.is_success:
                if get_reserved_name('versions', '') in wsgi_unquote(
                        resp.headers.get('Content-Location', '')):
                    # Have a latest version, but it's got a real version-id.
                    # Since the user specifically asked for null, return 404
                    close_if_possible(resp.app_iter)
                    raise HTTPNotFound(request=req)
                resp.headers['X-Object-Version-Id'] = 'null'
                if req.method == 'HEAD':
                    drain_and_close(resp)
            return resp
        else:
            resp = req.get_response(self.app)
            if resp.is_success:
                resp.headers['X-Object-Version-Id'] = version

            # Well, except for some delete marker business...
            is_del_marker = DELETE_MARKER_CONTENT_TYPE == resp.headers.get(
                'X-Backend-Content-Type', resp.headers['Content-Type'])

            if req.method == 'HEAD':
                drain_and_close(resp)

            if is_del_marker:
                hdrs = {
                    'X-Object-Version-Id': version,
                    'Content-Type': DELETE_MARKER_CONTENT_TYPE
                }
                raise HTTPNotFound(request=req, headers=hdrs)
            return resp
Exemple #13
0
 def handle_request(self, *args, **kwargs):
     resp = self.make_request(*args, **kwargs)
     # Drain the response body to prevent unexpected disconnect
     # in proxy-server
     drain_and_close(resp)
Exemple #14
0
    def make_request(self,
                     method,
                     path,
                     headers,
                     acceptable_statuses,
                     body_file=None,
                     params=None):
        """Makes a request to Swift with retries.

        :param method: HTTP method of request.
        :param path: Path of request.
        :param headers: Headers to be sent with request.
        :param acceptable_statuses: List of acceptable statuses for request.
        :param body_file: Body file to be passed along with request,
                          defaults to None.
        :param params: A dict of params to be set in request query string,
                       defaults to None.

        :returns: Response object on success.

        :raises UnexpectedResponse: Exception raised when make_request() fails
                                    to get a response with an acceptable status
        :raises Exception: Exception is raised when code fails in an
                           unexpected way.
        """

        headers = dict(headers)
        headers['user-agent'] = self.user_agent
        headers.setdefault('x-backend-allow-reserved-names', 'true')
        if self.use_replication_network:
            headers.setdefault(USE_REPLICATION_NETWORK_HEADER, 'true')

        for attempt in range(self.request_tries):
            resp = exc_type = exc_value = exc_traceback = None
            req = Request.blank(path,
                                environ={'REQUEST_METHOD': method},
                                headers=headers)
            if body_file is not None:
                if hasattr(body_file, 'seek'):
                    body_file.seek(0)
                req.body_file = body_file
            if params:
                req.params = params
            try:
                resp = req.get_response(self.app)
            except (Exception, Timeout):
                exc_type, exc_value, exc_traceback = exc_info()
            else:
                if resp.status_int in acceptable_statuses or \
                        resp.status_int // 100 in acceptable_statuses:
                    return resp
                elif not is_server_error(resp.status_int):
                    # No sense retrying when we expect the same result
                    break
            # sleep only between tries, not after each one
            if attempt < self.request_tries - 1:
                if resp:
                    # for non 2XX requests it's safe and useful to drain
                    # the response body so we log the correct status code
                    if resp.status_int // 100 != 2:
                        drain_and_close(resp)
                    else:
                        # Just close; the 499 is appropriate
                        close_if_possible(resp.app_iter)
                sleep(2**(attempt + 1))
        if resp:
            msg = 'Unexpected response: %s' % resp.status
            if resp.status_int // 100 != 2 and resp.body:
                # provide additional context (and drain the response body) for
                # non 2XX responses
                msg += ' (%s)' % resp.body
            raise UnexpectedResponse(msg, resp)
        if exc_type:
            # To make pep8 tool happy, in place of raise t, v, tb:
            six.reraise(exc_type, exc_value, exc_traceback)
Exemple #15
0
    def _recursive_get_head(self,
                            req,
                            target_etag=None,
                            follow_softlinks=True,
                            orig_req=None):
        if not orig_req:
            orig_req = req
        resp = self._app_call(req.environ)

        def build_traversal_req(symlink_target):
            """
            :returns: new request for target path if it's symlink otherwise
                      None
            """
            version, account, _junk = req.split_path(2, 3, True)
            account = self._response_header_value(
                TGT_ACCT_SYSMETA_SYMLINK_HDR) or wsgi_quote(account)
            target_path = os.path.join('/', version, account,
                                       symlink_target.lstrip('/'))
            self._last_target_path = target_path

            subreq_headers = dict(req.headers)
            if self._response_header_value(ALLOW_RESERVED_NAMES):
                # this symlink's sysmeta says it can point to reserved names,
                # we're infering that some piece of middleware had previously
                # authorized this request because users can't access reserved
                # names directly
                subreq_meth = make_pre_authed_request
                subreq_headers['X-Backend-Allow-Reserved-Names'] = 'true'
            else:
                subreq_meth = make_subrequest
            new_req = subreq_meth(orig_req.environ,
                                  path=target_path,
                                  method=req.method,
                                  headers=subreq_headers,
                                  swift_source='SYM')
            new_req.headers.pop('X-Backend-Storage-Policy-Index', None)
            return new_req

        symlink_target = self._response_header_value(
            TGT_OBJ_SYSMETA_SYMLINK_HDR)
        resp_etag = self._response_header_value(TGT_ETAG_SYSMETA_SYMLINK_HDR)
        if symlink_target and (resp_etag or follow_softlinks):
            # Should be a zero-byte object
            drain_and_close(resp)
            found_etag = resp_etag or self._response_header_value('etag')
            if target_etag and target_etag != found_etag:
                raise HTTPConflict(
                    body='X-Symlink-Target-Etag headers do not match',
                    headers={
                        'Content-Type': 'text/plain',
                        'Content-Location': self._last_target_path
                    })
            if self._loop_count >= self.symloop_max:
                raise LinkIterError()
            # format: /<account name>/<container name>/<object name>
            new_req = build_traversal_req(symlink_target)
            if not config_true_value(
                    self._response_header_value(SYMLOOP_EXTEND)):
                self._loop_count += 1
            return self._recursive_get_head(new_req,
                                            target_etag=resp_etag,
                                            orig_req=req)
        else:
            final_etag = self._response_header_value('etag')
            if final_etag and target_etag and target_etag != final_etag:
                # do *not* drain; we don't know how big this is
                close_if_possible(resp)
                body = ('Object Etag %r does not match '
                        'X-Symlink-Target-Etag header %r')
                raise HTTPConflict(body=body % (final_etag, target_etag),
                                   headers={
                                       'Content-Type': 'text/plain',
                                       'Content-Location':
                                       self._last_target_path
                                   })

            if self._last_target_path:
                # Content-Location will be applied only when one or more
                # symlink recursion occurred.
                # In this case, Content-Location is applied to show which
                # object path caused the error response.
                # To preserve '%2F'(= quote('/')) in X-Symlink-Target
                # header value as it is, Content-Location value comes from
                # TGT_OBJ_SYMLINK_HDR, not req.path
                self._response_headers.extend([('Content-Location',
                                                self._last_target_path)])

            return resp