Example #1
0
    def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
        """
        Fetch the submanifest, parse it, and return it.
        Raise exception on failures.
        """
        sub_req = make_request(
            req.environ, path='/'.join(['', version, acc, con, obj]),
            method='GET',
            headers={'x-auth-token': req.headers.get('x-auth-token')},
            agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO')
        sub_resp = sub_req.get_response(self.slo.app)

        if not is_success(sub_resp.status_int):
            raise ListingIterError(
                'ERROR: while fetching %s, GET of submanifest %s '
                'failed with status %d' % (req.path, sub_req.path,
                                           sub_resp.status_int))

        try:
            with closing_if_possible(sub_resp.app_iter):
                return json.loads(''.join(sub_resp.app_iter))
        except ValueError as err:
            raise ListingIterError(
                'ERROR: while fetching %s, JSON-decoding of submanifest %s '
                'failed with %s' % (req.path, sub_req.path, err))
Example #2
0
    def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
        """
        Fetch the submanifest, parse it, and return it.
        Raise exception on failures.
        """
        sub_req = make_request(
            req.environ,
            path='/'.join(['', version, acc, con, obj]),
            method='GET',
            headers={'x-auth-token': req.headers.get('x-auth-token')},
            agent=('%(orig)s ' + 'SLO MultipartGET'),
            swift_source='SLO')
        sub_resp = sub_req.get_response(self.slo.app)

        if not is_success(sub_resp.status_int):
            raise ListingIterError(
                'ERROR: while fetching %s, GET of submanifest %s '
                'failed with status %d' %
                (req.path, sub_req.path, sub_resp.status_int))

        try:
            with closing_if_possible(sub_resp.app_iter):
                return json.loads(''.join(sub_resp.app_iter))
        except ValueError as err:
            raise ListingIterError(
                'ERROR: while fetching %s, JSON-decoding of submanifest %s '
                'failed with %s' % (req.path, sub_req.path, err))
Example #3
0
    def handle_slo_get_or_head(self, req, start_response):
        """
        Takes a request and a start_response callable and does the normal WSGI
        thing with them. Returns an iterator suitable for sending up the WSGI
        chain.

        :param req: swob.Request object; is a GET or HEAD request aimed at
                    what may be a static large object manifest (or may not).
        :param start_response: WSGI start_response callable
        """
        resp_iter = self._app_call(req.environ)

        # make sure this response is for a static large object manifest
        for header, value in self._response_headers:
            if (header.lower() == 'x-static-large-object' and
                    config_true_value(value)):
                break
        else:
            # Not a static large object manifest. Just pass it through.
            start_response(self._response_status,
                           self._response_headers,
                           self._response_exc_info)
            return resp_iter

        # Handle pass-through request for the manifest itself
        if req.params.get('multipart-manifest') == 'get':
            new_headers = []
            for header, value in self._response_headers:
                if header.lower() == 'content-type':
                    new_headers.append(('Content-Type',
                                        'application/json; charset=utf-8'))
                else:
                    new_headers.append((header, value))
            self._response_headers = new_headers
            start_response(self._response_status,
                           self._response_headers,
                           self._response_exc_info)
            return resp_iter

        if self._need_to_refetch_manifest(req):
            req.environ['swift.non_client_disconnect'] = True
            close_if_possible(resp_iter)
            del req.environ['swift.non_client_disconnect']

            get_req = make_request(
                req.environ, method='GET',
                headers={'x-auth-token': req.headers.get('x-auth-token')},
                agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO')
            resp_iter = self._app_call(get_req.environ)

        # Any Content-Range from a manifest is almost certainly wrong for the
        # full large object.
        resp_headers = [(h, v) for h, v in self._response_headers
                        if not h.lower() == 'content-range']

        response = self.get_or_head_response(
            req, resp_headers, resp_iter)
        return response(req.environ, start_response)
Example #4
0
    def handle_slo_get_or_head(self, req, start_response):
        """
        Takes a request and a start_response callable and does the normal WSGI
        thing with them. Returns an iterator suitable for sending up the WSGI
        chain.

        :param req: swob.Request object; is a GET or HEAD request aimed at
                    what may be a static large object manifest (or may not).
        :param start_response: WSGI start_response callable
        """
        resp_iter = self._app_call(req.environ)

        # make sure this response is for a static large object manifest
        for header, value in self._response_headers:
            if (header.lower() == 'x-static-large-object'
                    and config_true_value(value)):
                break
        else:
            # Not a static large object manifest. Just pass it through.
            start_response(self._response_status, self._response_headers,
                           self._response_exc_info)
            return resp_iter

        # Handle pass-through request for the manifest itself
        if req.params.get('multipart-manifest') == 'get':
            new_headers = []
            for header, value in self._response_headers:
                if header.lower() == 'content-type':
                    new_headers.append(
                        ('Content-Type', 'application/json; charset=utf-8'))
                else:
                    new_headers.append((header, value))
            self._response_headers = new_headers
            start_response(self._response_status, self._response_headers,
                           self._response_exc_info)
            return resp_iter

        if self._need_to_refetch_manifest(req):
            req.environ['swift.non_client_disconnect'] = True
            close_if_possible(resp_iter)
            del req.environ['swift.non_client_disconnect']

            get_req = make_request(
                req.environ,
                method='GET',
                headers={'x-auth-token': req.headers.get('x-auth-token')},
                agent=('%(orig)s ' + 'SLO MultipartGET'),
                swift_source='SLO')
            resp_iter = self._app_call(get_req.environ)

        # Any Content-Range from a manifest is almost certainly wrong for the
        # full large object.
        resp_headers = [(h, v) for h, v in self._response_headers
                        if not h.lower() == 'content-range']

        response = self.get_or_head_response(req, resp_headers, resp_iter)
        return response(req.environ, start_response)
Example #5
0
    def _get_container_listing(self, req, version, account, container,
                               prefix, marker=''):
        con_req = make_request(
            req.environ, path='/'.join(['', version, account, container]),
            method='GET',
            headers={'x-auth-token': req.headers.get('x-auth-token')},
            agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
        con_req.query_string = 'format=json&prefix=%s' % quote(prefix)
        if marker:
            con_req.query_string += '&marker=%s' % quote(marker)

        con_resp = con_req.get_response(self.dlo.app)
        if not is_success(con_resp.status_int):
            return con_resp, None
        return None, json.loads(''.join(con_resp.app_iter))
Example #6
0
    def _get_container_listing(self, req, version, account, container, prefix, marker=""):
        con_req = make_request(
            req.environ,
            path="/".join(["", version, account, container]),
            method="GET",
            headers={"x-auth-token": req.headers.get("x-auth-token")},
            agent=("%(orig)s " + "DLO MultipartGET"),
            swift_source="DLO",
        )
        con_req.query_string = "format=json&prefix=%s" % quote(prefix)
        if marker:
            con_req.query_string += "&marker=%s" % quote(marker)

        con_resp = con_req.get_response(self.dlo.app)
        if not is_success(con_resp.status_int):
            return con_resp, None
        return None, json.loads("".join(con_resp.app_iter))
Example #7
0
    def __iter__(self):
        start_time = time.time()
        have_yielded_data = False

        if self.response and self.response.content_length:
            bytes_left = int(self.response.content_length)
        else:
            bytes_left = None

        try:
            for seg_path, seg_etag, seg_size, first_byte, last_byte in self.listing_iter:
                if time.time() - start_time > self.max_get_time:
                    raise SegmentError(
                        "ERROR: While processing manifest %s, "
                        "max LO GET time of %ds exceeded" % (self.name, self.max_get_time)
                    )
                seg_req = make_request(
                    self.req.environ,
                    path=seg_path,
                    method="GET",
                    headers={"x-auth-token": self.req.headers.get("x-auth-token")},
                    agent=("%(orig)s " + self.ua_suffix),
                    swift_source=self.swift_source,
                )
                if first_byte is not None or last_byte is not None:
                    seg_req.headers["Range"] = "bytes=%s-%s" % (
                        # The 0 is to avoid having a range like "bytes=-10",
                        # which actually means the *last* 10 bytes.
                        "0" if first_byte is None else first_byte,
                        "" if last_byte is None else last_byte,
                    )

                seg_resp = seg_req.get_response(self.app)
                if not is_success(seg_resp.status_int):
                    close_if_possible(seg_resp.app_iter)
                    raise SegmentError(
                        "ERROR: While processing manifest %s, "
                        "got %d while retrieving %s" % (self.name, seg_resp.status_int, seg_path)
                    )

                elif (seg_etag and (seg_resp.etag != seg_etag)) or (
                    seg_size and (seg_resp.content_length != seg_size) and not seg_req.range
                ):
                    # The content-length check is for security reasons. Seems
                    # possible that an attacker could upload a >1mb object and
                    # then replace it with a much smaller object with same
                    # etag. Then create a big nested SLO that calls that
                    # object many times which would hammer our obj servers. If
                    # this is a range request, don't check content-length
                    # because it won't match.
                    close_if_possible(seg_resp.app_iter)
                    raise SegmentError(
                        "Object segment no longer valid: "
                        "%(path)s etag: %(r_etag)s != %(s_etag)s or "
                        "%(r_size)s != %(s_size)s."
                        % {
                            "path": seg_req.path,
                            "r_etag": seg_resp.etag,
                            "r_size": seg_resp.content_length,
                            "s_etag": seg_etag,
                            "s_size": seg_size,
                        }
                    )

                for chunk in seg_resp.app_iter:
                    have_yielded_data = True
                    if bytes_left is None:
                        yield chunk
                    elif bytes_left >= len(chunk):
                        yield chunk
                        bytes_left -= len(chunk)
                    else:
                        yield chunk[:bytes_left]
                        bytes_left -= len(chunk)
                        close_if_possible(seg_resp.app_iter)
                        raise SegmentError(
                            "Too many bytes for %(name)s; truncating in "
                            "%(seg)s with %(left)d bytes left"
                            % {"name": self.name, "seg": seg_req.path, "left": bytes_left}
                        )
                close_if_possible(seg_resp.app_iter)

            if bytes_left:
                raise SegmentError("Not enough bytes for %s; closing connection" % self.name)

        except ListingIterError as err:
            # I have to save this error because yielding the ' ' below clears
            # the exception from the current stack frame.
            excinfo = sys.exc_info()
            self.logger.exception("ERROR: While processing manifest %s, %s", self.name, err)
            # Normally, exceptions before any data has been yielded will
            # cause Eventlet to send a 5xx response. In this particular
            # case of ListingIterError we don't want that and we'd rather
            # just send the normal 2xx response and then hang up early
            # since 5xx codes are often used to judge Service Level
            # Agreements and this ListingIterError indicates the user has
            # created an invalid condition.
            if not have_yielded_data:
                yield " "
            raise excinfo
        except SegmentError as err:
            self.logger.exception(err)
            # This doesn't actually change the response status (we're too
            # late for that), but this does make it to the logs.
            if self.response:
                self.response.status = HTTP_SERVICE_UNAVAILABLE
            raise
Example #8
0
    def __iter__(self):
        start_time = time.time()
        have_yielded_data = False

        if self.response and self.response.content_length:
            bytes_left = int(self.response.content_length)
        else:
            bytes_left = None

        try:
            for seg_path, seg_etag, seg_size, first_byte, last_byte \
                    in self.listing_iter:
                if time.time() - start_time > self.max_get_time:
                    raise SegmentError('ERROR: While processing manifest %s, '
                                       'max LO GET time of %ds exceeded' %
                                       (self.name, self.max_get_time))
                seg_req = make_request(self.req.environ,
                                       path=seg_path,
                                       method='GET',
                                       headers={
                                           'x-auth-token':
                                           self.req.headers.get('x-auth-token')
                                       },
                                       agent=('%(orig)s ' + self.ua_suffix),
                                       swift_source=self.swift_source)
                if first_byte is not None or last_byte is not None:
                    seg_req.headers['Range'] = "bytes=%s-%s" % (
                        # The 0 is to avoid having a range like "bytes=-10",
                        # which actually means the *last* 10 bytes.
                        '0' if first_byte is None else first_byte,
                        '' if last_byte is None else last_byte)

                seg_resp = seg_req.get_response(self.app)
                if not is_success(seg_resp.status_int):
                    close_if_possible(seg_resp.app_iter)
                    raise SegmentError(
                        'ERROR: While processing manifest %s, '
                        'got %d while retrieving %s' %
                        (self.name, seg_resp.status_int, seg_path))

                elif ((seg_etag and (seg_resp.etag != seg_etag))
                      or (seg_size and (seg_resp.content_length != seg_size)
                          and not seg_req.range)):
                    # The content-length check is for security reasons. Seems
                    # possible that an attacker could upload a >1mb object and
                    # then replace it with a much smaller object with same
                    # etag. Then create a big nested SLO that calls that
                    # object many times which would hammer our obj servers. If
                    # this is a range request, don't check content-length
                    # because it won't match.
                    close_if_possible(seg_resp.app_iter)
                    raise SegmentError(
                        'Object segment no longer valid: '
                        '%(path)s etag: %(r_etag)s != %(s_etag)s or '
                        '%(r_size)s != %(s_size)s.' % {
                            'path': seg_req.path,
                            'r_etag': seg_resp.etag,
                            'r_size': seg_resp.content_length,
                            's_etag': seg_etag,
                            's_size': seg_size
                        })

                for chunk in seg_resp.app_iter:
                    have_yielded_data = True
                    if bytes_left is None:
                        yield chunk
                    elif bytes_left >= len(chunk):
                        yield chunk
                        bytes_left -= len(chunk)
                    else:
                        yield chunk[:bytes_left]
                        bytes_left -= len(chunk)
                        close_if_possible(seg_resp.app_iter)
                        raise SegmentError(
                            'Too many bytes for %(name)s; truncating in '
                            '%(seg)s with %(left)d bytes left' % {
                                'name': self.name,
                                'seg': seg_req.path,
                                'left': bytes_left
                            })
                close_if_possible(seg_resp.app_iter)

            if bytes_left:
                raise SegmentError(
                    'Not enough bytes for %s; closing connection' % self.name)

        except ListingIterError as err:
            # I have to save this error because yielding the ' ' below clears
            # the exception from the current stack frame.
            excinfo = sys.exc_info()
            self.logger.exception('ERROR: While processing manifest %s, %s',
                                  self.name, err)
            # Normally, exceptions before any data has been yielded will
            # cause Eventlet to send a 5xx response. In this particular
            # case of ListingIterError we don't want that and we'd rather
            # just send the normal 2xx response and then hang up early
            # since 5xx codes are often used to judge Service Level
            # Agreements and this ListingIterError indicates the user has
            # created an invalid condition.
            if not have_yielded_data:
                yield ' '
            raise excinfo
        except SegmentError as err:
            self.logger.exception(err)
            # This doesn't actually change the response status (we're too
            # late for that), but this does make it to the logs.
            if self.response:
                self.response.status = HTTP_SERVICE_UNAVAILABLE
            raise