示例#1
0
    def _store_object(self, req, data_source, headers):
        content_type = req.headers.get('content-type', 'octet/stream')
        storage = self.app.storage
        policy = None
        container_info = self.container_info(self.account_name,
                                             self.container_name, req)
        if 'X-Oio-Storage-Policy' in req.headers:
            policy = req.headers.get('X-Oio-Storage-Policy')
            if not self.app.POLICIES.get_by_name(policy):
                raise HTTPBadRequest(
                    "invalid policy '%s', must be in %s" %
                    (policy, self.app.POLICIES.by_name.keys()))
        else:
            try:
                policy_index = int(
                    req.headers.get('X-Backend-Storage-Policy-Index',
                                    container_info['storage_policy']))
            except TypeError:
                policy_index = 0
            if policy_index != 0:
                policy = self.app.POLICIES.get_by_index(policy_index).name
            else:
                content_length = int(req.headers.get('content-length', 0))
                policy = self._get_auto_policy_from_size(content_length)

        metadata = self.load_object_metadata(headers)
        oio_headers = {REQID_HEADER: self.trans_id}
        # only send headers if needed
        if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
            oio_headers[FORCEVERSIONING_HEADER] = \
                headers.get(FORCEVERSIONING_HEADER)
        try:
            _chunks, _size, checksum, _meta = self._object_create(
                self.account_name,
                self.container_name,
                obj_name=self.object_name,
                file_or_path=data_source,
                mime_type=content_type,
                policy=policy,
                headers=oio_headers,
                etag=req.headers.get('etag', '').strip('"'),
                properties=metadata)
            # TODO(FVE): when oio-sds supports it, do that in a callback
            # passed to object_create (or whatever upload method supports it)
            footer_md = self.load_object_metadata(self._get_footers(req))
            if footer_md:
                storage.object_set_properties(self.account_name,
                                              self.container_name,
                                              self.object_name,
                                              version=_meta.get(
                                                  'version', None),
                                              properties=footer_md)
        except exceptions.Conflict:
            raise HTTPConflict(request=req)
        except exceptions.PreconditionFailed:
            raise HTTPPreconditionFailed(request=req)
        except SourceReadTimeout as err:
            self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
            self.app.logger.increment('client_timeouts')
            raise HTTPRequestTimeout(request=req)
        except exceptions.SourceReadError:
            req.client_disconnect = True
            self.app.logger.warning(
                _('Client disconnected without sending last chunk'))
            self.app.logger.increment('client_disconnects')
            raise HTTPClientDisconnect(request=req)
        except exceptions.EtagMismatch:
            return HTTPUnprocessableEntity(request=req)
        except (exceptions.ServiceBusy, exceptions.OioTimeout,
                exceptions.DeadlineReached):
            raise
        except exceptions.NoSuchContainer:
            raise HTTPNotFound(request=req)
        except exceptions.ClientException as err:
            # 481 = CODE_POLICY_NOT_SATISFIABLE
            if err.status == 481:
                raise exceptions.ServiceBusy()
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)
        except Exception:
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)

        last_modified = int(_meta.get('mtime', math.ceil(time.time())))

        resp = HTTPCreated(request=req,
                           etag=checksum,
                           last_modified=last_modified,
                           headers={
                               'x-object-sysmeta-version-id':
                               _meta.get('version', None)
                           })
        return resp
 def __call__(self, env, start_response):
     if not self.storage_domain:
         return self.app(env, start_response)
     given_domain = env['HTTP_HOST']
     port = ''
     if ':' in given_domain:
         given_domain, port = given_domain.rsplit(':', 1)
     if given_domain == self.storage_domain[1:]:  # strip initial '.'
         return self.app(env, start_response)
     a_domain = given_domain
     if not a_domain.endswith(self.storage_domain):
         if self.memcache is None:
             self.memcache = cache_from_env(env)
         error = True
         for tries in xrange(self.lookup_depth):
             found_domain = None
             if self.memcache:
                 memcache_key = ''.join(['cname-', a_domain])
                 found_domain = self.memcache.get(memcache_key)
             if not found_domain:
                 ttl, found_domain = lookup_cname(a_domain)
                 if self.memcache:
                     memcache_key = ''.join(['cname-', given_domain])
                     self.memcache.set(memcache_key,
                                       found_domain,
                                       timeout=ttl)
             if found_domain is None or found_domain == a_domain:
                 # no CNAME records or we're at the last lookup
                 error = True
                 found_domain = None
                 break
             elif found_domain.endswith(self.storage_domain):
                 # Found it!
                 self.logger.info(
                     _('Mapped %(given_domain)s to %(found_domain)s') % {
                         'given_domain': given_domain,
                         'found_domain': found_domain
                     })
                 if port:
                     env['HTTP_HOST'] = ':'.join([found_domain, port])
                 else:
                     env['HTTP_HOST'] = found_domain
                 error = False
                 break
             else:
                 # try one more deep in the chain
                 self.logger.debug(
                     _('Following CNAME chain for  '
                       '%(given_domain)s to %(found_domain)s') % {
                           'given_domain': given_domain,
                           'found_domain': found_domain
                       })
                 a_domain = found_domain
         if error:
             if found_domain:
                 msg = 'CNAME lookup failed after %d tries' % \
                     self.lookup_depth
             else:
                 msg = 'CNAME lookup failed to resolve to a valid domain'
             resp = HTTPBadRequest(request=Request(env),
                                   body=msg,
                                   content_type='text/plain')
             return resp(env, start_response)
     return self.app(env, start_response)
示例#3
0
    def authorize(self, req):
        """
        Returns None if the request is authorized to continue or a standard
        WSGI response callable if not.
        """
        try:
            _junk, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            self.logger.increment('errors')
            return HTTPNotFound(request=req)

        if self._get_account_prefix(account) is None:
            self.logger.debug("Account name: %s doesn't start with "
                              "reseller_prefix(s): %s." %
                              (account, ','.join(self.reseller_prefixes)))
            return self.denied_response(req)

        # At this point, TempAuth is convinced that it is authoritative.
        # If you are sending an ACL header, it must be syntactically valid
        # according to TempAuth's rules for ACL syntax.
        acl_data = req.headers.get('x-account-access-control')
        if acl_data is not None:
            error = self.extract_acl_and_report_errors(req)
            if error:
                msg = 'X-Account-Access-Control invalid: %s\n\nInput: %s\n' % (
                    error, acl_data)
                headers = [('Content-Type', 'text/plain; charset=UTF-8')]
                return HTTPBadRequest(request=req, headers=headers, body=msg)

        user_groups = (req.remote_user or '').split(',')
        account_user = user_groups[1] if len(user_groups) > 1 else None

        if '.reseller_admin' in user_groups and \
                account not in self.reseller_prefixes and \
                not self._dot_account(account):
            req.environ['swift_owner'] = True
            self.logger.debug("User %s has reseller admin authorizing." %
                              account_user)
            return None

        if wsgi_to_str(account) in user_groups and \
                (req.method not in ('DELETE', 'PUT') or container):
            # The user is admin for the account and is not trying to do an
            # account DELETE or PUT
            account_prefix = self._get_account_prefix(account)
            require_group = self.account_rules.get(account_prefix).get(
                'require_group')
            if require_group and require_group in user_groups:
                req.environ['swift_owner'] = True
                self.logger.debug("User %s has admin and %s group."
                                  " Authorizing." %
                                  (account_user, require_group))
                return None
            elif not require_group:
                req.environ['swift_owner'] = True
                self.logger.debug("User %s has admin authorizing." %
                                  account_user)
                return None

        if (req.environ.get('swift_sync_key')
                and (req.environ['swift_sync_key'] == req.headers.get(
                    'x-container-sync-key', None))
                and 'x-timestamp' in req.headers):
            self.logger.debug("Allow request with container sync-key: %s." %
                              req.environ['swift_sync_key'])
            return None

        if req.method == 'OPTIONS':
            # allow OPTIONS requests to proceed as normal
            self.logger.debug("Allow OPTIONS request.")
            return None

        referrers, groups = parse_acl(getattr(req, 'acl', None))

        if referrer_allowed(req.referer, referrers):
            if obj or '.rlistings' in groups:
                self.logger.debug("Allow authorizing %s via referer ACL." %
                                  req.referer)
                return None

        for user_group in user_groups:
            if user_group in groups:
                self.logger.debug("User %s allowed in ACL: %s authorizing." %
                                  (account_user, user_group))
                return None

        # Check for access via X-Account-Access-Control
        acct_acls = self.account_acls(req)
        if acct_acls:
            # At least one account ACL is set in this account's sysmeta data,
            # so we should see whether this user is authorized by the ACLs.
            user_group_set = set(user_groups)
            if user_group_set.intersection(acct_acls['admin']):
                req.environ['swift_owner'] = True
                self.logger.debug('User %s allowed by X-Account-Access-Control'
                                  ' (admin)' % account_user)
                return None
            if (user_group_set.intersection(acct_acls['read-write'])
                    and (container or req.method in ('GET', 'HEAD'))):
                # The RW ACL allows all operations to containers/objects, but
                # only GET/HEAD to accounts (and OPTIONS, above)
                self.logger.debug('User %s allowed by X-Account-Access-Control'
                                  ' (read-write)' % account_user)
                return None
            if (user_group_set.intersection(acct_acls['read-only'])
                    and req.method in ('GET', 'HEAD')):
                self.logger.debug('User %s allowed by X-Account-Access-Control'
                                  ' (read-only)' % account_user)
                return None

        return self.denied_response(req)
示例#4
0
文件: bulk.py 项目: parkys1/swift
    def handle_delete_iter(self,
                           req,
                           objs_to_delete=None,
                           user_agent='BulkDelete',
                           swift_source='BD',
                           out_content_type='text/plain'):
        """
        A generator that can be assigned to a swob Response's app_iter which,
        when iterated over, will delete the objects specified in request body.
        Will occasionally yield whitespace while request is being processed.
        When the request is completed will yield a response body that can be
        parsed to determine success. See above documentation for details.

        :params req: a swob Request
        :params objs_to_delete: a list of dictionaries that specifies the
            objects to be deleted. If None, uses self.get_objs_to_delete to
            query request.
        """
        last_yield = time()
        separator = ''
        failed_files = []
        resp_dict = {
            'Response Status': HTTPOk().status,
            'Response Body': '',
            'Number Deleted': 0,
            'Number Not Found': 0
        }
        try:
            if not out_content_type:
                raise HTTPNotAcceptable(request=req)
            if out_content_type.endswith('/xml'):
                yield '<?xml version="1.0" encoding="UTF-8"?>\n'

            try:
                vrs, account, _junk = req.split_path(2, 3, True)
            except ValueError:
                raise HTTPNotFound(request=req)

            incoming_format = req.headers.get('Content-Type')
            if incoming_format and \
                    not incoming_format.startswith('text/plain'):
                # For now only accept newline separated object names
                raise HTTPNotAcceptable(request=req)

            if objs_to_delete is None:
                objs_to_delete = self.get_objs_to_delete(req)
            failed_file_response_type = HTTPBadRequest
            req.environ['eventlet.minimum_write_chunk_size'] = 0
            for obj_to_delete in objs_to_delete:
                if last_yield + self.yield_frequency < time():
                    separator = '\r\n\r\n'
                    last_yield = time()
                    yield ' '
                obj_to_delete = obj_to_delete.strip()
                if not obj_to_delete:
                    continue
                delete_path = '/'.join(
                    ['', vrs, account,
                     obj_to_delete.lstrip('/')])
                if not check_utf8(delete_path):
                    failed_files.append([
                        quote(obj_to_delete),
                        HTTPPreconditionFailed().status
                    ])
                    continue
                new_env = req.environ.copy()
                new_env['PATH_INFO'] = delete_path
                del (new_env['wsgi.input'])
                new_env['CONTENT_LENGTH'] = 0
                new_env['HTTP_USER_AGENT'] = \
                    '%s %s' % (req.environ.get('HTTP_USER_AGENT'), user_agent)
                new_env['swift.source'] = swift_source
                delete_obj_req = Request.blank(delete_path, new_env)
                resp = delete_obj_req.get_response(self.app)
                if resp.status_int // 100 == 2:
                    resp_dict['Number Deleted'] += 1
                elif resp.status_int == HTTP_NOT_FOUND:
                    resp_dict['Number Not Found'] += 1
                elif resp.status_int == HTTP_UNAUTHORIZED:
                    failed_files.append(
                        [quote(obj_to_delete),
                         HTTPUnauthorized().status])
                    raise HTTPUnauthorized(request=req)
                else:
                    if resp.status_int // 100 == 5:
                        failed_file_response_type = HTTPBadGateway
                    failed_files.append([quote(obj_to_delete), resp.status])

            if failed_files:
                resp_dict['Response Status'] = \
                    failed_file_response_type().status
            elif not (resp_dict['Number Deleted']
                      or resp_dict['Number Not Found']):
                resp_dict['Response Status'] = HTTPBadRequest().status
                resp_dict['Response Body'] = 'Invalid bulk delete.'

        except HTTPException as err:
            resp_dict['Response Status'] = err.status
            resp_dict['Response Body'] = err.body
        except Exception:
            self.logger.exception('Error in bulk delete.')
            resp_dict['Response Status'] = HTTPServerError().status

        yield separator + get_response_body(out_content_type, resp_dict,
                                            failed_files)
示例#5
0
文件: server.py 项目: eckeman/swift
    def account_update(self, req, account, container, broker):
        """
        Update the account server(s) with latest container info.

        :param req: swob.Request object
        :param account: account name
        :param container: container name
        :param broker: container DB broker object
        :returns: if all the account requests return a 404 error code,
                  HTTPNotFound response object,
                  if the account cannot be updated due to a malformed header,
                  an HTTPBadRequest response object,
                  otherwise None.
        """
        account_hosts = [
            h.strip() for h in req.headers.get('X-Account-Host', '').split(',')
        ]
        account_devices = [
            d.strip()
            for d in req.headers.get('X-Account-Device', '').split(',')
        ]
        account_partition = req.headers.get('X-Account-Partition', '')

        if len(account_hosts) != len(account_devices):
            # This shouldn't happen unless there's a bug in the proxy,
            # but if there is, we want to know about it.
            self.logger.error(
                _('ERROR Account update failed: different  '
                  'numbers of hosts and devices in request: '
                  '"%s" vs "%s"') % (req.headers.get('X-Account-Host', ''),
                                     req.headers.get('X-Account-Device', '')))
            return HTTPBadRequest(req=req)

        if account_partition:
            updates = zip(account_hosts, account_devices)
        else:
            updates = []

        account_404s = 0

        for account_host, account_device in updates:
            account_ip, account_port = account_host.rsplit(':', 1)
            new_path = '/' + '/'.join([account, container])
            info = broker.get_info()
            account_headers = HeaderKeyDict({
                'x-put-timestamp':
                info['put_timestamp'],
                'x-delete-timestamp':
                info['delete_timestamp'],
                'x-object-count':
                info['object_count'],
                'x-bytes-used':
                info['bytes_used'],
                'x-trans-id':
                req.headers.get('x-trans-id', '-'),
                'X-Backend-Storage-Policy-Index':
                info['storage_policy_index'],
                'user-agent':
                'container-server %s' % os.getpid(),
                'referer':
                req.as_referer()
            })
            if req.headers.get('x-account-override-deleted', 'no').lower() == \
                    'yes':
                account_headers['x-account-override-deleted'] = 'yes'
            try:
                with ConnectionTimeout(self.conn_timeout):
                    conn = http_connect(account_ip, account_port,
                                        account_device, account_partition,
                                        'PUT', new_path, account_headers)
                with Timeout(self.node_timeout):
                    account_response = conn.getresponse()
                    account_response.read()
                    if account_response.status == HTTP_NOT_FOUND:
                        account_404s += 1
                    elif not is_success(account_response.status):
                        self.logger.error(
                            _('ERROR Account update failed '
                              'with %(ip)s:%(port)s/%(device)s (will retry '
                              'later): Response %(status)s %(reason)s'), {
                                  'ip': account_ip,
                                  'port': account_port,
                                  'device': account_device,
                                  'status': account_response.status,
                                  'reason': account_response.reason
                              })
            except (Exception, Timeout):
                self.logger.exception(
                    _('ERROR account update failed with '
                      '%(ip)s:%(port)s/%(device)s (will retry later)'), {
                          'ip': account_ip,
                          'port': account_port,
                          'device': account_device
                      })
        if updates and account_404s == len(updates):
            return HTTPNotFound(req=req)
        else:
            return None
示例#6
0
    def handle_multipart_put(self, req, start_response):
        """
        Will handle the PUT of a SLO manifest.
        Heads every object in manifest to check if is valid and if so will
        save a manifest generated from the user input. Uses WSGIContext to
        call self and start_response and returns a WSGI iterator.

        :params req: a swob.Request with an obj in path
        :raises: HttpException on errors
        """
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(req.environ, start_response)
        if req.content_length > self.max_manifest_size:
            raise HTTPRequestEntityTooLarge("Manifest File > %d bytes" %
                                            self.max_manifest_size)
        if req.headers.get('X-Copy-From'):
            raise HTTPMethodNotAllowed(
                'Multipart Manifest PUTs cannot be COPY requests')
        if req.content_length is None and \
                req.headers.get('transfer-encoding', '').lower() != 'chunked':
            raise HTTPLengthRequired(request=req)
        parsed_data = parse_and_validate_input(
            req.body_file.read(self.max_manifest_size), req.path,
            self.min_segment_size)
        problem_segments = []

        if len(parsed_data) > self.max_manifest_segments:
            raise HTTPRequestEntityTooLarge(
                'Number of segments must be <= %d' %
                self.max_manifest_segments)
        total_size = 0
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            out_content_type = 'text/plain'
        data_for_storage = []
        slo_etag = md5()
        last_obj_path = None
        for index, seg_dict in enumerate(parsed_data):
            obj_name = seg_dict['path']
            if isinstance(obj_name, six.text_type):
                obj_name = obj_name.encode('utf-8')
            obj_path = '/'.join(['', vrs, account, obj_name.lstrip('/')])

            new_env = req.environ.copy()
            new_env['PATH_INFO'] = obj_path
            new_env['REQUEST_METHOD'] = 'HEAD'
            new_env['swift.source'] = 'SLO'
            del (new_env['wsgi.input'])
            del (new_env['QUERY_STRING'])
            new_env['CONTENT_LENGTH'] = 0
            new_env['HTTP_USER_AGENT'] = \
                '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT')
            if obj_path != last_obj_path:
                last_obj_path = obj_path
                head_seg_resp = \
                    Request.blank(obj_path, new_env).get_response(self)

            if head_seg_resp.is_success:
                segment_length = head_seg_resp.content_length
                if seg_dict.get('range'):
                    # Since we now know the length, we can normalize the
                    # range. We know that there is exactly one range
                    # requested since we checked that earlier in
                    # parse_and_validate_input().
                    ranges = seg_dict['range'].ranges_for_length(
                        head_seg_resp.content_length)

                    if not ranges:
                        problem_segments.append(
                            [quote(obj_name), 'Unsatisfiable Range'])
                    elif ranges == [(0, head_seg_resp.content_length)]:
                        # Just one range, and it exactly matches the object.
                        # Why'd we do this again?
                        del seg_dict['range']
                        segment_length = head_seg_resp.content_length
                    else:
                        rng = ranges[0]
                        seg_dict['range'] = '%d-%d' % (rng[0], rng[1] - 1)
                        segment_length = rng[1] - rng[0]

                if segment_length < self.min_segment_size and \
                        index < len(parsed_data) - 1:
                    problem_segments.append([
                        quote(obj_name),
                        'Too small; each segment, except the last, must be '
                        'at least %d bytes.' % self.min_segment_size
                    ])
                total_size += segment_length
                if seg_dict['size_bytes'] is not None and \
                        seg_dict['size_bytes'] != head_seg_resp.content_length:
                    problem_segments.append([quote(obj_name), 'Size Mismatch'])
                if seg_dict['etag'] is None or \
                        seg_dict['etag'] == head_seg_resp.etag:
                    if seg_dict.get('range'):
                        slo_etag.update(
                            '%s:%s;' % (head_seg_resp.etag, seg_dict['range']))
                    else:
                        slo_etag.update(head_seg_resp.etag)
                else:
                    problem_segments.append([quote(obj_name), 'Etag Mismatch'])
                if head_seg_resp.last_modified:
                    last_modified = head_seg_resp.last_modified
                else:
                    # shouldn't happen
                    last_modified = datetime.now()

                last_modified_formatted = \
                    last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f')
                seg_data = {
                    'name': '/' + seg_dict['path'].lstrip('/'),
                    'bytes': head_seg_resp.content_length,
                    'hash': head_seg_resp.etag,
                    'content_type': head_seg_resp.content_type,
                    'last_modified': last_modified_formatted
                }
                if seg_dict.get('range'):
                    seg_data['range'] = seg_dict['range']

                if config_true_value(
                        head_seg_resp.headers.get('X-Static-Large-Object')):
                    seg_data['sub_slo'] = True
                data_for_storage.append(seg_data)

            else:
                problem_segments.append(
                    [quote(obj_name), head_seg_resp.status])
        if problem_segments:
            resp_body = get_response_body(out_content_type, {},
                                          problem_segments)
            raise HTTPBadRequest(resp_body, content_type=out_content_type)
        env = req.environ

        if not env.get('CONTENT_TYPE'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream'
        env['swift.content_type_overridden'] = True
        env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size
        env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True'
        json_data = json.dumps(data_for_storage)
        if six.PY3:
            json_data = json_data.encode('utf-8')
        env['CONTENT_LENGTH'] = str(len(json_data))
        env['wsgi.input'] = BytesIO(json_data)

        slo_put_context = SloPutContext(self, slo_etag)
        return slo_put_context.handle_slo_put(req, start_response)
示例#7
0
文件: bulk.py 项目: shenps/swift
    def handle_delete(self, req, objs_to_delete=None, user_agent='BulkDelete',
                      swift_source='BD'):
        """
        :params req: a swob Request
        :raises HTTPException: on unhandled errors
        :returns: a swob Response
        """
        try:
            vrs, account, _junk = req.split_path(2, 3, True)
        except ValueError:
            return HTTPNotFound(request=req)

        incoming_format = req.headers.get('Content-Type')
        if incoming_format and not incoming_format.startswith('text/plain'):
            # For now only accept newline separated object names
            return HTTPNotAcceptable(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            return HTTPNotAcceptable(request=req)

        if objs_to_delete is None:
            objs_to_delete = self.get_objs_to_delete(req)
        failed_files = []
        success_count = not_found_count = 0
        failed_file_response_type = HTTPBadRequest
        for obj_to_delete in objs_to_delete:
            obj_to_delete = obj_to_delete.strip().lstrip('/')
            if not obj_to_delete:
                continue
            delete_path = '/'.join(['', vrs, account, obj_to_delete])
            if not check_utf8(delete_path):
                failed_files.append([quote(delete_path),
                                     HTTPPreconditionFailed().status])
                continue
            new_env = req.environ.copy()
            new_env['PATH_INFO'] = delete_path
            del(new_env['wsgi.input'])
            new_env['CONTENT_LENGTH'] = 0
            new_env['HTTP_USER_AGENT'] = \
                '%s %s' % (req.environ.get('HTTP_USER_AGENT'), user_agent)
            new_env['swift.source'] = swift_source
            delete_obj_req = Request.blank(delete_path, new_env)
            resp = delete_obj_req.get_response(self.app)
            if resp.status_int // 100 == 2:
                success_count += 1
            elif resp.status_int == HTTP_NOT_FOUND:
                not_found_count += 1
            elif resp.status_int == HTTP_UNAUTHORIZED:
                return HTTPUnauthorized(request=req)
            else:
                if resp.status_int // 100 == 5:
                    failed_file_response_type = HTTPBadGateway
                failed_files.append([quote(delete_path), resp.status])

        resp_body = get_response_body(
            out_content_type,
            {'Number Deleted': success_count,
             'Number Not Found': not_found_count},
            failed_files)
        if (success_count or not_found_count) and not failed_files:
            return HTTPOk(resp_body, content_type=out_content_type)
        if failed_files:
            return failed_file_response_type(
                resp_body, content_type=out_content_type)
        return HTTPBadRequest('Invalid bulk delete.')
示例#8
0
class ObjectController(object):
    """Implements the WSGI application for the Swift Object Server."""
    def __init__(self, conf):
        """
        Creates a new WSGI application for the Swift Object Server. An
        example configuration is given at
        <source-dir>/etc/object-server.conf-sample or
        /etc/swift/object-server.conf-sample.
        """
        self.logger = get_logger(conf, log_route='object-server')
        self.devices = conf.get('devices', '/srv/node/')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        self.node_timeout = int(conf.get('node_timeout', 3))
        self.conn_timeout = float(conf.get('conn_timeout', 0.5))
        self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
        self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
        self.keep_cache_size = int(conf.get('keep_cache_size', 5242880))
        self.keep_cache_private = \
            config_true_value(conf.get('keep_cache_private', 'false'))
        self.log_requests = config_true_value(conf.get('log_requests', 'true'))
        self.max_upload_time = int(conf.get('max_upload_time', 86400))
        self.slow = int(conf.get('slow', 0))
        self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024
        default_allowed_headers = '''
            content-disposition,
            content-encoding,
            x-delete-at,
            x-object-manifest,
            x-static-large-object,
        '''
        self.allowed_headers = set(
            i.strip().lower() for i in conf.get(
                'allowed_headers', default_allowed_headers).split(',')
            if i.strip() and i.strip().lower() not in DISALLOWED_HEADERS)
        self.expiring_objects_account = \
            (conf.get('auto_create_account_prefix') or '.') + \
            'expiring_objects'
        self.expiring_objects_container_divisor = \
            int(conf.get('expiring_objects_container_divisor') or 86400)

    def async_update(self, op, account, container, obj, host, partition,
                     contdevice, headers_out, objdevice):
        """
        Sends or saves an async update.

        :param op: operation performed (ex: 'PUT', or 'DELETE')
        :param account: account name for the object
        :param container: container name for the object
        :param obj: object name
        :param host: host that the container is on
        :param partition: partition that the container is on
        :param contdevice: device name that the container is on
        :param headers_out: dictionary of headers to send in the container
                            request
        :param objdevice: device name that the object is in
        """
        full_path = '/%s/%s/%s' % (account, container, obj)
        if all([host, partition, contdevice]):
            try:
                with ConnectionTimeout(self.conn_timeout):
                    ip, port = host.rsplit(':', 1)
                    conn = http_connect(ip, port, contdevice, partition, op,
                                        full_path, headers_out)
                with Timeout(self.node_timeout):
                    response = conn.getresponse()
                    response.read()
                    if is_success(response.status):
                        return
                    else:
                        self.logger.error(
                            _('ERROR Container update failed '
                              '(saving for async update later): %(status)d '
                              'response from %(ip)s:%(port)s/%(dev)s'), {
                                  'status': response.status,
                                  'ip': ip,
                                  'port': port,
                                  'dev': contdevice
                              })
            except (Exception, Timeout):
                self.logger.exception(
                    _('ERROR container update failed with '
                      '%(ip)s:%(port)s/%(dev)s (saving for async update later)'
                      ), {
                          'ip': ip,
                          'port': port,
                          'dev': contdevice
                      })
        async_dir = os.path.join(self.devices, objdevice, ASYNCDIR)
        ohash = hash_path(account, container, obj)
        self.logger.increment('async_pendings')
        write_pickle(
            {
                'op': op,
                'account': account,
                'container': container,
                'obj': obj,
                'headers': headers_out
            },
            os.path.join(
                async_dir, ohash[-3:],
                ohash + '-' + normalize_timestamp(headers_out['x-timestamp'])),
            os.path.join(self.devices, objdevice, 'tmp'))

    def container_update(self, op, account, container, obj, headers_in,
                         headers_out, objdevice):
        """
        Update the container when objects are updated.

        :param op: operation performed (ex: 'PUT', or 'DELETE')
        :param account: account name for the object
        :param container: container name for the object
        :param obj: object name
        :param headers_in: dictionary of headers from the original request
        :param headers_out: dictionary of headers to send in the container
                            request(s)
        :param objdevice: device name that the object is in
        """
        conthosts = [
            h.strip()
            for h in headers_in.get('X-Container-Host', '').split(',')
        ]
        contdevices = [
            d.strip()
            for d in headers_in.get('X-Container-Device', '').split(',')
        ]
        contpartition = headers_in.get('X-Container-Partition', '')

        if len(conthosts) != len(contdevices):
            # This shouldn't happen unless there's a bug in the proxy,
            # but if there is, we want to know about it.
            self.logger.error(
                _('ERROR Container update failed: different  '
                  'numbers of hosts and devices in request: '
                  '"%s" vs "%s"' % (headers_in.get('X-Container-Host', ''),
                                    headers_in.get('X-Container-Device', ''))))
            return

        if contpartition:
            updates = zip(conthosts, contdevices)
        else:
            updates = []

        for conthost, contdevice in updates:
            self.async_update(op, account, container, obj, conthost,
                              contpartition, contdevice, headers_out,
                              objdevice)

    def delete_at_update(self, op, delete_at, account, container, obj,
                         headers_in, objdevice):
        """
        Update the expiring objects container when objects are updated.

        :param op: operation performed (ex: 'PUT', or 'DELETE')
        :param account: account name for the object
        :param container: container name for the object
        :param obj: object name
        :param headers_in: dictionary of headers from the original request
        :param objdevice: device name that the object is in
        """
        # Quick cap that will work from now until Sat Nov 20 17:46:39 2286
        # At that time, Swift will be so popular and pervasive I will have
        # created income for thousands of future programmers.
        delete_at = max(min(delete_at, 9999999999), 0)
        updates = [(None, None)]

        partition = None
        hosts = contdevices = [None]
        headers_out = {
            'x-timestamp': headers_in['x-timestamp'],
            'x-trans-id': headers_in.get('x-trans-id', '-')
        }
        if op != 'DELETE':
            partition = headers_in.get('X-Delete-At-Partition', None)
            hosts = headers_in.get('X-Delete-At-Host', '')
            contdevices = headers_in.get('X-Delete-At-Device', '')
            updates = [
                upd for upd in zip((h.strip() for h in hosts.split(',')), (
                    c.strip() for c in contdevices.split(',')))
                if all(upd) and partition
            ]
            if not updates:
                updates = [(None, None)]
            headers_out['x-size'] = '0'
            headers_out['x-content-type'] = 'text/plain'
            headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'

        for host, contdevice in updates:
            self.async_update(
                op, self.expiring_objects_account,
                str(delete_at / self.expiring_objects_container_divisor *
                    self.expiring_objects_container_divisor),
                '%s-%s/%s/%s' % (delete_at, account, container, obj), host,
                partition, contdevice, headers_out, objdevice)

    @public
    @timing_stats()
    def POST(self, request):
        """Handle HTTP POST requests for the Swift Object Server."""
        try:
            device, partition, account, container, obj = \
                split_path(unquote(request.path), 5, 5, True)
            validate_device_partition(device, partition)
        except ValueError, err:
            return HTTPBadRequest(body=str(err),
                                  request=request,
                                  content_type='text/plain')
        if 'x-timestamp' not in request.headers or \
                not check_float(request.headers['x-timestamp']):
            return HTTPBadRequest(body='Missing timestamp',
                                  request=request,
                                  content_type='text/plain')
        new_delete_at = int(request.headers.get('X-Delete-At') or 0)
        if new_delete_at and new_delete_at < time.time():
            return HTTPBadRequest(body='X-Delete-At in past',
                                  request=request,
                                  content_type='text/plain')
        if self.mount_check and not check_mount(self.devices, device):
            return HTTPInsufficientStorage(drive=device, request=request)
        file = DiskFile(self.devices,
                        device,
                        partition,
                        account,
                        container,
                        obj,
                        self.logger,
                        disk_chunk_size=self.disk_chunk_size)

        if file.is_deleted() or file.is_expired():
            return HTTPNotFound(request=request)
        try:
            file.get_data_file_size()
        except (DiskFileError, DiskFileNotExist):
            file.quarantine()
            return HTTPNotFound(request=request)
        metadata = {'X-Timestamp': request.headers['x-timestamp']}
        metadata.update(val for val in request.headers.iteritems()
                        if val[0].lower().startswith('x-object-meta-'))
        for header_key in self.allowed_headers:
            if header_key in request.headers:
                header_caps = header_key.title()
                metadata[header_caps] = request.headers[header_key]
        old_delete_at = int(file.metadata.get('X-Delete-At') or 0)
        if old_delete_at != new_delete_at:
            if new_delete_at:
                self.delete_at_update('PUT', new_delete_at, account, container,
                                      obj, request.headers, device)
            if old_delete_at:
                self.delete_at_update('DELETE', old_delete_at, account,
                                      container, obj, request.headers, device)
        file.put_metadata(metadata)
        return HTTPAccepted(request=request)
示例#9
0
 def check_container_obj(alternative):
     if not alternative[1] or not alternative[2]:
         raise HTTPBadRequest(body="Malformed copy-source header")
     return True
示例#10
0
    def handle_extract_iter(self,
                            req,
                            compress_type,
                            out_content_type='text/plain'):
        """
        A generator that can be assigned to a swob Response's app_iter which,
        when iterated over, will extract and PUT the objects pulled from the
        request body. Will occasionally yield whitespace while request is being
        processed. When the request is completed will yield a response body
        that can be parsed to determine success. See above documentation for
        details.

        :params req: a swob Request
        :params compress_type: specifying the compression type of the tar.
            Accepts '', 'gz', or 'bz2'
        """
        resp_dict = {
            'Response Status': HTTPCreated().status,
            'Response Body': '',
            'Number Files Created': 0
        }
        failed_files = []
        last_yield = time()
        separator = ''
        existing_containers = set()
        try:
            if not out_content_type:
                raise HTTPNotAcceptable(request=req)
            if out_content_type.endswith('/xml'):
                yield '<?xml version="1.0" encoding="UTF-8"?>\n'

            if req.content_length is None and \
                    req.headers.get('transfer-encoding',
                                    '').lower() != 'chunked':
                raise HTTPLengthRequired(request=req)
            try:
                vrs, account, extract_base = req.split_path(2, 3, True)
            except ValueError:
                raise HTTPNotFound(request=req)
            extract_base = extract_base or ''
            extract_base = extract_base.rstrip('/')
            tar = tarfile.open(mode='r|' + compress_type,
                               fileobj=req.body_file)
            failed_response_type = HTTPBadRequest
            req.environ['eventlet.minimum_write_chunk_size'] = 0
            while True:
                if last_yield + self.yield_frequency < time():
                    separator = '\r\n\r\n'
                    last_yield = time()
                    yield ' '
                tar_info = tar.next()
                if tar_info is None or \
                        len(failed_files) >= self.max_failed_extractions:
                    break
                if tar_info.isfile():
                    obj_path = tar_info.name
                    if obj_path.startswith('./'):
                        obj_path = obj_path[2:]
                    obj_path = obj_path.lstrip('/')
                    if extract_base:
                        obj_path = extract_base + '/' + obj_path
                    if '/' not in obj_path:
                        continue  # ignore base level file

                    destination = '/'.join(['', vrs, account, obj_path])
                    container = obj_path.split('/', 1)[0]
                    if not check_utf8(destination):
                        failed_files.append([
                            quote(obj_path[:MAX_PATH_LENGTH]),
                            HTTPPreconditionFailed().status
                        ])
                        continue
                    if tar_info.size > MAX_FILE_SIZE:
                        failed_files.append([
                            quote(obj_path[:MAX_PATH_LENGTH]),
                            HTTPRequestEntityTooLarge().status
                        ])
                        continue
                    if container not in existing_containers:
                        try:
                            self.create_container(
                                req, '/'.join(['', vrs, account, container]))
                            existing_containers.add(container)
                        except CreateContainerError, err:
                            failed_files.append([
                                quote(obj_path[:MAX_PATH_LENGTH]), err.status
                            ])
                            if err.status_int == HTTP_UNAUTHORIZED:
                                raise HTTPUnauthorized(request=req)
                            continue
                        except ValueError:
                            failed_files.append([
                                quote(obj_path[:MAX_PATH_LENGTH]),
                                HTTPBadRequest().status
                            ])
                            continue
                        if len(existing_containers) > self.max_containers:
                            raise HTTPBadRequest(
                                'More than %d base level containers in tar.' %
                                self.max_containers)
示例#11
0
 def PUT(self, request):
     """Handle HTTP PUT requests for the Swift Object Server."""
     try:
         device, partition, account, container, obj = \
             split_path(unquote(request.path), 5, 5, True)
         validate_device_partition(device, partition)
     except ValueError, err:
         return HTTPBadRequest(body=str(err),
                               request=request,
                               content_type='text/plain')
     if self.mount_check and not check_mount(self.devices, device):
         return HTTPInsufficientStorage(drive=device, request=request)
     if 'x-timestamp' not in request.headers or \
             not check_float(request.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing timestamp',
                               request=request,
                               content_type='text/plain')
     error_response = check_object_creation(request, obj)
     if error_response:
         return error_response
     new_delete_at = int(request.headers.get('X-Delete-At') or 0)
     if new_delete_at and new_delete_at < time.time():
         return HTTPBadRequest(body='X-Delete-At in past',
                               request=request,
                               content_type='text/plain')
     file = DiskFile(self.devices,
                     device,
                     partition,
                     account,
                     container,
                     obj,
示例#12
0
                    else:
                        if resp.status_int == HTTP_UNAUTHORIZED:
                            failed_files.append([
                                quote(obj_path[:MAX_PATH_LENGTH]),
                                HTTPUnauthorized().status
                            ])
                            raise HTTPUnauthorized(request=req)
                        if resp.status_int // 100 == 5:
                            failed_response_type = HTTPBadGateway
                        failed_files.append(
                            [quote(obj_path[:MAX_PATH_LENGTH]), resp.status])

            if failed_files:
                resp_dict['Response Status'] = failed_response_type().status
            elif not resp_dict['Number Files Created']:
                resp_dict['Response Status'] = HTTPBadRequest().status
                resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files'

        except HTTPException, err:
            resp_dict['Response Status'] = err.status
            resp_dict['Response Body'] = err.body
        except tarfile.TarError, tar_error:
            resp_dict['Response Status'] = HTTPBadRequest().status
            resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error
        except Exception:
            self.logger.exception('Error in extract archive.')
            resp_dict['Response Status'] = HTTPServerError().status

        yield separator + get_response_body(out_content_type, resp_dict,
                                            failed_files)
    def __call__(self, request):

        if request.method not in ("POST", "PUT"):
            return self.app

        try:
            ver, account, container, obj = request.split_path(
                2, 4, rest_with_last=True)
        except ValueError:
            return self.app

        if not container:
            # account request, so we pay attention to the quotas
            new_quota = request.headers.get(
                'X-Account-Meta-Quota-Bytes')
            remove_quota = request.headers.get(
                'X-Remove-Account-Meta-Quota-Bytes')
        else:
            # container or object request; even if the quota headers are set
            # in the request, they're meaningless
            new_quota = remove_quota = None

        if remove_quota:
            new_quota = 0    # X-Remove dominates if both are present

        if request.environ.get('reseller_request') is True:
            if new_quota and not new_quota.isdigit():
                return HTTPBadRequest()
            return self.app

        # deny quota set for non-reseller
        if new_quota is not None:
            return HTTPForbidden()

        if request.method == "POST" or not obj:
            return self.app

        content_length = (request.content_length or 0)

        account_info = get_account_info(request.environ, self.app)
        if not account_info or not account_info['bytes']:
            return self.app
        try:
            quota = int(account_info['meta'].get('quota-bytes', -1))
        except ValueError:
            return self.app
        if quota < 0:
            return self.app

        new_size = int(account_info['bytes']) + content_length
        if quota < new_size:
            resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
            if 'swift.authorize' in request.environ:
                orig_authorize = request.environ['swift.authorize']

                def reject_authorize(*args, **kwargs):
                    aresp = orig_authorize(*args, **kwargs)
                    if aresp:
                        return aresp
                    return resp
                request.environ['swift.authorize'] = reject_authorize
            else:
                return resp

        return self.app
示例#14
0
    def PUT(self, request):
        """Handle HTTP PUT requests for the Swift Object Server."""
        device, partition, account, container, obj = \
            split_and_validate_path(request, 5, 5, True)

        if 'x-timestamp' not in request.headers or \
                not check_float(request.headers['x-timestamp']):
            return HTTPBadRequest(body='Missing timestamp',
                                  request=request,
                                  content_type='text/plain')
        error_response = check_object_creation(request, obj)
        if error_response:
            return error_response
        new_delete_at = int(request.headers.get('X-Delete-At') or 0)
        if new_delete_at and new_delete_at < time.time():
            return HTTPBadRequest(body='X-Delete-At in past',
                                  request=request,
                                  content_type='text/plain')
        try:
            fsize = request.message_length()
        except ValueError as e:
            return HTTPBadRequest(body=str(e),
                                  request=request,
                                  content_type='text/plain')
        try:
            disk_file = self._diskfile(device, partition, account, container,
                                       obj)
        except DiskFileDeviceUnavailable:
            return HTTPInsufficientStorage(drive=device, request=request)
        with disk_file.open():
            orig_metadata = disk_file.get_metadata()
        old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
        orig_timestamp = orig_metadata.get('X-Timestamp')
        if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']:
            return HTTPConflict(request=request)
        upload_expiration = time.time() + self.max_upload_time
        etag = md5()
        elapsed_time = 0
        try:
            with disk_file.create(size=fsize) as writer:
                reader = request.environ['wsgi.input'].read
                for chunk in iter(lambda: reader(self.network_chunk_size), ''):
                    start_time = time.time()
                    if start_time > upload_expiration:
                        self.logger.increment('PUT.timeouts')
                        return HTTPRequestTimeout(request=request)
                    etag.update(chunk)
                    writer.write(chunk)
                    sleep()
                    elapsed_time += time.time() - start_time
                upload_size = writer.upload_size
                if upload_size:
                    self.logger.transfer_rate('PUT.' + device + '.timing',
                                              elapsed_time, upload_size)
                if fsize is not None and fsize != upload_size:
                    return HTTPClientDisconnect(request=request)
                etag = etag.hexdigest()
                if 'etag' in request.headers and \
                        request.headers['etag'].lower() != etag:
                    return HTTPUnprocessableEntity(request=request)
                metadata = {
                    'X-Timestamp': request.headers['x-timestamp'],
                    'Content-Type': request.headers['content-type'],
                    'ETag': etag,
                    'Content-Length': str(upload_size),
                }
                metadata.update(val for val in request.headers.iteritems()
                                if val[0].lower().startswith('x-object-meta-')
                                and len(val[0]) > 14)
                for header_key in self.allowed_headers:
                    if header_key in request.headers:
                        header_caps = header_key.title()
                        metadata[header_caps] = request.headers[header_key]
                writer.put(metadata)
        except DiskFileNoSpace:
            return HTTPInsufficientStorage(drive=device, request=request)
        if old_delete_at != new_delete_at:
            if new_delete_at:
                self.delete_at_update('PUT', new_delete_at, account, container,
                                      obj, request, device)
            if old_delete_at:
                self.delete_at_update('DELETE', old_delete_at, account,
                                      container, obj, request, device)
        if not orig_timestamp or \
                orig_timestamp < request.headers['x-timestamp']:
            self.container_update(
                'PUT', account, container, obj, request,
                HeaderKeyDict({
                    'x-size': metadata['Content-Length'],
                    'x-content-type': metadata['Content-Type'],
                    'x-timestamp': metadata['X-Timestamp'],
                    'x-etag': metadata['ETag']
                }), device)
        resp = HTTPCreated(request=request, etag=etag)
        return resp
示例#15
0
    def GET(self):
        """
        GET handler on Proxy
        """
        if self.is_range_request:
            msg = 'Storlet execution with range header is not supported'
            raise HTTPBadRequest(msg.encode('utf8'),
                                 request=self.request)

        params = self.verify_access_to_storlet()
        self.augment_storlet_request(params)

        # Range requests:
        # Range requests are not allowed with storlet invocation.
        # To run a storlet on a selected input range use the X-Storlet-Range
        # header.
        # If the range request is to be executed on the proxy we
        # create an HTTP Range request based on X-Storlet-Range
        # and let the request continue so that we get the required
        # range as input to the storlet that would get executed on
        # the proxy.
        if self.execute_range_on_proxy:
            self.request.headers['Range'] = \
                self.request.headers['X-Storlet-Range']

        original_resp = self.request.get_response(self.app)
        if original_resp.status_int == 403:
            # The user is unauthoried to read from the container.
            # It might be, however, that the user is permitted
            # to read given that the required storlet is executed.
            if not self.request.environ['HTTP_X_USER_NAME']:
                # The requester is not even an authenticated user.
                self.logger.info(('Storlet run request by an'
                                  ' authenticated user'))
                raise HTTPUnauthorized(b'User is not authorized')

            user_name = self.request.environ['HTTP_X_USER_NAME']
            storlet_name = self.request.headers['X-Run-Storlet']
            internal_referer = '//%s' % self._build_acl_string(user_name,
                                                               storlet_name)
            self.logger.info(('Got 403 for original GET %s request. '
                              'Trying with storlet internal referer %s' %
                              (self.path, internal_referer)))
            self.request.referrer = self.request.referer = internal_referer
            original_resp = self.request.get_response(self.app)

        if original_resp.is_success:
            # The get request may be a SLO object GET request.
            # Simplest solution would be to invoke a HEAD
            # for every GET request to test if we are in SLO case.
            # In order to save the HEAD overhead we implemented
            # a slightly more involved flow:
            # At proxy side, we augment request with Storlet stuff
            # and let the request flow.
            # At object side, we invoke the plain (non Storlet)
            # request and test if we are in SLO case.
            # and invoke Storlet only if non SLO case.
            # Back at proxy side, we test if test received
            # full object to detect if we are in SLO case,
            # and invoke Storlet only if in SLO case.
            if self.is_proxy_runnable(original_resp):
                self.gather_extra_sources()
                return self.apply_storlet(original_resp)
            else:
                # Non proxy GET case: Storlet was already invoked at
                # object side
                # TODO(kota_): Do we need to pop the Transfer-Encoding/
                #              Content-Length header from the resp?
                if 'Transfer-Encoding' in original_resp.headers:
                    original_resp.headers.pop('Transfer-Encoding')

                original_resp.headers['Content-Length'] = None
                return original_resp

        else:
            # In failure case, we need nothing to do, just return original
            # response
            return original_resp
示例#16
0
    def __call__(self, req):
        try:
            (version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app

        # verify new quota headers are properly formatted
        if not obj and req.method in ('PUT', 'POST'):
            val = req.headers.get('X-Container-Meta-Quota-Bytes')
            if val and not val.isdigit():
                return HTTPBadRequest(body='Invalid bytes quota.')
            val = req.headers.get('X-Container-Meta-Quota-Count')
            if val and not val.isdigit():
                return HTTPBadRequest(body='Invalid count quota.')

        # check user uploads against quotas
        elif obj and req.method in ('PUT', 'COPY'):
            container_info = None
            if req.method == 'PUT':
                container_info = get_container_info(
                    req.environ, self.app, swift_source='CQ')
            if req.method == 'COPY' and 'Destination' in req.headers:
                dest_account = account
                if 'Destination-Account' in req.headers:
                    dest_account = req.headers.get('Destination-Account')
                    dest_account = check_account_format(req, dest_account)
                dest_container, dest_object = check_destination_header(req)
                path_info = req.environ['PATH_INFO']
                req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
                    version, dest_account, dest_container, dest_object)
                try:
                    container_info = get_container_info(
                        req.environ, self.app, swift_source='CQ')
                finally:
                    req.environ['PATH_INFO'] = path_info
            if not container_info or not is_success(container_info['status']):
                # this will hopefully 404 later
                return self.app

            if 'quota-bytes' in container_info.get('meta', {}) and \
                    'bytes' in container_info and \
                    container_info['meta']['quota-bytes'].isdigit():
                content_length = (req.content_length or 0)
                if 'x-copy-from' in req.headers or req.method == 'COPY':
                    if 'x-copy-from' in req.headers:
                        container, obj = check_copy_from_header(req)
                    path = '/%s/%s/%s/%s' % (version, account,
                                             container, obj)
                    object_info = get_object_info(req.environ, self.app, path)
                    if not object_info or not object_info['length']:
                        content_length = 0
                    else:
                        content_length = int(object_info['length'])
                new_size = int(container_info['bytes']) + content_length
                if int(container_info['meta']['quota-bytes']) < new_size:
                    return self.bad_response(req, container_info)

            if 'quota-count' in container_info.get('meta', {}) and \
                    'object_count' in container_info and \
                    container_info['meta']['quota-count'].isdigit():
                new_count = int(container_info['object_count']) + 1
                if int(container_info['meta']['quota-count']) < new_count:
                    return self.bad_response(req, container_info)

        return self.app
示例#17
0
def parse_and_validate_input(req_body, req_path, min_segment_size):
    """
    Given a request body, parses it and returns a list of dictionaries.

    The output structure is nearly the same as the input structure, but it
    is not an exact copy. Given a valid input dictionary `d_in`, its
    corresponding output dictionary `d_out` will be as follows:

      * d_out['etag'] == d_in['etag']

      * d_out['path'] == d_in['path']

      * d_in['size_bytes'] can be a string ("12") or an integer (12), but
        d_out['size_bytes'] is an integer.

      * (optional) d_in['range'] is a string of the form "M-N", "M-", or
        "-N", where M and N are non-negative integers. d_out['range'] is the
        corresponding swob.Range object. If d_in does not have a key
        'range', neither will d_out.

    :raises: HTTPException on parse errors or semantic errors (e.g. bogus
        JSON structure, syntactically invalid ranges)

    :returns: a list of dictionaries on success
    """
    try:
        parsed_data = json.loads(req_body)
    except ValueError:
        raise HTTPBadRequest("Manifest must be valid JSON.\n")

    if not isinstance(parsed_data, list):
        raise HTTPBadRequest("Manifest must be a list.\n")

    # If we got here, req_path refers to an object, so this won't ever raise
    # ValueError.
    vrs, account, _junk = split_path(req_path, 3, 3, True)

    errors = []
    num_segs = len(parsed_data)
    for seg_index, seg_dict in enumerate(parsed_data):
        if not isinstance(seg_dict, dict):
            errors.append("Index %d: not a JSON object" % seg_index)
            continue

        missing_keys = [k for k in REQUIRED_SLO_KEYS if k not in seg_dict]
        if missing_keys:
            errors.append(
                "Index %d: missing keys %s" %
                (seg_index, ", ".join('"%s"' % (mk, )
                                      for mk in sorted(missing_keys))))
            continue

        extraneous_keys = [k for k in seg_dict if k not in ALLOWED_SLO_KEYS]
        if extraneous_keys:
            errors.append(
                "Index %d: extraneous keys %s" %
                (seg_index, ", ".join('"%s"' % (ek, )
                                      for ek in sorted(extraneous_keys))))
            continue

        if not isinstance(seg_dict['path'], basestring):
            errors.append("Index %d: \"path\" must be a string" % seg_index)
            continue
        if not (seg_dict['etag'] is None
                or isinstance(seg_dict['etag'], basestring)):
            errors.append("Index %d: \"etag\" must be a string or null" %
                          seg_index)
            continue

        if '/' not in seg_dict['path'].strip('/'):
            errors.append(
                "Index %d: path does not refer to an object. Path must be of "
                "the form /container/object." % seg_index)
            continue

        seg_size = seg_dict['size_bytes']
        if seg_size is not None:
            try:
                seg_size = int(seg_size)
                seg_dict['size_bytes'] = seg_size
            except (TypeError, ValueError):
                errors.append("Index %d: invalid size_bytes" % seg_index)
                continue
            if (seg_size < min_segment_size and seg_index < num_segs - 1):
                errors.append("Index %d: too small; each segment, except "
                              "the last, must be at least %d bytes." %
                              (seg_index, min_segment_size))
                continue

        obj_path = '/'.join(['', vrs, account, seg_dict['path'].lstrip('/')])
        if req_path == quote(obj_path):
            errors.append(
                "Index %d: manifest must not include itself as a segment" %
                seg_index)
            continue

        if seg_dict.get('range'):
            try:
                seg_dict['range'] = Range('bytes=%s' % seg_dict['range'])
            except ValueError:
                errors.append("Index %d: invalid range" % seg_index)
                continue

            if len(seg_dict['range'].ranges) > 1:
                errors.append("Index %d: multiple ranges (only one allowed)" %
                              seg_index)
                continue

            # If the user *told* us the object's size, we can check range
            # satisfiability right now. If they lied about the size, we'll
            # fail that validation later.
            if (seg_size is not None and
                    len(seg_dict['range'].ranges_for_length(seg_size)) != 1):
                errors.append("Index %d: unsatisfiable range" % seg_index)
                continue

    if errors:
        error_message = "".join(e + "\n" for e in errors)
        raise HTTPBadRequest(error_message,
                             headers={"Content-Type": "text/plain"})

    return parsed_data
示例#18
0
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not containers:
            return HTTPNotFound(request=req)
        partition, nodes = self.app.object_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)
        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req.headers['X-Timestamp'] = \
                    normalize_timestamp(float(req.headers['x-timestamp']))
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
        else:
            req.headers['X-Timestamp'] = normalize_timestamp(time.time())

        headers = self._backend_requests(
            req, len(nodes), container_partition, containers)
        resp = self.make_requests(req, self.app.object_ring,
                                  partition, 'DELETE', req.path_info, headers)
        return resp

    @public
    @cors_validation
    @delay_denial
    def COPY(self, req):
        """HTTP COPY request handler."""
示例#19
0
文件: bulk.py 项目: shenps/swift
                        success_count += 1
                    else:
                        if resp.status_int == HTTP_UNAUTHORIZED:
                            return HTTPUnauthorized(request=req)
                        failed_files.append([
                            quote(destination[:MAX_PATH_LENGTH]), resp.status])

            resp_body = get_response_body(
                out_content_type,
                {'Number Files Created': success_count},
                failed_files)
            if success_count and not failed_files:
                return HTTPCreated(resp_body, content_type=out_content_type)
            if failed_files:
                return HTTPBadGateway(resp_body, content_type=out_content_type)
            return HTTPBadRequest('Invalid Tar File: No Valid Files')

        except tarfile.TarError, tar_error:
            return HTTPBadRequest('Invalid Tar File: %s' % tar_error)

    @wsgify
    def __call__(self, req):
        extract_type = req.params.get('extract-archive')
        if extract_type is not None and req.method == 'PUT':
            archive_type = {
                'tar': '', 'tar.gz': 'gz',
                'tar.bz2': 'bz2'}.get(extract_type.lower().strip('.'))
            if archive_type is not None:
                return self.handle_extract(req, archive_type)
            else:
                return HTTPBadRequest("Unsupported archive format")
示例#20
0
    def POST(self, req):
        """HTTP POST request handler."""
        if 'x-delete-after' in req.headers:
            try:
                x_delete_after = int(req.headers['x-delete-after'])
            except ValueError:
                return HTTPBadRequest(request=req,
                                      content_type='text/plain',
                                      body='Non-integer X-Delete-After')
            req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
        if self.app.object_post_as_copy:
            req.method = 'PUT'
            req.path_info = '/%s/%s/%s' % (
                self.account_name, self.container_name, self.object_name)
            req.headers['Content-Length'] = 0
            req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name,
                                               self.object_name))
            req.headers['X-Fresh-Metadata'] = 'true'
            req.environ['swift_versioned_copy'] = True
            if req.environ.get('QUERY_STRING'):
                req.environ['QUERY_STRING'] += '&multipart-manifest=get'
            else:
                req.environ['QUERY_STRING'] = 'multipart-manifest=get'
            resp = self.PUT(req)
            # Older editions returned 202 Accepted on object POSTs, so we'll
            # convert any 201 Created responses to that for compatibility with
            # picky clients.
            if resp.status_int != HTTP_CREATED:
                return resp
            return HTTPAccepted(request=req)
        else:
            error_response = check_metadata(req, 'object')
            if error_response:
                return error_response
            container_info = self.container_info(
                self.account_name, self.container_name)
            container_partition = container_info['partition']
            containers = container_info['nodes']
            req.acl = container_info['write_acl']
            if 'swift.authorize' in req.environ:
                aresp = req.environ['swift.authorize'](req)
                if aresp:
                    return aresp
            if not containers:
                return HTTPNotFound(request=req)
            if 'x-delete-at' in req.headers:
                try:
                    x_delete_at = int(req.headers['x-delete-at'])
                    if x_delete_at < time.time():
                        return HTTPBadRequest(
                            body='X-Delete-At in past', request=req,
                            content_type='text/plain')
                except ValueError:
                    return HTTPBadRequest(request=req,
                                          content_type='text/plain',
                                          body='Non-integer X-Delete-At')
                delete_at_container = str(
                    x_delete_at /
                    self.app.expiring_objects_container_divisor *
                    self.app.expiring_objects_container_divisor)
                delete_at_part, delete_at_nodes = \
                    self.app.container_ring.get_nodes(
                        self.app.expiring_objects_account, delete_at_container)
            else:
                delete_at_part = delete_at_nodes = None
            partition, nodes = self.app.object_ring.get_nodes(
                self.account_name, self.container_name, self.object_name)
            req.headers['X-Timestamp'] = normalize_timestamp(time.time())

            headers = self._backend_requests(
                req, len(nodes), container_partition, containers,
                delete_at_part, delete_at_nodes)

            resp = self.make_requests(req, self.app.object_ring, partition,
                                      'POST', req.path_info, headers)
            return resp
示例#21
0
文件: bulk.py 项目: shenps/swift
    def handle_extract(self, req, compress_type):
        """
        :params req: a swob Request
        :params compress_type: specifying the compression type of the tar.
                               Accepts '', 'gz, or 'bz2'
        :raises HTTPException: on unhandled errors
        :returns: a swob response to request
        """
        success_count = 0
        failed_files = []
        existing_containers = set()
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            return HTTPNotAcceptable(request=req)
        if req.content_length is None and \
                req.headers.get('transfer-encoding', '').lower() != 'chunked':
            return HTTPLengthRequired(request=req)
        try:
            vrs, account, extract_base = req.split_path(2, 3, True)
        except ValueError:
            return HTTPNotFound(request=req)
        extract_base = extract_base or ''
        extract_base = extract_base.rstrip('/')
        try:
            tar = tarfile.open(mode='r|' + compress_type,
                               fileobj=req.body_file)
            while True:
                tar_info = tar.next()
                if tar_info is None or \
                        len(failed_files) >= self.max_failed_extractions:
                    break
                if tar_info.isfile():
                    obj_path = tar_info.name
                    if obj_path.startswith('./'):
                        obj_path = obj_path[2:]
                    obj_path = obj_path.lstrip('/')
                    if extract_base:
                        obj_path = extract_base + '/' + obj_path
                    if '/' not in obj_path:
                        continue  # ignore base level file

                    destination = '/'.join(
                        ['', vrs, account, obj_path])
                    container = obj_path.split('/', 1)[0]
                    if not check_utf8(destination):
                        failed_files.append(
                            [quote(destination[:MAX_PATH_LENGTH]),
                             HTTPPreconditionFailed().status])
                        continue
                    if tar_info.size > MAX_FILE_SIZE:
                        failed_files.append([
                            quote(destination[:MAX_PATH_LENGTH]),
                            HTTPRequestEntityTooLarge().status])
                        continue
                    if container not in existing_containers:
                        try:
                            self.create_container(
                                req, '/'.join(['', vrs, account, container]))
                            existing_containers.add(container)
                        except CreateContainerError, err:
                            if err.status_int == HTTP_UNAUTHORIZED:
                                return HTTPUnauthorized(request=req)
                            failed_files.append([
                                quote(destination[:MAX_PATH_LENGTH]),
                                err.status])
                            continue
                        except ValueError:
                            failed_files.append([
                                quote(destination[:MAX_PATH_LENGTH]),
                                HTTP_BAD_REQUEST])
                            continue
                        if len(existing_containers) > self.max_containers:
                            return HTTPBadRequest(
                                'More than %d base level containers in tar.' %
                                self.max_containers)
示例#22
0
    def PUT(self, req):
        """HTTP PUT request handler."""
        container_info = self.container_info(
            self.account_name, self.container_name)
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not containers:
            return HTTPNotFound(request=req)
        if 'x-delete-after' in req.headers:
            try:
                x_delete_after = int(req.headers['x-delete-after'])
            except ValueError:
                    return HTTPBadRequest(request=req,
                                          content_type='text/plain',
                                          body='Non-integer X-Delete-After')
            req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
        partition, nodes = self.app.object_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)
        # do a HEAD request for container sync and checking object versions
        if 'x-timestamp' in req.headers or \
                (object_versions and not
                 req.environ.get('swift_versioned_copy')):
            hreq = Request.blank(req.path_info, headers={'X-Newest': 'True'},
                                 environ={'REQUEST_METHOD': 'HEAD'})
            hresp = self.GETorHEAD_base(
                hreq, _('Object'), self.app.object_ring, partition,
                hreq.path_info)
        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req.headers['X-Timestamp'] = \
                    normalize_timestamp(float(req.headers['x-timestamp']))
                if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
                    float(hresp.environ['swift_x_timestamp']) >= \
                        float(req.headers['x-timestamp']):
                    return HTTPAccepted(request=req)
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
        else:
            req.headers['X-Timestamp'] = normalize_timestamp(time.time())
        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        if not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                'application/octet-stream'
            content_type_manually_set = False
        error_response = check_object_creation(req, self.object_name) or \
            check_content_type(req)
        if error_response:
            return error_response
        if object_versions and not req.environ.get('swift_versioned_copy'):
            is_manifest = 'x-object-manifest' in req.headers or \
                          'x-object-manifest' in hresp.headers
            if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
                # This is a version manifest and needs to be handled
                # differently. First copy the existing data to a new object,
                # then write the data from this request to the version manifest
                # object.
                lcontainer = object_versions.split('/')[0]
                prefix_len = '%03x' % len(self.object_name)
                lprefix = prefix_len + self.object_name + '/'
                ts_source = hresp.environ.get('swift_x_timestamp')
                if ts_source is None:
                    ts_source = time.mktime(time.strptime(
                                            hresp.headers['last-modified'],
                                            '%a, %d %b %Y %H:%M:%S GMT'))
                new_ts = normalize_timestamp(ts_source)
                vers_obj_name = lprefix + new_ts
                copy_headers = {
                    'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
                copy_environ = {'REQUEST_METHOD': 'COPY',
                                'swift_versioned_copy': True
                                }
                copy_req = Request.blank(req.path_info, headers=copy_headers,
                                         environ=copy_environ)
                copy_resp = self.COPY(copy_req)
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

        reader = req.environ['wsgi.input'].read
        data_source = iter(lambda: reader(self.app.client_chunk_size), '')
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            source_header = unquote(source_header)
            acct = req.path_info.split('/', 2)[1]
            if isinstance(acct, unicode):
                acct = acct.encode('utf-8')
            if not source_header.startswith('/'):
                source_header = '/' + source_header
            source_header = '/' + acct + source_header
            try:
                src_container_name, src_obj_name = \
                    source_header.split('/', 3)[2:]
            except ValueError:
                return HTTPPreconditionFailed(
                    request=req,
                    body='X-Copy-From header must be of the form'
                         '<container name>/<object name>')
            source_req = req.copy_get()
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            source_resp = self.GET(source_req)
            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            new_req = Request.blank(req.path_info,
                                    environ=req.environ, headers=req.headers)
            data_source = source_resp.app_iter
            new_req.content_length = source_resp.content_length
            if new_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            if new_req.content_length > MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=req)
            new_req.etag = source_resp.etag
            # we no longer need the X-Copy-From header
            del new_req.headers['X-Copy-From']
            if not content_type_manually_set:
                new_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if not config_true_value(
                    new_req.headers.get('x-fresh-metadata', 'false')):
                copy_headers_into(source_resp, new_req)
                copy_headers_into(req, new_req)
            # copy over x-static-large-object for POSTs and manifest copies
            if 'X-Static-Large-Object' in source_resp.headers and \
                    req.params.get('multipart-manifest') == 'get':
                new_req.headers['X-Static-Large-Object'] = \
                    source_resp.headers['X-Static-Large-Object']

            req = new_req

        if 'x-delete-at' in req.headers:
            try:
                x_delete_at = int(req.headers['x-delete-at'])
                if x_delete_at < time.time():
                    return HTTPBadRequest(
                        body='X-Delete-At in past', request=req,
                        content_type='text/plain')
            except ValueError:
                return HTTPBadRequest(request=req, content_type='text/plain',
                                      body='Non-integer X-Delete-At')
            delete_at_container = str(
                x_delete_at /
                self.app.expiring_objects_container_divisor *
                self.app.expiring_objects_container_divisor)
            delete_at_part, delete_at_nodes = \
                self.app.container_ring.get_nodes(
                    self.app.expiring_objects_account, delete_at_container)
        else:
            delete_at_part = delete_at_nodes = None

        node_iter = GreenthreadSafeIterator(
            self.iter_nodes(self.app.object_ring, partition))
        pile = GreenPile(len(nodes))
        chunked = req.headers.get('transfer-encoding')

        outgoing_headers = self._backend_requests(
            req, len(nodes), container_partition, containers,
            delete_at_part, delete_at_nodes)

        for nheaders in outgoing_headers:
            # RFC2616:8.2.3 disallows 100-continue without a body
            if (req.content_length > 0) or chunked:
                nheaders['Expect'] = '100-continue'
            pile.spawn(self._connect_put_node, node_iter, partition,
                       req.path_info, nheaders, self.app.logger.thread_locals)

        conns = [conn for conn in pile if conn]
        if len(conns) <= len(nodes) / 2:
            self.app.logger.error(
                _('Object PUT returning 503, %(conns)s/%(nodes)s '
                  'required connections'),
                {'conns': len(conns), 'nodes': len(nodes) // 2 + 1})
            return HTTPServiceUnavailable(request=req)
        bytes_transferred = 0
        try:
            with ContextPool(len(nodes)) as pool:
                for conn in conns:
                    conn.failed = False
                    conn.queue = Queue(self.app.put_queue_depth)
                    pool.spawn(self._send_file, conn, req.path)
                while True:
                    with ChunkReadTimeout(self.app.client_timeout):
                        try:
                            chunk = next(data_source)
                        except StopIteration:
                            if chunked:
                                [conn.queue.put('0\r\n\r\n') for conn in conns]
                            break
                    bytes_transferred += len(chunk)
                    if bytes_transferred > MAX_FILE_SIZE:
                        return HTTPRequestEntityTooLarge(request=req)
                    for conn in list(conns):
                        if not conn.failed:
                            conn.queue.put(
                                '%x\r\n%s\r\n' % (len(chunk), chunk)
                                if chunked else chunk)
                        else:
                            conns.remove(conn)
                    if len(conns) <= len(nodes) / 2:
                        self.app.logger.error(_(
                            'Object PUT exceptions during'
                            ' send, %(conns)s/%(nodes)s required connections'),
                            {'conns': len(conns), 'nodes': len(nodes) / 2 + 1})
                        return HTTPServiceUnavailable(request=req)
                for conn in conns:
                    if conn.queue.unfinished_tasks:
                        conn.queue.join()
            conns = [conn for conn in conns if not conn.failed]
        except ChunkReadTimeout, err:
            self.app.logger.warn(
                _('ERROR Client read timeout (%ss)'), err.seconds)
            self.app.logger.increment('client_timeouts')
            return HTTPRequestTimeout(request=req)
示例#23
0
文件: bulk.py 项目: parkys1/swift
    def handle_extract_iter(self,
                            req,
                            compress_type,
                            out_content_type='text/plain'):
        """
        A generator that can be assigned to a swob Response's app_iter which,
        when iterated over, will extract and PUT the objects pulled from the
        request body. Will occasionally yield whitespace while request is being
        processed. When the request is completed will yield a response body
        that can be parsed to determine success. See above documentation for
        details.

        :params req: a swob Request
        :params compress_type: specifying the compression type of the tar.
            Accepts '', 'gz', or 'bz2'
        """
        resp_dict = {
            'Response Status': HTTPCreated().status,
            'Response Body': '',
            'Number Files Created': 0
        }
        failed_files = []
        last_yield = time()
        separator = ''
        containers_accessed = set()
        try:
            if not out_content_type:
                raise HTTPNotAcceptable(request=req)
            if out_content_type.endswith('/xml'):
                yield '<?xml version="1.0" encoding="UTF-8"?>\n'

            if req.content_length is None and \
                    req.headers.get('transfer-encoding',
                                    '').lower() != 'chunked':
                raise HTTPLengthRequired(request=req)
            try:
                vrs, account, extract_base = req.split_path(2, 3, True)
            except ValueError:
                raise HTTPNotFound(request=req)
            extract_base = extract_base or ''
            extract_base = extract_base.rstrip('/')
            tar = tarfile.open(mode='r|' + compress_type,
                               fileobj=req.body_file)
            failed_response_type = HTTPBadRequest
            req.environ['eventlet.minimum_write_chunk_size'] = 0
            containers_created = 0
            while True:
                if last_yield + self.yield_frequency < time():
                    separator = '\r\n\r\n'
                    last_yield = time()
                    yield ' '
                tar_info = tar.next()
                if tar_info is None or \
                        len(failed_files) >= self.max_failed_extractions:
                    break
                if tar_info.isfile():
                    obj_path = tar_info.name
                    if obj_path.startswith('./'):
                        obj_path = obj_path[2:]
                    obj_path = obj_path.lstrip('/')
                    if extract_base:
                        obj_path = extract_base + '/' + obj_path
                    if '/' not in obj_path:
                        continue  # ignore base level file

                    destination = '/'.join(['', vrs, account, obj_path])
                    container = obj_path.split('/', 1)[0]
                    if not check_utf8(destination):
                        failed_files.append([
                            quote(obj_path[:MAX_PATH_LENGTH]),
                            HTTPPreconditionFailed().status
                        ])
                        continue
                    if tar_info.size > MAX_FILE_SIZE:
                        failed_files.append([
                            quote(obj_path[:MAX_PATH_LENGTH]),
                            HTTPRequestEntityTooLarge().status
                        ])
                        continue
                    container_failure = None
                    if container not in containers_accessed:
                        cont_path = '/'.join(['', vrs, account, container])
                        try:
                            if self.create_container(req, cont_path):
                                containers_created += 1
                                if containers_created > self.max_containers:
                                    raise HTTPBadRequest(
                                        'More than %d containers to create '
                                        'from tar.' % self.max_containers)
                        except CreateContainerError as err:
                            # the object PUT to this container still may
                            # succeed if acls are set
                            container_failure = [
                                quote(cont_path[:MAX_PATH_LENGTH]), err.status
                            ]
                            if err.status_int == HTTP_UNAUTHORIZED:
                                raise HTTPUnauthorized(request=req)
                        except ValueError:
                            failed_files.append([
                                quote(obj_path[:MAX_PATH_LENGTH]),
                                HTTPBadRequest().status
                            ])
                            continue

                    tar_file = tar.extractfile(tar_info)
                    new_env = req.environ.copy()
                    new_env['REQUEST_METHOD'] = 'PUT'
                    new_env['wsgi.input'] = tar_file
                    new_env['PATH_INFO'] = destination
                    new_env['CONTENT_LENGTH'] = tar_info.size
                    new_env['swift.source'] = 'EA'
                    new_env['HTTP_USER_AGENT'] = \
                        '%s BulkExpand' % req.environ.get('HTTP_USER_AGENT')
                    create_obj_req = Request.blank(destination, new_env)
                    resp = create_obj_req.get_response(self.app)
                    containers_accessed.add(container)
                    if resp.is_success:
                        resp_dict['Number Files Created'] += 1
                    else:
                        if container_failure:
                            failed_files.append(container_failure)
                        if resp.status_int == HTTP_UNAUTHORIZED:
                            failed_files.append([
                                quote(obj_path[:MAX_PATH_LENGTH]),
                                HTTPUnauthorized().status
                            ])
                            raise HTTPUnauthorized(request=req)
                        if resp.status_int // 100 == 5:
                            failed_response_type = HTTPBadGateway
                        failed_files.append(
                            [quote(obj_path[:MAX_PATH_LENGTH]), resp.status])

            if failed_files:
                resp_dict['Response Status'] = failed_response_type().status
            elif not resp_dict['Number Files Created']:
                resp_dict['Response Status'] = HTTPBadRequest().status
                resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files'

        except HTTPException as err:
            resp_dict['Response Status'] = err.status
            resp_dict['Response Body'] = err.body
        except tarfile.TarError as tar_error:
            resp_dict['Response Status'] = HTTPBadRequest().status
            resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error
        except Exception:
            self.logger.exception('Error in extract archive.')
            resp_dict['Response Status'] = HTTPServerError().status

        yield separator + get_response_body(out_content_type, resp_dict,
                                            failed_files)
示例#24
0
    def get_object_head_resp(self, req):
        storage = self.app.storage
        oio_headers = {REQID_HEADER: self.trans_id}
        oio_cache = req.environ.get('oio.cache')
        perfdata = req.environ.get('oio.perfdata')
        version = req.environ.get('oio.query', {}).get('version')
        force_master = False
        while True:
            try:
                if self.app.check_state:
                    metadata, chunks = storage.object_locate(
                        self.account_name,
                        self.container_name,
                        self.object_name,
                        version=version,
                        headers=oio_headers,
                        force_master=force_master,
                        cache=oio_cache,
                        perfdata=perfdata)
                else:
                    metadata = storage.object_get_properties(
                        self.account_name,
                        self.container_name,
                        self.object_name,
                        version=version,
                        headers=oio_headers,
                        force_master=force_master,
                        cache=oio_cache,
                        perfdata=perfdata)
                break
            except (exceptions.NoSuchObject, exceptions.NoSuchContainer):
                if force_master or not \
                        self.container_name.endswith(MULTIUPLOAD_SUFFIX):
                    # Either the request failed with the master,
                    # or it is not an MPU
                    return HTTPNotFound(request=req)

                # This part appears in the manifest, so it should be there.
                # To be sure, we must go check the master
                # in case of desynchronization.
                force_master = True

        if self.app.check_state:
            storage_method = STORAGE_METHODS.load(metadata['chunk_method'])
            # TODO(mbo): use new property of STORAGE_METHODS
            min_chunks = storage_method.ec_nb_data if storage_method.ec else 1

            chunks_by_pos = _sort_chunks(chunks, storage_method.ec)
            for idx, entries in enumerate(chunks_by_pos.iteritems()):
                if idx != entries[0]:
                    return HTTPBadRequest(request=req)
                nb_chunks_ok = 0
                for entry in entries[1]:
                    try:
                        storage.blob_client.chunk_head(entry['url'],
                                                       headers=oio_headers)
                        nb_chunks_ok += 1
                    except exceptions.OioException:
                        pass
                    if nb_chunks_ok >= min_chunks:
                        break
                else:
                    return HTTPBadRequest(request=req)

        resp = self.make_object_response(req, metadata)
        return resp
示例#25
0
文件: server.py 项目: eckeman/swift
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     req_timestamp = valid_timestamp(req)
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     requested_policy_index = self.get_and_validate_policy_index(req)
     broker = self._get_container_broker(drive, part, account, container)
     if obj:  # put container object
         # obj put expects the policy_index header, default is for
         # legacy support during upgrade.
         obj_policy_index = requested_policy_index or 0
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(req_timestamp.internal, obj_policy_index)
             except DatabaseAlreadyExists:
                 pass
         if not os.path.exists(broker.db_file):
             return HTTPNotFound()
         broker.put_object(obj, req_timestamp.internal,
                           int(req.headers['x-size']),
                           req.headers['x-content-type'],
                           req.headers['x-etag'], 0, obj_policy_index)
         return HTTPCreated(request=req)
     else:  # put container
         if requested_policy_index is None:
             # use the default index sent by the proxy if available
             new_container_policy = req.headers.get(
                 'X-Backend-Storage-Policy-Default', int(POLICIES.default))
         else:
             new_container_policy = requested_policy_index
         created = self._update_or_create(req, broker,
                                          req_timestamp.internal,
                                          new_container_policy,
                                          requested_policy_index)
         metadata = {}
         metadata.update((key, (value, req_timestamp.internal))
                         for key, value in req.headers.iteritems()
                         if key.lower() in self.save_headers
                         or is_sys_or_user_meta('container', key))
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
         resp = self.account_update(req, account, container, broker)
         if resp:
             return resp
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
示例#26
0
    def _store_object(self, req, data_source, headers):
        content_type = req.headers.get('content-type', 'octet/stream')
        policy = None
        container_info = self.container_info(self.account_name,
                                             self.container_name, req)
        if 'X-Oio-Storage-Policy' in req.headers:
            policy = req.headers.get('X-Oio-Storage-Policy')
            if not self.app.POLICIES.get_by_name(policy):
                raise HTTPBadRequest(
                    "invalid policy '%s', must be in %s" %
                    (policy, self.app.POLICIES.by_name.keys()))
        else:
            try:
                policy_index = int(
                    req.headers.get('X-Backend-Storage-Policy-Index',
                                    container_info['storage_policy']))
            except TypeError:
                policy_index = 0
            if policy_index != 0:
                policy = self.app.POLICIES.get_by_index(policy_index).name
            else:
                content_length = int(req.headers.get('content-length', 0))
                policy = self._get_auto_policy_from_size(content_length)

        ct_props = {'properties': {}, 'system': {}}
        metadata = self.load_object_metadata(headers)
        oio_headers = {REQID_HEADER: self.trans_id}
        oio_cache = req.environ.get('oio.cache')
        perfdata = req.environ.get('oio.perfdata')
        # only send headers if needed
        if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
            oio_headers[FORCEVERSIONING_HEADER] = \
                headers.get(FORCEVERSIONING_HEADER)
        # In case a shard is being created, save the name of the S3 bucket
        # in a container property. This will be used when aggregating
        # container statistics to make bucket statistics.
        if BUCKET_NAME_HEADER in headers:
            bname = headers[BUCKET_NAME_HEADER]
            # FIXME(FVE): the segments container is not part of another bucket!
            # We should not have to strip this here.
            if bname and bname.endswith(MULTIUPLOAD_SUFFIX):
                bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
            ct_props['system'][BUCKET_NAME_PROP] = bname
        try:
            _chunks, _size, checksum, _meta = self._object_create(
                self.account_name,
                self.container_name,
                obj_name=self.object_name,
                file_or_path=data_source,
                mime_type=content_type,
                policy=policy,
                headers=oio_headers,
                etag=req.headers.get('etag', '').strip('"'),
                properties=metadata,
                container_properties=ct_props,
                cache=oio_cache,
                perfdata=perfdata)
            # TODO(FVE): when oio-sds supports it, do that in a callback
            # passed to object_create (or whatever upload method supports it)
            footer_md = self.load_object_metadata(self._get_footers(req))
            if footer_md:
                self.app.storage.object_set_properties(self.account_name,
                                                       self.container_name,
                                                       self.object_name,
                                                       version=_meta.get(
                                                           'version', None),
                                                       properties=footer_md,
                                                       headers=oio_headers,
                                                       cache=oio_cache,
                                                       perfdata=perfdata)
        except exceptions.Conflict:
            raise HTTPConflict(request=req)
        except exceptions.PreconditionFailed:
            raise HTTPPreconditionFailed(request=req)
        except SourceReadTimeout as err:
            self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
            self.app.logger.increment('client_timeouts')
            raise HTTPRequestTimeout(request=req)
        except exceptions.SourceReadError as err:
            req.client_disconnect = True
            self.app.logger.warning(
                _('Client disconnected without sending last chunk') +
                (': %s' % str(err)))
            self.app.logger.increment('client_disconnects')
            raise HTTPClientDisconnect(request=req)
        except exceptions.EtagMismatch:
            return HTTPUnprocessableEntity(request=req)
        except (exceptions.ServiceBusy, exceptions.OioTimeout,
                exceptions.DeadlineReached):
            raise
        except exceptions.NoSuchContainer:
            raise HTTPNotFound(request=req)
        except exceptions.ClientException as err:
            # 481 = CODE_POLICY_NOT_SATISFIABLE
            if err.status == 481:
                raise exceptions.ServiceBusy()
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)
        except Exception:
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)

        last_modified = int(_meta.get('mtime', math.ceil(time.time())))

        resp = HTTPCreated(request=req,
                           etag=checksum,
                           last_modified=last_modified,
                           headers={
                               'x-object-sysmeta-version-id':
                               _meta.get('version', None)
                           })
        return resp
示例#27
0
    def handle_request(self, req):
        """
        Entry point for proxy server.
        Should return a WSGI-style callable (such as swob.Response).

        :param req: swob.Request object
        """
        try:
            self.logger.set_statsd_prefix('proxy-server')
            if req.content_length and req.content_length < 0:
                self.logger.increment('errors')
                return HTTPBadRequest(request=req,
                                      body='Invalid Content-Length')

            try:
                if not check_utf8(req.path_info):
                    self.logger.increment('errors')
                    return HTTPPreconditionFailed(
                        request=req, body='Invalid UTF8 or contains NULL')
            except UnicodeError:
                self.logger.increment('errors')
                return HTTPPreconditionFailed(
                    request=req, body='Invalid UTF8 or contains NULL')

            try:
                controller, path_parts = self.get_controller(req.path)
                p = req.path_info
                if isinstance(p, unicode):
                    p = p.encode('utf-8')
            except ValueError:
                self.logger.increment('errors')
                return HTTPNotFound(request=req)
            if not controller:
                self.logger.increment('errors')
                return HTTPPreconditionFailed(request=req, body='Bad URL')
            if self.deny_host_headers and \
                    req.host.split(':')[0] in self.deny_host_headers:
                return HTTPForbidden(request=req, body='Invalid host header')

            self.logger.set_statsd_prefix('proxy-server.' +
                                          controller.server_type.lower())
            controller = controller(self, **path_parts)
            if 'swift.trans_id' not in req.environ:
                # if this wasn't set by an earlier middleware, set it now
                trans_id = generate_trans_id(self.trans_id_suffix)
                req.environ['swift.trans_id'] = trans_id
                self.logger.txn_id = trans_id
            req.headers['x-trans-id'] = req.environ['swift.trans_id']
            controller.trans_id = req.environ['swift.trans_id']
            self.logger.client_ip = get_remote_client(req)
            try:
                handler = getattr(controller, req.method)
                getattr(handler, 'publicly_accessible')
            except AttributeError:
                allowed_methods = getattr(controller, 'allowed_methods', set())
                return HTTPMethodNotAllowed(
                    request=req, headers={'Allow': ', '.join(allowed_methods)})
            if 'swift.authorize' in req.environ:
                # We call authorize before the handler, always. If authorized,
                # we remove the swift.authorize hook so isn't ever called
                # again. If not authorized, we return the denial unless the
                # controller's method indicates it'd like to gather more
                # information and try again later.
                resp = req.environ['swift.authorize'](req)
                if not resp:
                    # No resp means authorized, no delayed recheck required.
                    del req.environ['swift.authorize']
                else:
                    # Response indicates denial, but we might delay the denial
                    # and recheck later. If not delayed, return the error now.
                    if not getattr(handler, 'delay_denial', None):
                        return resp
            # Save off original request method (GET, POST, etc.) in case it
            # gets mutated during handling.  This way logging can display the
            # method the client actually sent.
            req.environ['swift.orig_req_method'] = req.method
            return handler(req)
        except HTTPException as error_response:
            return error_response
        except (Exception, Timeout):
            self.logger.exception(_('ERROR Unhandled exception in request'))
            return HTTPServerError(request=req)
示例#28
0
文件: server.py 项目: r0mik/swift
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing timestamp',
                               request=req,
                               content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err = validate_sync_to(req.headers['x-container-sync-to'],
                                self.allowed_sync_hosts)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     broker = self._get_container_broker(drive, part, account, container)
     if obj:  # put container object
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
             except swift.common.db.DatabaseAlreadyExists:
                 pass
         if not os.path.exists(broker.db_file):
             return HTTPNotFound()
         broker.put_object(obj, timestamp, int(req.headers['x-size']),
                           req.headers['x-content-type'],
                           req.headers['x-etag'])
         return HTTPCreated(request=req)
     else:  # put container
         if not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
                 created = True
             except swift.common.db.DatabaseAlreadyExists:
                 pass
         else:
             created = broker.is_deleted()
             broker.update_put_timestamp(timestamp)
             if broker.is_deleted():
                 return HTTPConflict(request=req)
         metadata = {}
         metadata.update((key, (value, timestamp))
                         for key, value in req.headers.iteritems()
                         if key.lower() in self.save_headers
                         or key.lower().startswith('x-container-meta-'))
         if metadata:
             if 'X-Container-Sync-To' in metadata:
                 if 'X-Container-Sync-To' not in broker.metadata or \
                         metadata['X-Container-Sync-To'][0] != \
                         broker.metadata['X-Container-Sync-To'][0]:
                     broker.set_x_container_sync_points(-1, -1)
             broker.update_metadata(metadata)
         resp = self.account_update(req, account, container, broker)
         if resp:
             return resp
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
示例#29
0
    def handle_get_token(self, req):
        """
        Handles the various `request for token and service end point(s)` calls.
        There are various formats to support the various auth servers in the
        past. Examples::

            GET <auth-prefix>/v1/<act>/auth
                X-Auth-User: <act>:<usr>  or  X-Storage-User: <usr>
                X-Auth-Key: <key>         or  X-Storage-Pass: <key>
            GET <auth-prefix>/auth
                X-Auth-User: <act>:<usr>  or  X-Storage-User: <act>:<usr>
                X-Auth-Key: <key>         or  X-Storage-Pass: <key>
            GET <auth-prefix>/v1.0
                X-Auth-User: <act>:<usr>  or  X-Storage-User: <act>:<usr>
                X-Auth-Key: <key>         or  X-Storage-Pass: <key>

        On successful authentication, the response will have X-Auth-Token and
        X-Storage-Token set to the token to use with Swift and X-Storage-URL
        set to the URL to the default Swift cluster to use.

        :param req: The swob.Request to process.
        :returns: swob.Response, 2xx on success with data set as explained
                  above.
        """
        # Validate the request info
        try:
            pathsegs = split_path(req.path_info, 1, 3, True)
        except ValueError:
            self.logger.increment('errors')
            return HTTPNotFound(request=req)
        if pathsegs[0] == 'v1' and pathsegs[2] == 'auth':
            account = pathsegs[1]
            user = req.headers.get('x-storage-user')
            if not user:
                user = req.headers.get('x-auth-user')
                if not user or ':' not in user:
                    self.logger.increment('token_denied')
                    auth = 'Swift realm="%s"' % account
                    return HTTPUnauthorized(request=req,
                                            headers={'Www-Authenticate': auth})
                account2, user = user.split(':', 1)
                if wsgi_to_str(account) != account2:
                    self.logger.increment('token_denied')
                    auth = 'Swift realm="%s"' % account
                    return HTTPUnauthorized(request=req,
                                            headers={'Www-Authenticate': auth})
            key = req.headers.get('x-storage-pass')
            if not key:
                key = req.headers.get('x-auth-key')
        elif pathsegs[0] in ('auth', 'v1.0'):
            user = req.headers.get('x-auth-user')
            if not user:
                user = req.headers.get('x-storage-user')
            if not user or ':' not in user:
                self.logger.increment('token_denied')
                auth = 'Swift realm="unknown"'
                return HTTPUnauthorized(request=req,
                                        headers={'Www-Authenticate': auth})
            account, user = user.split(':', 1)
            key = req.headers.get('x-auth-key')
            if not key:
                key = req.headers.get('x-storage-pass')
        else:
            return HTTPBadRequest(request=req)
        if not all((account, user, key)):
            self.logger.increment('token_denied')
            realm = account or 'unknown'
            return HTTPUnauthorized(
                request=req,
                headers={'Www-Authenticate': 'Swift realm="%s"' % realm})
        # Authenticate user
        account_user = account + ':' + user
        if account_user not in self.users:
            self.logger.increment('token_denied')
            auth = 'Swift realm="%s"' % account
            return HTTPUnauthorized(request=req,
                                    headers={'Www-Authenticate': auth})
        if self.users[account_user]['key'] != key:
            self.logger.increment('token_denied')
            auth = 'Swift realm="unknown"'
            return HTTPUnauthorized(request=req,
                                    headers={'Www-Authenticate': auth})
        account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
        # Get memcache client
        memcache_client = cache_from_env(req.environ)
        if not memcache_client:
            raise Exception('Memcache required')
        # See if a token already exists and hasn't expired
        token = None
        memcache_user_key = '%s/user/%s' % (self.reseller_prefix, account_user)
        candidate_token = memcache_client.get(memcache_user_key)
        if candidate_token:
            memcache_token_key = \
                '%s/token/%s' % (self.reseller_prefix, candidate_token)
            cached_auth_data = memcache_client.get(memcache_token_key)
            if cached_auth_data:
                expires, old_groups = cached_auth_data
                old_groups = [
                    group.encode('utf8') if six.PY2 else group
                    for group in old_groups.split(',')
                ]
                new_groups = self._get_user_groups(account, account_user,
                                                   account_id)

                if expires > time() and \
                        set(old_groups) == set(new_groups.split(',')):
                    token = candidate_token
        # Create a new token if one didn't exist
        if not token:
            # Generate new token
            token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
            expires = time() + self.token_life
            groups = self._get_user_groups(account, account_user, account_id)
            # Save token
            memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token)
            memcache_client.set(memcache_token_key, (expires, groups),
                                time=float(expires - time()))
            # Record the token with the user info for future use.
            memcache_user_key = \
                '%s/user/%s' % (self.reseller_prefix, account_user)
            memcache_client.set(memcache_user_key,
                                token,
                                time=float(expires - time()))
        resp = Response(request=req,
                        headers={
                            'x-auth-token': token,
                            'x-storage-token': token,
                            'x-auth-token-expires': str(int(expires - time()))
                        })
        url = self.users[account_user]['url'].replace('$HOST', resp.host_url)
        if self.storage_url_scheme != 'default':
            url = self.storage_url_scheme + ':' + url.split(':', 1)[1]
        resp.headers['x-storage-url'] = url
        return resp
示例#30
0
    def GET(self, req):
        """
        Handle HTTP GET request.

        The body of the response to a successful GET request contains a listing
        of either objects or shard ranges. The exact content of the listing is
        determined by a combination of request headers and query string
        parameters, as follows:

        * The type of the listing is determined by the
          ``X-Backend-Record-Type`` header. If this header has value ``shard``
          then the response body will be a list of shard ranges; if this header
          has value ``auto``, and the container state is ``sharding`` or
          ``sharded``, then the listing will be a list of shard ranges;
          otherwise the response body will be a list of objects.

        * Both shard range and object listings may be constrained to a name
          range by the ``marker`` and ``end_marker`` query string parameters.
          Object listings will only contain objects whose names are greater
          than any ``marker`` value and less than any ``end_marker`` value.
          Shard range listings will only contain shard ranges whose namespace
          is greater than or includes any ``marker`` value and is less than or
          includes any ``end_marker`` value.

        * Shard range listings may also be constrained by an ``includes`` query
          string parameter. If this parameter is present the listing will only
          contain shard ranges whose namespace includes the value of the
          parameter; any ``marker`` or ``end_marker`` parameters are ignored

        * The length of an object listing may be constrained by the ``limit``
          parameter. Object listings may also be constrained by ``prefix``,
          ``delimiter`` and ``path`` query string parameters.

        * Shard range listings will include deleted shard ranges if and only if
          the ``X-Backend-Include-Deleted`` header value is one of
          :attr:`swift.common.utils.TRUE_VALUES`. Object listings never
          include deleted objects.

        * Shard range listings may be constrained to include only shard ranges
          whose state is specified by a query string ``states`` parameter. If
          present, the ``states`` parameter should be a comma separated list of
          either the string or integer representation of
          :data:`~swift.common.utils.ShardRange.STATES`.

          Two alias values may be used in a ``states`` parameter value:
          ``listing`` will cause the listing to include all shard ranges in a
          state suitable for contributing to an object listing; ``updating``
          will cause the listing to include all shard ranges in a state
          suitable to accept an object update.

          If either of these aliases is used then the shard range listing will
          if necessary be extended with a synthesised 'filler' range in order
          to satisfy the requested name range when insufficient actual shard
          ranges are found. Any 'filler' shard range will cover the otherwise
          uncovered tail of the requested name range and will point back to the
          same container.

        * Listings are not normally returned from a deleted container. However,
          the ``X-Backend-Override-Deleted`` header may be used with a value in
          :attr:`swift.common.utils.TRUE_VALUES` to force a shard range
          listing to be returned from a deleted container whose DB file still
          exists.

        :param req: an instance of :class:`swift.common.swob.Request`
        :returns: an instance of :class:`swift.common.swob.Response`
        """
        drive, part, account, container, obj = split_and_validate_path(
            req, 4, 5, True)
        path = get_param(req, 'path')
        prefix = get_param(req, 'prefix')
        delimiter = get_param(req, 'delimiter')
        if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
            # delimiters can be made more flexible later
            return HTTPPreconditionFailed(body='Bad delimiter')
        marker = get_param(req, 'marker', '')
        end_marker = get_param(req, 'end_marker')
        limit = constraints.CONTAINER_LISTING_LIMIT
        given_limit = get_param(req, 'limit')
        reverse = config_true_value(get_param(req, 'reverse'))
        if given_limit and given_limit.isdigit():
            limit = int(given_limit)
            if limit > constraints.CONTAINER_LISTING_LIMIT:
                return HTTPPreconditionFailed(
                    request=req,
                    body='Maximum limit is %d' %
                    constraints.CONTAINER_LISTING_LIMIT)
        out_content_type = listing_formats.get_listing_content_type(req)
        try:
            check_drive(self.root, drive, self.mount_check)
        except ValueError:
            return HTTPInsufficientStorage(drive=drive, request=req)
        broker = self._get_container_broker(drive,
                                            part,
                                            account,
                                            container,
                                            pending_timeout=0.1,
                                            stale_reads_ok=True)
        info, is_deleted = broker.get_info_is_deleted()
        record_type = req.headers.get('x-backend-record-type', '').lower()
        if record_type == 'auto' and info.get('db_state') in (SHARDING,
                                                              SHARDED):
            record_type = 'shard'
        if record_type == 'shard':
            override_deleted = info and config_true_value(
                req.headers.get('x-backend-override-deleted', False))
            resp_headers = gen_resp_headers(info,
                                            is_deleted=is_deleted
                                            and not override_deleted)
            if is_deleted and not override_deleted:
                return HTTPNotFound(request=req, headers=resp_headers)
            resp_headers['X-Backend-Record-Type'] = 'shard'
            includes = get_param(req, 'includes')
            states = get_param(req, 'states')
            fill_gaps = False
            if states:
                states = list_from_csv(states)
                fill_gaps = any(('listing' in states, 'updating' in states))
                try:
                    states = broker.resolve_shard_range_states(states)
                except ValueError:
                    return HTTPBadRequest(request=req, body='Bad state')
            include_deleted = config_true_value(
                req.headers.get('x-backend-include-deleted', False))
            container_list = broker.get_shard_ranges(
                marker,
                end_marker,
                includes,
                reverse,
                states=states,
                include_deleted=include_deleted,
                fill_gaps=fill_gaps)
        else:
            resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
            if is_deleted:
                return HTTPNotFound(request=req, headers=resp_headers)
            resp_headers['X-Backend-Record-Type'] = 'object'
            # Use the retired db while container is in process of sharding,
            # otherwise use current db
            src_broker = broker.get_brokers()[0]
            container_list = src_broker.list_objects_iter(
                limit,
                marker,
                end_marker,
                prefix,
                delimiter,
                path,
                storage_policy_index=info['storage_policy_index'],
                reverse=reverse)
        return self.create_listing(req, out_content_type, info, resp_headers,
                                   broker.metadata, container_list, container)