Пример #1
0
    def _enforce_image_property_quota(self,
                                      image_meta,
                                      orig_image_meta=None,
                                      purge_props=False,
                                      req=None):
        if CONF.image_property_quota < 0:
            # If value is negative, allow unlimited number of properties
            return

        props = image_meta['properties'].keys()

        # NOTE(ameade): If we are not removing existing properties,
        # take them in to account
        if (not purge_props) and orig_image_meta:
            original_props = orig_image_meta['properties'].keys()
            props.extend(original_props)
            props = set(props)

        if len(props) > CONF.image_property_quota:
            msg = (_("The limit has been exceeded on the number of allowed "
                     "image properties. Attempted: %s, Maximum: %s") %
                   (len(props), CONF.image_property_quota))
            LOG.info(msg)
            raise HTTPRequestEntityTooLarge(explanation=msg,
                                            request=req,
                                            content_type="text/plain")
Пример #2
0
def check_object_creation(req, object_name):
    """
    Check to ensure that everything is alright about an object to be created.

    :param req: HTTP request object
    :param object_name: name of object to be created
    :raises HTTPRequestEntityTooLarge: the object is too large
    :raises HTTPLengthRequered: missing content-length header and not
                                a chunked request
    :raises HTTPBadRequest: missing or bad content-type header, or
                            bad metadata
    """
    if req.content_length and req.content_length > MAX_FILE_SIZE:
        return HTTPRequestEntityTooLarge(body='Your request is too large.',
                                         request=req,
                                         content_type='text/plain')
    if req.content_length is None and \
            req.headers.get('transfer-encoding') != 'chunked':
        return HTTPLengthRequired(request=req)
    if 'X-Copy-From' in req.headers and req.content_length:
        return HTTPBadRequest(body='Copy requests require a zero byte body',
                              request=req,
                              content_type='text/plain')
    if len(object_name) > MAX_OBJECT_NAME_LENGTH:
        return HTTPBadRequest(body='Object name length of %d longer than %d' %
                              (len(object_name), MAX_OBJECT_NAME_LENGTH),
                              request=req,
                              content_type='text/plain')
    if 'Content-Type' not in req.headers:
        return HTTPBadRequest(request=req,
                              content_type='text/plain',
                              body='No content type')
    if not check_utf8(req.headers['Content-Type']):
        return HTTPBadRequest(request=req,
                              body='Invalid Content-Type',
                              content_type='text/plain')
    if 'x-object-manifest' in req.headers:
        value = req.headers['x-object-manifest']
        container = prefix = None
        try:
            container, prefix = value.split('/', 1)
        except ValueError:
            pass
        if not container or not prefix or '?' in value or '&' in value or \
                prefix[0] == '/':
            return HTTPBadRequest(
                request=req,
                body='X-Object-Manifest must in the format container/prefix')
    return check_metadata(req, 'object')
Пример #3
0
    def make_service_call(self, req, env, method_dict, variables):

        try:
            req_method = req.method.upper()

            call_pattern = method_dict[req_method]

            call_pattern = self.replace_variables(req, call_pattern, variables)

            self.logger.debug("Call on resource is: %s" % str(call_pattern))

            response = eval(call_pattern)

        except BadRequestException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPBadRequest(body=detail,
                                  content_type=content_type,
                                  request=req)

        except NotImplementedException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPNotImplemented(body=detail,
                                      content_type=content_type,
                                      request=req)

        except ServiceUnavailableException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPInternalServerError(body=detail,
                                           content_type=content_type,
                                           request=req)

        except ItemNotFoundException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPNotFound(body=detail,
                                content_type=content_type,
                                request=req)

        except OverLimitException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPRequestEntityTooLarge(body=detail,
                                             content_type=content_type,
                                             request=req)

        except ImmutableEntityException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPUnprocessableEntity(body=detail,
                                           content_type=content_type,
                                           request=req)

        except LoadBalancerFaultException as error:

            content_type, detail = self.get_error_body(req, error)

            return HTTPUnauthorized(body=detail,
                                    content_type=content_type,
                                    request=req)

        except Exception, msg:

            exc_type, exc_value, exc_tb = sys.exc_info()

            tb = traceback.format_exception(exc_type, exc_value, exc_tb)

            tb.reverse()

            tb_str = ''.join(tb)

            self.logger.debug("cannot eval(%s) because %s. Traceback: %s" %
                              (call_pattern, msg, tb_str))

            error = ServiceUnavailableException(
                "The Load Balancing Service is currently unavailable." +
                " Please contact support")

            content_type, detail = self.get_error_body(req, error)

            return HTTPInternalServerError(body=detail,
                                           content_type=content_type,
                                           request=req)
Пример #4
0
    def _upload(self, req, image_meta):
        """
        Uploads the payload of the request to a backend store in
        Glance. If the `x-image-meta-store` header is set, Glance
        will attempt to use that scheme; if not, Glance will use the
        scheme set by the flag `default_store` to find the backing store.

        :param req: The WSGI/Webob Request object
        :param image_meta: Mapping of metadata about image

        :raises HTTPConflict if image already exists
        :retval The location where the image was stored
        """

        copy_from = self._copy_from(req)
        if copy_from:
            try:
                image_data, image_size = self._get_from_store(req.context,
                                                              copy_from)
            except Exception as e:
                self._safe_kill(req, image_meta['id'])
                msg = _("Copy from external source failed: %s") % e
                LOG.debug(msg)
                return
            image_meta['size'] = image_size or image_meta['size']
        else:
            try:
                req.get_content_type('application/octet-stream')
            except exception.InvalidContentType:
                self._safe_kill(req, image_meta['id'])
                msg = _("Content-Type must be application/octet-stream")
                LOG.debug(msg)
                raise HTTPBadRequest(explanation=msg)

            image_data = req.body_file

        scheme = req.headers.get('x-image-meta-store', CONF.default_store)

        store = self.get_store_or_400(req, scheme)

        image_id = image_meta['id']
        LOG.debug(_("Setting image %s to status 'saving'"), image_id)
        registry.update_image_metadata(req.context, image_id,
                                       {'status': 'saving'})

        LOG.debug(_("Uploading image data for image %(image_id)s "
                    "to %(scheme)s store"), locals())

        try:
            self.notifier.info("image.prepare", redact_loc(image_meta))
            location, size, checksum = store.add(
                image_meta['id'],
                utils.CooperativeReader(image_data),
                image_meta['size'])

            def _kill_mismatched(image_meta, attr, actual):
                supplied = image_meta.get(attr)
                if supplied and supplied != actual:
                    msg = _("Supplied %(attr)s (%(supplied)s) and "
                            "%(attr)s generated from uploaded image "
                            "(%(actual)s) did not match. Setting image "
                            "status to 'killed'.") % locals()
                    LOG.error(msg)
                    self._safe_kill(req, image_id)
                    self._initiate_deletion(req, location, image_id)
                    raise HTTPBadRequest(explanation=msg,
                                         content_type="text/plain",
                                         request=req)

            # Verify any supplied size/checksum value matches size/checksum
            # returned from store when adding image
            _kill_mismatched(image_meta, 'size', size)
            _kill_mismatched(image_meta, 'checksum', checksum)

            # Update the database with the checksum returned
            # from the backend store
            LOG.debug(_("Updating image %(image_id)s data. "
                      "Checksum set to %(checksum)s, size set "
                      "to %(size)d"), locals())
            update_data = {'checksum': checksum,
                           'size': size}
            image_meta = registry.update_image_metadata(req.context,
                                                        image_id,
                                                        update_data)
            self.notifier.info('image.upload', redact_loc(image_meta))

            return location

        except exception.Duplicate as e:
            msg = _("Attempt to upload duplicate image: %s") % e
            LOG.debug(msg)
            self._safe_kill(req, image_id)
            raise HTTPConflict(explanation=msg, request=req)

        except exception.Forbidden as e:
            msg = _("Forbidden upload attempt: %s") % e
            LOG.debug(msg)
            self._safe_kill(req, image_id)
            raise HTTPForbidden(explanation=msg,
                                request=req,
                                content_type="text/plain")

        except exception.StorageFull as e:
            msg = _("Image storage media is full: %s") % e
            LOG.error(msg)
            self._safe_kill(req, image_id)
            self.notifier.error('image.upload', msg)
            raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
                                            content_type='text/plain')

        except exception.StorageWriteDenied as e:
            msg = _("Insufficient permissions on image storage media: %s") % e
            LOG.error(msg)
            self._safe_kill(req, image_id)
            self.notifier.error('image.upload', msg)
            raise HTTPServiceUnavailable(explanation=msg, request=req,
                                         content_type='text/plain')

        except exception.ImageSizeLimitExceeded as e:
            msg = _("Denying attempt to upload image larger than %d bytes."
                    % CONF.image_size_cap)
            LOG.info(msg)
            self._safe_kill(req, image_id)
            raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
                                            content_type='text/plain')

        except HTTPError as e:
            self._safe_kill(req, image_id)
            #NOTE(bcwaldon): Ideally, we would just call 'raise' here,
            # but something in the above function calls is affecting the
            # exception context and we must explicitly re-raise the
            # caught exception.
            raise e

        except Exception as e:
            LOG.exception(_("Failed to upload image"))
            self._safe_kill(req, image_id)
            raise HTTPInternalServerError(request=req)
Пример #5
0
        except exception.Forbidden, e:
            msg = _("Forbidden upload attempt: %s") % e
            LOG.error(msg)
            self._safe_kill(req, image_id)
            self.notifier.error('image.upload', msg)
            raise HTTPForbidden(explanation=msg,
                                request=req,
                                content_type="text/plain")

        except exception.StorageFull, e:
            msg = _("Image storage media is full: %s") % e
            LOG.error(msg)
            self._safe_kill(req, image_id)
            self.notifier.error('image.upload', msg)
            raise HTTPRequestEntityTooLarge(explanation=msg,
                                            request=req,
                                            content_type='text/plain')

        except exception.StorageWriteDenied, e:
            msg = _("Insufficient permissions on image storage media: %s") % e
            LOG.error(msg)
            self._safe_kill(req, image_id)
            self.notifier.error('image.upload', msg)
            raise HTTPServiceUnavailable(explanation=msg,
                                         request=req,
                                         content_type='text/plain')

        except exception.ImageSizeLimitExceeded, e:
            msg = _("Denying attempt to upload image larger than %d.")
            self._safe_kill(req, image_id)
            raise HTTPBadRequest(explanation=msg % CONF.image_size_cap,
Пример #6
0
    def __call__(self):
        """
        :return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure
        """
        if self.headers.has_key('content-length'):
            if int(self.headers['content-length']) >= MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=self.req)

        parsed = urlparse(self.url)
        if self.proxy:
            proxy_parsed = urlparse(self.proxy)

        if self._proxy_request_check(parsed.path):
            host, port = self.split_netloc(proxy_parsed)
            path = self.url
            ssl = True if proxy_parsed.scheme == 'https' else False
        else:
            host, port = self.split_netloc(parsed)
            path = parsed.path
            ssl = True if parsed.scheme == 'https' else False
        self.headers['host'] = '%s:%s' % (host, port)

        if self.method == 'PUT' and len(parsed.path.split('/')) >= 5:
            if self.headers.has_key('content-length') and int(
                    self.headers['content-length']) != 0:
                if not self.headers.has_key('expect'):
                    self.headers['expect'] = '100-continue'
            chunked = self.req.headers.get('transfer-encoding')
            if isinstance(self.req.environ['wsgi.input'], str):
                reader = self.req.environ['wsgi.input'].read
                data_source = iter(lambda: reader(self.chunk_size), '')
            else:
                data_source = self.req.environ['wsgi.input']
            bytes_transferred = 0
            try:
                conn = self._connect_put_node(host,
                                              port,
                                              self.method,
                                              path,
                                              headers=self.headers,
                                              query_string=parsed.query,
                                              ssl=ssl)
                if not conn:
                    return HTTPServiceUnavailable(request=self.req)
                with ContextPool(1) as pool:
                    conn.failed = False
                    conn.queue = Queue(10)
                    pool.spawn(self._send_file, conn, path)
                    while True:
                        with ChunkReadTimeout(self.client_timeout):
                            try:
                                chunk = next(data_source)
                            except StopIteration:
                                if chunked:
                                    conn.queue.put('0\r\n\r\n')
                                break
                            except TypeError, err:
                                self.logger.info('Chunk Read Error: %s' % err)
                                break
                            except Exception, err:
                                self.logger.info('Chunk Read Error: %s' % err)
                                return HTTPServerError(request=self.req)
                        bytes_transferred += len(chunk)
                        if bytes_transferred > MAX_FILE_SIZE:
                            return HTTPRequestEntityTooLarge(request=self.req)
                        if not conn.failed:
                            conn.queue.put('%x\r\n%s\r\n' %
                                           (len(chunk),
                                            chunk) if chunked else chunk)
Пример #7
0
    def __call__(self):
        """
        :return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure
        """
        if self.headers.has_key('content-length'):
            if int(self.headers['content-length']) >= MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=self.req)

        parsed = urlparse(self.url)
        if self.proxy:
            proxy_parsed = urlparse(self.proxy)

        if self._proxy_request_check(parsed.path):
            host, port = self.split_netloc(proxy_parsed)
            path = self.url
        else:
            host, port = self.split_netloc(parsed)
            path = parsed.path
        self.headers['host'] = '%s:%s' % (host, port)

        if self.method == 'PUT' and len(parsed.path.split('/')) == 5:
            chunked = self.req.headers.get('transfer-encoding')
            reader = self.req.environ['wsgi.input'].read
            data_source = iter(lambda: reader(self.chunk_size), '')
            bytes_transferred = 0
            # pile = GreenPile()
            # pile.spawn(self._connect_server, host, port, self.method, path, self.headers, parsed.query)
            # conns = [conn for conn in pile if conn]
            # conn = conns[0]
            try:
                with ConnectionTimeout(self.conn_timeout):
                    conn = http_connect_raw(host,
                                            port,
                                            self.method,
                                            path,
                                            headers=self.headers,
                                            query_string=parsed.query)
                with ContextPool(1) as pool:
                    conn.failed = False
                    conn.queue = Queue(10)
                    pool.spawn(self._send_file, conn, path)
                    while True:
                        with ChunkReadTimeout(self.client_timeout):
                            try:
                                chunk = next(data_source)
                            except StopIteration:
                                if chunked:
                                    conn.queue.put('0\r\n\r\n')
                                break
                            except TypeError, err:
                                self.logger.info('Chunk Read Error: %s' % err)
                                break
                            except Exception, err:
                                self.logger.info('Chunk Read Error: %s' % err)
                                return HTTPServerError(request=self.req)
                        bytes_transferred += len(chunk)
                        if bytes_transferred > MAX_FILE_SIZE:
                            return HTTPRequestEntityTooLarge(request=self.req)
                        if not conn.failed:
                            conn.queue.put('%x\r\n%s\r\n' %
                                           (len(chunk),
                                            chunk) if chunked else chunk)
Пример #8
0
    def PUT(self, req):
        """HTTP PUT request handler."""
        (container_partition, containers, _junk, req.acl,
         req.environ['swift_sync_key'], object_versions) = \
            self.container_info(self.account_name, self.container_name,
                account_autocreate=self.app.account_autocreate)
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not containers:
            return HTTPNotFound(request=req)
        if 'x-delete-after' in req.headers:
            try:
                x_delete_after = int(req.headers['x-delete-after'])
            except ValueError:
                return HTTPBadRequest(request=req,
                                      content_type='text/plain',
                                      body='Non-integer X-Delete-After')
            req.headers['x-delete-at'] = '%d' % (time.time() + x_delete_after)
        if 'x-delete-at' in req.headers:
            try:
                x_delete_at = int(req.headers['x-delete-at'])
                if x_delete_at < time.time():
                    return HTTPBadRequest(body='X-Delete-At in past',
                                          request=req,
                                          content_type='text/plain')
            except ValueError:
                return HTTPBadRequest(request=req,
                                      content_type='text/plain',
                                      body='Non-integer X-Delete-At')
            delete_at_container = str(
                x_delete_at / self.app.expiring_objects_container_divisor *
                self.app.expiring_objects_container_divisor)
            delete_at_part, delete_at_nodes = \
                self.app.container_ring.get_nodes(
                    self.app.expiring_objects_account, delete_at_container)
        else:
            delete_at_part = delete_at_nodes = None
        partition, nodes = self.app.object_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)
        # do a HEAD request for container sync and checking object versions
        if 'x-timestamp' in req.headers or (
                object_versions
                and not req.environ.get('swift_versioned_copy')):
            hreq = Request.blank(req.path_info,
                                 headers={'X-Newest': 'True'},
                                 environ={'REQUEST_METHOD': 'HEAD'})
            hresp = self.GETorHEAD_base(hreq, _('Object'), partition, nodes,
                                        hreq.path_info, len(nodes))
        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req.headers['X-Timestamp'] = \
                    normalize_timestamp(float(req.headers['x-timestamp']))
                if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
                    float(hresp.environ['swift_x_timestamp']) >= \
                        float(req.headers['x-timestamp']):
                    return HTTPAccepted(request=req)
            except ValueError:
                return HTTPBadRequest(
                    request=req,
                    content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                    'was %r' % req.headers['x-timestamp'])
        else:
            req.headers['X-Timestamp'] = normalize_timestamp(time.time())
        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        if not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                                                'application/octet-stream'
            content_type_manually_set = False
        error_response = check_object_creation(req, self.object_name)
        if error_response:
            return error_response
        if object_versions and not req.environ.get('swift_versioned_copy'):
            is_manifest = 'x-object-manifest' in req.headers or \
                          'x-object-manifest' in hresp.headers
            if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
                # This is a version manifest and needs to be handled
                # differently. First copy the existing data to a new object,
                # then write the data from this request to the version manifest
                # object.
                lcontainer = object_versions.split('/')[0]
                prefix_len = '%03x' % len(self.object_name)
                lprefix = prefix_len + self.object_name + '/'
                ts_source = hresp.environ.get('swift_x_timestamp')
                if ts_source is None:
                    ts_source = time.mktime(
                        time.strptime(hresp.headers['last-modified'],
                                      '%a, %d %b %Y %H:%M:%S GMT'))
                new_ts = normalize_timestamp(ts_source)
                vers_obj_name = lprefix + new_ts
                copy_headers = {
                    'Destination': '%s/%s' % (lcontainer, vers_obj_name)
                }
                copy_environ = {
                    'REQUEST_METHOD': 'COPY',
                    'swift_versioned_copy': True
                }
                copy_req = Request.blank(req.path_info,
                                         headers=copy_headers,
                                         environ=copy_environ)
                copy_resp = self.COPY(copy_req)
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

        reader = req.environ['wsgi.input'].read
        data_source = iter(lambda: reader(self.app.client_chunk_size), '')
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            source_header = unquote(source_header)
            acct = req.path_info.split('/', 2)[1]
            if isinstance(acct, unicode):
                acct = acct.encode('utf-8')
            if not source_header.startswith('/'):
                source_header = '/' + source_header
            source_header = '/' + acct + source_header
            try:
                src_container_name, src_obj_name = \
                    source_header.split('/', 3)[2:]
            except ValueError:
                return HTTPPreconditionFailed(
                    request=req,
                    body='X-Copy-From header must be of the form'
                    '<container name>/<object name>')
            source_req = req.copy_get()
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            source_resp = self.GET(source_req)
            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            new_req = Request.blank(req.path_info,
                                    environ=req.environ,
                                    headers=req.headers)
            data_source = source_resp.app_iter
            new_req.content_length = source_resp.content_length
            if new_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            new_req.etag = source_resp.etag
            # we no longer need the X-Copy-From header
            del new_req.headers['X-Copy-From']
            if not content_type_manually_set:
                new_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if new_req.headers.get('x-fresh-metadata', 'false').lower() \
                    not in TRUE_VALUES:
                for k, v in source_resp.headers.items():
                    if k.lower().startswith('x-object-meta-'):
                        new_req.headers[k] = v
                for k, v in req.headers.items():
                    if k.lower().startswith('x-object-meta-'):
                        new_req.headers[k] = v
            req = new_req
        node_iter = self.iter_nodes(partition, nodes, self.app.object_ring)
        pile = GreenPile(len(nodes))
        for container in containers:
            nheaders = dict(req.headers.iteritems())
            nheaders['Connection'] = 'close'
            nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container
            nheaders['X-Container-Partition'] = container_partition
            nheaders['X-Container-Device'] = container['device']
            nheaders['Expect'] = '100-continue'
            if delete_at_nodes:
                node = delete_at_nodes.pop(0)
                nheaders['X-Delete-At-Host'] = '%(ip)s:%(port)s' % node
                nheaders['X-Delete-At-Partition'] = delete_at_part
                nheaders['X-Delete-At-Device'] = node['device']
            pile.spawn(self._connect_put_node, node_iter, partition,
                       req.path_info, nheaders, self.app.logger.thread_locals)
        conns = [conn for conn in pile if conn]
        if len(conns) <= len(nodes) / 2:
            self.app.logger.error(
                _('Object PUT returning 503, %(conns)s/%(nodes)s '
                  'required connections'), {
                      'conns': len(conns),
                      'nodes': len(nodes) // 2 + 1
                  })
            return HTTPServiceUnavailable(request=req)
        chunked = req.headers.get('transfer-encoding')
        bytes_transferred = 0
        try:
            with ContextPool(len(nodes)) as pool:
                for conn in conns:
                    conn.failed = False
                    conn.queue = Queue(self.app.put_queue_depth)
                    pool.spawn(self._send_file, conn, req.path)
                while True:
                    with ChunkReadTimeout(self.app.client_timeout):
                        try:
                            chunk = next(data_source)
                        except StopIteration:
                            if chunked:
                                [conn.queue.put('0\r\n\r\n') for conn in conns]
                            break
                    bytes_transferred += len(chunk)
                    if bytes_transferred > MAX_FILE_SIZE:
                        return HTTPRequestEntityTooLarge(request=req)
                    for conn in list(conns):
                        if not conn.failed:
                            conn.queue.put('%x\r\n%s\r\n' %
                                           (len(chunk),
                                            chunk) if chunked else chunk)
                        else:
                            conns.remove(conn)
                    if len(conns) <= len(nodes) / 2:
                        self.app.logger.error(
                            _('Object PUT exceptions during'
                              ' send, %(conns)s/%(nodes)s required connections'
                              ), {
                                  'conns': len(conns),
                                  'nodes': len(nodes) / 2 + 1
                              })
                        return HTTPServiceUnavailable(request=req)
                for conn in conns:
                    if conn.queue.unfinished_tasks:
                        conn.queue.join()
            conns = [conn for conn in conns if not conn.failed]
        except ChunkReadTimeout, err:
            self.app.logger.warn(_('ERROR Client read timeout (%ss)'),
                                 err.seconds)
            self.app.logger.increment('client_timeouts')
            return HTTPRequestTimeout(request=req)