예제 #1
0
 def _if_none_match_wrapper(self, req, *args, **kwargs):
     if req.if_none_match is None:
         return fnc(self, req, *args, **kwargs)
     oio_headers = {'X-oio-req-id': self.trans_id}
     try:
         metadata = self.app.storage.object_show(self.account_name,
                                                 self.container_name,
                                                 self.object_name,
                                                 version=req.environ.get(
                                                     'oio.query',
                                                     {}).get('version'),
                                                 headers=oio_headers)
     except (NoSuchObject, NoSuchContainer):
         return fnc(self, req, *args, **kwargs)
     # req.if_none_match will check for '*'.
     if metadata.get('hash') in req.if_none_match:
         if req.method in ('HEAD', 'GET'):
             raise HTTPNotModified(request=req)
         else:
             raise HTTPPreconditionFailed(request=req)
     return fnc(self, req, *args, **kwargs)
예제 #2
0
    def __call__(self, env, start_response):
        """
        WSGI entry point.
        Wraps env in swob.Request object and passes it down.

        :param env: WSGI environment dictionary
        :param start_response: WSGI callable
        """
        try:
            if self.memcache is None:
                self.memcache = cache_from_env(env, True)
            req = self.update_request(Request(env))
            return self.handle_request(req)(env, start_response)
        except UnicodeError:
            err = HTTPPreconditionFailed(
                request=req, body='Invalid UTF8 or contains NULL')
            return err(env, start_response)
        except (Exception, Timeout):
            start_response('500 Server Error',
                           [('Content-Type', 'text/plain')])
            return ['Internal server error.\n']
예제 #3
0
def check_path_header(req, name, length, error_msg):
    """
    Validate that the value of path-like header is
    well formatted. We assume the caller ensures that
    specific header is present in req.headers.

    :param req: HTTP request object
    :param name: header name
    :param length: length of path segment check
    :param error_msg: error message for client
    :returns: A tuple with path parts according to length
    :raise: HTTPPreconditionFailed if header value
            is not well formatted.
    """
    hdr = wsgi_unquote(req.headers.get(name))
    if not hdr.startswith('/'):
        hdr = '/' + hdr
    try:
        return split_path(hdr, length, length, True)
    except ValueError:
        raise HTTPPreconditionFailed(request=req, body=error_msg)
예제 #4
0
    def handle_multipart_delete(self, req):
        """
        Will delete all the segments in the SLO manifest and then, if
        successful, will delete the manifest file.
        :params req: a swob.Request with an obj in path
        :raises HTTPServerError: on invalid manifest
        :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
        """
        if not check_utf8(req.path_info):
            raise HTTPPreconditionFailed(
                request=req, body='Invalid UTF8 or contains NULL')

        resp = HTTPOk(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if out_content_type:
            resp.content_type = out_content_type
        resp.app_iter = self.bulk_deleter.handle_delete_iter(
            req, objs_to_delete=self.get_segments_to_delete_iter(req),
            user_agent='MultipartDELETE', swift_source='SLO',
            out_content_type=out_content_type)
        return resp
예제 #5
0
파일: server.py 프로젝트: waiterZen/swift
 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body='Invalid UTF8')
     else:
         try:
             # disallow methods which are not publicly accessible
             try:
                 method = getattr(self, req.method)
                 getattr(method, 'publicly_accessible')
             except AttributeError:
                 res = HTTPMethodNotAllowed()
             else:
                 res = method(req)
         except (Exception, Timeout):
             self.logger.exception(_('ERROR __call__ error with %(method)s'
                                     ' %(path)s '),
                                   {'method': req.method, 'path': req.path})
             res = HTTPInternalServerError(body=traceback.format_exc())
     trans_time = '%.4f' % (time.time() - start_time)
     additional_info = ''
     if res.headers.get('x-container-timestamp') is not None:
         additional_info += 'x-container-timestamp: %s' % \
             res.headers['x-container-timestamp']
     log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % (
         req.remote_addr,
         time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()),
         req.method, req.path,
         res.status.split()[0], res.content_length or '-',
         req.headers.get('x-trans-id', '-'),
         req.referer or '-', req.user_agent or '-',
         trans_time,
         additional_info)
     if req.method.upper() == 'REPLICATE':
         self.logger.debug(log_message)
     else:
         self.logger.info(log_message)
     return res(env, start_response)
예제 #6
0
 def DELETE(self, request):
     """Handle HTTP DELETE requests for the Swift Object Server."""
     device, partition, account, container, obj = \
         split_and_validate_path(request, 5, 5, True)
     if 'x-timestamp' not in request.headers or \
             not check_float(request.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing timestamp',
                               request=request,
                               content_type='text/plain')
     try:
         disk_file = self._diskfile(device, partition, account, container,
                                    obj)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     if 'x-if-delete-at' in request.headers and \
             int(request.headers['x-if-delete-at']) != \
             int(disk_file.metadata.get('X-Delete-At') or 0):
         return HTTPPreconditionFailed(
             request=request,
             body='X-If-Delete-At and X-Delete-At do not match')
     old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
     if old_delete_at:
         self.delete_at_update('DELETE', old_delete_at, account, container,
                               obj, request, device)
     orig_timestamp = disk_file.metadata.get('X-Timestamp', 0)
     req_timestamp = request.headers['X-Timestamp']
     if disk_file.is_deleted() or disk_file.is_expired():
         response_class = HTTPNotFound
     else:
         if orig_timestamp < req_timestamp:
             response_class = HTTPNoContent
         else:
             response_class = HTTPConflict
     if orig_timestamp < req_timestamp:
         disk_file.delete(req_timestamp)
         self.container_update(
             'DELETE', account, container, obj, request,
             HeaderKeyDict({'x-timestamp': req_timestamp}), device)
     resp = response_class(request=request)
     return resp
예제 #7
0
def account_listing_bucket_response(account,
                                    req,
                                    response_content_type,
                                    listing=None):
    if response_content_type != 'application/json':
        # AWS S3 is always call wit format=json
        # check method GET in ServiceController (swift3/controllers/service.py)
        return HTTPPreconditionFailed(body='Invalid content type')

    data = []
    for entry in listing:
        data.append({
            'name': entry['name'],
            'count': entry['objects'],
            'bytes': entry['bytes'],
            'last_modified': Timestamp(entry['mtime']).isoformat
        })
    account_list = json.dumps(data, encoding="utf-8")
    ret = HTTPOk(body=account_list, request=req, headers={})
    ret.content_type = response_content_type
    ret.charset = 'utf-8'
    return ret
예제 #8
0
 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
     else:
         try:
             # disallow methods which are not publicly accessible
             try:
                 if req.method not in self.allowed_methods:
                     raise AttributeError('Not allowed method.')
             except AttributeError:
                 res = HTTPMethodNotAllowed()
             else:
                 method = getattr(self, req.method)
                 res = method(req)
         except HTTPException as error_response:
             res = error_response
         except (Exception, Timeout):
             self.logger.exception(
                 _('ERROR __call__ error with %(method)s'
                   ' %(path)s '), {
                       'method': req.method,
                       'path': req.path
                   })
             res = HTTPInternalServerError(body=traceback.format_exc())
     if self.log_requests:
         trans_time = time.time() - start_time
         additional_info = ''
         if res.headers.get('x-container-timestamp') is not None:
             additional_info += 'x-container-timestamp: %s' % \
                 res.headers['x-container-timestamp']
         log_msg = get_log_line(req, res, trans_time, additional_info)
         if req.method.upper() == 'REPLICATE':
             self.logger.debug(log_msg)
         else:
             self.logger.info(log_msg)
     return res(env, start_response)
예제 #9
0
    def get_segments_to_delete_iter(self, req):
        """
        A generator function to be used to delete all the segments and
        sub-segments referenced in a manifest.

        :params req: a swob.Request with an SLO manifest in path
        :raises HTTPPreconditionFailed: on invalid UTF8 in request path
        :raises HTTPBadRequest: on too many buffered sub segments and
                                on invalid SLO manifest path
        """
        if not check_utf8(req.path_info):
            raise HTTPPreconditionFailed(
                request=req, body='Invalid UTF8 or contains NULL')
        vrs, account, container, obj = req.split_path(4, 4, True)

        segments = [{
            'sub_slo': True,
            'name': ('/%s/%s' % (container, obj)).decode('utf-8')}]
        while segments:
            if len(segments) > MAX_BUFFERED_SLO_SEGMENTS:
                raise HTTPBadRequest(
                    'Too many buffered slo segments to delete.')
            seg_data = segments.pop(0)
            if seg_data.get('sub_slo'):
                try:
                    segments.extend(
                        self.get_slo_segments(seg_data['name'], req))
                except HTTPException as err:
                    # allow bulk delete response to report errors
                    seg_data['error'] = {'code': err.status_int,
                                         'message': err.body}

                # add manifest back to be deleted after segments
                seg_data['sub_slo'] = False
                segments.append(seg_data)
            else:
                seg_data['name'] = seg_data['name'].encode('utf-8')
                yield seg_data
예제 #10
0
def check_path_header(req, name, length, error_msg):
    # FIXME: replace swift.common.constraints check_path_header
    #        when swift3 supports swift 2.2 or later
    """
    Validate that the value of path-like header is
    well formatted. We assume the caller ensures that
    specific header is present in req.headers.

    :param req: HTTP request object
    :param name: header name
    :param length: length of path segment check
    :param error_msg: error message for client
    :returns: A tuple with path parts according to length
    :raise: HTTPPreconditionFailed if header value
            is not well formatted.
    """
    src_header = unquote(req.headers.get(name))
    if not src_header.startswith('/'):
        src_header = '/' + src_header
    try:
        return utils.split_path(src_header, length, length, True)
    except ValueError:
        raise HTTPPreconditionFailed(request=req, body=error_msg)
예제 #11
0
파일: server.py 프로젝트: kvite/swift
 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
     else:
         try:
             # disallow methods which have not been marked 'public'
             try:
                 method = getattr(self, req.method)
                 getattr(method, 'publicly_accessible')
                 replication_method = getattr(method, 'replication', False)
                 if (self.replication_server is not None
                         and self.replication_server != replication_method):
                     raise AttributeError('Not allowed method.')
             except AttributeError:
                 res = HTTPMethodNotAllowed()
             else:
                 res = method(req)
         except HTTPException as error_response:
             res = error_response
         except (Exception, Timeout):
             self.logger.exception(
                 _('ERROR __call__ error with %(method)s %(path)s '), {
                     'method': req.method,
                     'path': req.path
                 })
             res = HTTPInternalServerError(body=traceback.format_exc())
     if self.log_requests:
         trans_time = time.time() - start_time
         log_message = get_log_line(req, res, trans_time, '')
         if req.method.upper() == 'REPLICATE':
             self.logger.debug(log_message)
         else:
             self.logger.info(log_message)
     return res(env, start_response)
예제 #12
0
    def COPY(self):
        """
        COPY handler on Proxy
        """
        if not self.request.headers.get('Destination'):
            return HTTPPreconditionFailed(request=self.request,
                                          body='Destination header required')

        params = self.verify_access_to_storlet()
        self.augment_storlet_request(params)
        self._validate_copy_request()
        dest_container, dest_object = check_destination_header(self.request)

        # re-write the existing request as a PUT instead of creating a new one
        # TODO(eranr): do we want a new sub_request or re-write existing one as
        # we do below. See proxy obj controller COPY.
        self.request.method = 'PUT'
        self.request.path_info = '/v1/%s/%s/%s' % \
                                 (self.account, dest_container, dest_object)
        self.request.headers['Content-Length'] = 0
        del self.request.headers['Destination']

        return self.base_handle_copy_request(self.container, self.obj,
                                             dest_container, dest_object)
예제 #13
0
 def GET(self, req):
     """Handle HTTP GET request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     path = get_param(req, 'path')
     prefix = get_param(req, 'prefix')
     delimiter = get_param(req, 'delimiter')
     if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
         # delimiters can be made more flexible later
         return HTTPPreconditionFailed(body='Bad delimiter')
     marker = get_param(req, 'marker', '')
     end_marker = get_param(req, 'end_marker')
     limit = CONTAINER_LISTING_LIMIT
     given_limit = get_param(req, 'limit')
     if given_limit and given_limit.isdigit():
         limit = int(given_limit)
         if limit > CONTAINER_LISTING_LIMIT:
             return HTTPPreconditionFailed(request=req,
                                           body='Maximum limit is %d' %
                                           CONTAINER_LISTING_LIMIT)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive,
                                         part,
                                         account,
                                         container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     resp_headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     for key, (value, timestamp) in broker.metadata.iteritems():
         if value and (key.lower() in self.save_headers
                       or is_sys_or_user_meta('container', key)):
             resp_headers[key] = value
     ret = Response(request=req,
                    headers=resp_headers,
                    content_type=out_content_type,
                    charset='utf-8')
     container_list = broker.list_objects_iter(limit, marker, end_marker,
                                               prefix, delimiter, path)
     if out_content_type == 'application/json':
         ret.body = json.dumps(
             [self.update_data_record(record) for record in container_list])
     elif out_content_type.endswith('/xml'):
         doc = Element('container', name=container.decode('utf-8'))
         for obj in container_list:
             record = self.update_data_record(obj)
             if 'subdir' in record:
                 name = record['subdir'].decode('utf-8')
                 sub = SubElement(doc, 'subdir', name=name)
                 SubElement(sub, 'name').text = name
             else:
                 obj_element = SubElement(doc, 'object')
                 for field in [
                         "name", "hash", "bytes", "content_type",
                         "last_modified"
                 ]:
                     SubElement(obj_element, field).text = str(
                         record.pop(field)).decode('utf-8')
                 for field in sorted(record):
                     SubElement(obj_element, field).text = str(
                         record[field]).decode('utf-8')
         ret.body = tostring(doc, encoding='UTF-8').replace(
             "<?xml version='1.0' encoding='UTF-8'?>",
             '<?xml version="1.0" encoding="UTF-8"?>', 1)
     else:
         if not container_list:
             return HTTPNoContent(request=req, headers=resp_headers)
         ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
     return ret
예제 #14
0
 def GET(self, request):
     """Handle HTTP GET requests for the Swift Object Server."""
     device, partition, account, container, obj = \
         split_and_validate_path(request, 5, 5, True)
     try:
         disk_file = self._diskfile(device,
                                    partition,
                                    account,
                                    container,
                                    obj,
                                    keep_data_fp=True,
                                    iter_hook=sleep)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     if disk_file.is_deleted() or disk_file.is_expired():
         if request.headers.get('if-match') == '*':
             return HTTPPreconditionFailed(request=request)
         else:
             return HTTPNotFound(request=request)
     try:
         file_size = disk_file.get_data_file_size()
     except (DiskFileError, DiskFileNotExist):
         disk_file.quarantine()
         return HTTPNotFound(request=request)
     if request.headers.get('if-match') not in (None, '*') and \
             disk_file.metadata['ETag'] not in request.if_match:
         disk_file.close()
         return HTTPPreconditionFailed(request=request)
     if request.headers.get('if-none-match') is not None:
         if disk_file.metadata['ETag'] in request.if_none_match:
             resp = HTTPNotModified(request=request)
             resp.etag = disk_file.metadata['ETag']
             disk_file.close()
             return resp
     try:
         if_unmodified_since = request.if_unmodified_since
     except (OverflowError, ValueError):
         # catches timestamps before the epoch
         return HTTPPreconditionFailed(request=request)
     if if_unmodified_since and \
             datetime.fromtimestamp(
                 float(disk_file.metadata['X-Timestamp']), UTC) > \
             if_unmodified_since:
         disk_file.close()
         return HTTPPreconditionFailed(request=request)
     try:
         if_modified_since = request.if_modified_since
     except (OverflowError, ValueError):
         # catches timestamps before the epoch
         return HTTPPreconditionFailed(request=request)
     if if_modified_since and \
             datetime.fromtimestamp(
                 float(disk_file.metadata['X-Timestamp']), UTC) < \
             if_modified_since:
         disk_file.close()
         return HTTPNotModified(request=request)
     response = Response(app_iter=disk_file,
                         request=request,
                         conditional_response=True)
     response.headers['Content-Type'] = disk_file.metadata.get(
         'Content-Type', 'application/octet-stream')
     for key, value in disk_file.metadata.iteritems():
         if key.lower().startswith('x-object-meta-') or \
                 key.lower() in self.allowed_headers:
             response.headers[key] = value
     response.etag = disk_file.metadata['ETag']
     response.last_modified = float(disk_file.metadata['X-Timestamp'])
     response.content_length = file_size
     if response.content_length < self.keep_cache_size and \
             (self.keep_cache_private or
              ('X-Auth-Token' not in request.headers and
               'X-Storage-Token' not in request.headers)):
         disk_file.keep_cache = True
     if 'Content-Encoding' in disk_file.metadata:
         response.content_encoding = disk_file.metadata['Content-Encoding']
     response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']
     return request.get_response(response)
예제 #15
0
    def handle_obj_versions_delete(self, req, object_versions, account_name,
                                   container_name, object_name):
        lcontainer = object_versions.split('/')[0]
        prefix_len = '%03x' % len(object_name)
        lprefix = prefix_len + object_name + '/'

        item_iter = self._listing_iter(account_name, lcontainer, lprefix, req)

        authed = False
        for previous_version in item_iter:
            if not authed:
                # we're about to start making COPY requests - need to
                # validate the write access to the versioned container
                if 'swift.authorize' in req.environ:
                    container_info = get_container_info(req.environ, self.app)
                    req.acl = container_info.get('write_acl')
                    aresp = req.environ['swift.authorize'](req)
                    if aresp:
                        return aresp
                    authed = True

            # there are older versions so copy the previous version to the
            # current object and delete the previous version
            prev_obj_name = previous_version['name'].encode('utf-8')

            copy_path = '/v1/' + account_name + '/' + \
                        lcontainer + '/' + prev_obj_name

            copy_headers = {
                'X-Newest': 'True',
                'Destination': container_name + '/' + object_name,
                'x-auth-token': req.headers.get('x-auth-token')
            }

            copy_req = make_pre_authed_request(req.environ,
                                               path=copy_path,
                                               headers=copy_headers,
                                               method='COPY',
                                               swift_source='VW')
            copy_resp = copy_req.get_response(self.app)

            # if the version isn't there, keep trying with previous version
            if copy_resp.status_int == HTTP_NOT_FOUND:
                continue

            if not is_success(copy_resp.status_int):
                if is_client_error(copy_resp.status_int):
                    # some user error, maybe permissions
                    return HTTPPreconditionFailed(request=req)
                else:
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

            # reset these because the COPY changed them
            new_del_req = make_pre_authed_request(req.environ,
                                                  path=copy_path,
                                                  method='DELETE',
                                                  swift_source='VW')
            req = new_del_req

            # remove 'X-If-Delete-At', since it is not for the older copy
            if 'X-If-Delete-At' in req.headers:
                del req.headers['X-If-Delete-At']
            break

        # handle DELETE request here in case it was modified
        return req.get_response(self.app)
예제 #16
0
파일: server.py 프로젝트: eckeman/swift
    def __call__(self, env, start_response):
        """WSGI Application entry point for the Swift Object Server."""
        start_time = time.time()
        req = Request(env)
        self.logger.txn_id = req.headers.get('x-trans-id', None)

        if not check_utf8(req.path_info):
            res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
        else:
            try:
                # disallow methods which have not been marked 'public'
                try:
                    if req.method not in self.allowed_methods:
                        raise AttributeError('Not allowed method.')
                except AttributeError:
                    res = HTTPMethodNotAllowed()
                else:
                    method = getattr(self, req.method)
                    res = method(req)
            except DiskFileCollision:
                res = HTTPForbidden(request=req)
            except HTTPException as error_response:
                res = error_response
            except (Exception, Timeout):
                self.logger.exception(
                    _('ERROR __call__ error with %(method)s'
                      ' %(path)s '), {
                          'method': req.method,
                          'path': req.path
                      })
                res = HTTPInternalServerError(body=traceback.format_exc())
        trans_time = time.time() - start_time
        if self.log_requests:
            log_line = get_log_line(req, res, trans_time, '')
            if req.method in ('REPLICATE', 'REPLICATION') or \
                    'X-Backend-Replication' in req.headers:
                self.logger.debug(log_line)
            else:
                self.logger.info(log_line)
        if req.method in ('PUT', 'DELETE'):
            slow = self.slow - trans_time
            if slow > 0:
                sleep(slow)

        # To be able to zero-copy send the object, we need a few things.
        # First, we have to be responding successfully to a GET, or else we're
        # not sending the object. Second, we have to be able to extract the
        # socket file descriptor from the WSGI input object. Third, the
        # diskfile has to support zero-copy send.
        #
        # There's a good chance that this could work for 206 responses too,
        # but the common case is sending the whole object, so we'll start
        # there.
        if req.method == 'GET' and res.status_int == 200 and \
           isinstance(env['wsgi.input'], wsgi.Input):
            app_iter = getattr(res, 'app_iter', None)
            checker = getattr(app_iter, 'can_zero_copy_send', None)
            if checker and checker():
                # For any kind of zero-copy thing like sendfile or splice, we
                # need the file descriptor. Eventlet doesn't provide a clean
                # way of getting that, so we resort to this.
                wsock = env['wsgi.input'].get_socket()
                wsockfd = wsock.fileno()

                # Don't call zero_copy_send() until after we force the HTTP
                # headers out of Eventlet and into the socket.
                def zero_copy_iter():
                    # If possible, set TCP_CORK so that headers don't
                    # immediately go on the wire, but instead, wait for some
                    # response body to make the TCP frames as large as
                    # possible (and hence as few packets as possible).
                    #
                    # On non-Linux systems, we might consider TCP_NODELAY, but
                    # since the only known zero-copy-capable diskfile uses
                    # Linux-specific syscalls, we'll defer that work until
                    # someone needs it.
                    if hasattr(socket, 'TCP_CORK'):
                        wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK,
                                         1)
                    yield EventletPlungerString()
                    try:
                        app_iter.zero_copy_send(wsockfd)
                    except Exception:
                        self.logger.exception("zero_copy_send() blew up")
                        raise
                    yield ''

                # Get headers ready to go out
                res(env, start_response)
                return zero_copy_iter()
            else:
                return res(env, start_response)
        else:
            return res(env, start_response)
예제 #17
0
파일: server.py 프로젝트: waiterZen/swift
 resp_headers = {
     'X-Account-Container-Count': info['container_count'],
     'X-Account-Object-Count': info['object_count'],
     'X-Account-Bytes-Used': info['bytes_used'],
     'X-Timestamp': info['created_at'],
     'X-PUT-Timestamp': info['put_timestamp']}
 resp_headers.update((key, value)
                     for key, (value, timestamp) in
                     broker.metadata.iteritems() if value != '')
 try:
     prefix = get_param(req, 'prefix')
     delimiter = get_param(req, 'delimiter')
     if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
         # delimiters can be made more flexible later
         self.logger.increment('GET.errors')
         return HTTPPreconditionFailed(body='Bad delimiter')
     limit = ACCOUNT_LISTING_LIMIT
     given_limit = get_param(req, 'limit')
     if given_limit and given_limit.isdigit():
         limit = int(given_limit)
         if limit > ACCOUNT_LISTING_LIMIT:
             self.logger.increment('GET.errors')
             return HTTPPreconditionFailed(request=req,
                                           body='Maximum limit is %d' %
                                           ACCOUNT_LISTING_LIMIT)
     marker = get_param(req, 'marker', '')
     end_marker = get_param(req, 'end_marker')
     query_format = get_param(req, 'format')
 except UnicodeDecodeError, err:
     self.logger.increment('GET.errors')
     return HTTPBadRequest(body='parameters not utf8',
예제 #18
0
    def container_request(self, req, start_response, enabled):
        if CLIENT_VERSIONS_LOC in req.headers and \
                CLIENT_HISTORY_LOC in req.headers:
            if not req.headers[CLIENT_HISTORY_LOC]:
                # defer to versions location entirely
                del req.headers[CLIENT_HISTORY_LOC]
            elif req.headers[CLIENT_VERSIONS_LOC]:
                raise HTTPBadRequest(
                    request=req,
                    content_type='text/plain',
                    body='Only one of %s or %s may be specified' %
                    (CLIENT_VERSIONS_LOC, CLIENT_HISTORY_LOC))
            else:
                # history location is present and versions location is
                # present but empty -- clean it up
                del req.headers[CLIENT_VERSIONS_LOC]

        if CLIENT_VERSIONS_LOC in req.headers or \
                CLIENT_HISTORY_LOC in req.headers:
            if CLIENT_VERSIONS_LOC in req.headers:
                val = req.headers[CLIENT_VERSIONS_LOC]
                mode = 'stack'
            else:
                val = req.headers[CLIENT_HISTORY_LOC]
                mode = 'history'

            if not val:
                # empty value is the same as X-Remove-Versions-Location
                req.headers['X-Remove-Versions-Location'] = 'x'
            elif not config_true_value(enabled) and \
                    req.method in ('PUT', 'POST'):
                # differently from previous version, we are actually
                # returning an error if user tries to set versions location
                # while feature is explicitly disabled.
                raise HTTPPreconditionFailed(
                    request=req,
                    content_type='text/plain',
                    body='Versioned Writes is disabled')
            else:
                # OK, we received a value, have versioning enabled, and aren't
                # trying to set two modes at once. Validate the value and
                # translate to sysmeta.
                location = check_container_format(req, val)
                req.headers[SYSMETA_VERSIONS_LOC] = location
                req.headers[SYSMETA_VERSIONS_MODE] = mode

                # reset original header on container server to maintain sanity
                # now only sysmeta is source of Versions Location
                req.headers[CLIENT_VERSIONS_LOC] = ''

                # if both add and remove headers are in the same request
                # adding location takes precedence over removing
                for header in [
                        'X-Remove-Versions-Location',
                        'X-Remove-History-Location'
                ]:
                    if header in req.headers:
                        del req.headers[header]

        if any(
                req.headers.get(header) for header in
            ['X-Remove-Versions-Location', 'X-Remove-History-Location']):
            req.headers.update({
                CLIENT_VERSIONS_LOC: '',
                SYSMETA_VERSIONS_LOC: '',
                SYSMETA_VERSIONS_MODE: ''
            })
            for header in [
                    'X-Remove-Versions-Location', 'X-Remove-History-Location'
            ]:
                if header in req.headers:
                    del req.headers[header]

        # send request and translate sysmeta headers from response
        vw_ctx = VersionedWritesContext(self.app, self.logger)
        return vw_ctx.handle_container_request(req.environ, start_response)
예제 #19
0
    def _listing_pages_iter(self,
                            account_name,
                            lcontainer,
                            lprefix,
                            env,
                            marker='',
                            end_marker='',
                            reverse=True):
        '''Get "pages" worth of objects that start with a prefix.

        The optional keyword arguments ``marker``, ``end_marker``, and
        ``reverse`` are used similar to how they are for containers. We're
        either coming:

           - directly from ``_listing_iter``, in which case none of the
             optional args are specified, or

           - from ``_in_proxy_reverse_listing``, in which case ``reverse``
             is ``False`` and both ``marker`` and ``end_marker`` are specified
             (although they may still be blank).
        '''
        while True:
            lreq = make_pre_authed_request(env,
                                           method='GET',
                                           swift_source='VW',
                                           path='/v1/%s/%s' %
                                           (account_name, lcontainer))
            lreq.environ['QUERY_STRING'] = \
                'format=json&prefix=%s&marker=%s' % (
                    quote(lprefix), quote(marker))
            if end_marker:
                lreq.environ['QUERY_STRING'] += '&end_marker=%s' % (
                    quote(end_marker))
            if reverse:
                lreq.environ['QUERY_STRING'] += '&reverse=on'
            lresp = lreq.get_response(self.app)
            if not is_success(lresp.status_int):
                if lresp.status_int == HTTP_NOT_FOUND:
                    raise ListingIterNotFound()
                elif is_client_error(lresp.status_int):
                    raise HTTPPreconditionFailed()
                else:
                    raise ListingIterError()

            if not lresp.body:
                break

            sublisting = json.loads(lresp.body)
            if not sublisting:
                break

            # When using the ``reverse`` param, check that the listing is
            # actually reversed
            first_item = sublisting[0]['name'].encode('utf-8')
            last_item = sublisting[-1]['name'].encode('utf-8')
            page_is_after_marker = marker and first_item > marker
            if reverse and (first_item < last_item or page_is_after_marker):
                # Apparently there's at least one pre-2.6.0 container server
                yield self._in_proxy_reverse_listing(account_name, lcontainer,
                                                     lprefix, env, marker,
                                                     sublisting)
                return

            marker = last_item
            yield sublisting
예제 #20
0
파일: server.py 프로젝트: eckeman/swift
    def PUT(self, request):
        """Handle HTTP PUT requests for the Swift Object Server."""
        device, partition, account, container, obj, policy_idx = \
            get_name_and_placement(request, 5, 5, True)
        req_timestamp = valid_timestamp(request)
        error_response = check_object_creation(request, obj)
        if error_response:
            return error_response
        new_delete_at = int(request.headers.get('X-Delete-At') or 0)
        if new_delete_at and new_delete_at < time.time():
            return HTTPBadRequest(body='X-Delete-At in past',
                                  request=request,
                                  content_type='text/plain')
        try:
            fsize = request.message_length()
        except ValueError as e:
            return HTTPBadRequest(body=str(e),
                                  request=request,
                                  content_type='text/plain')
        try:
            disk_file = self.get_diskfile(device,
                                          partition,
                                          account,
                                          container,
                                          obj,
                                          policy_idx=policy_idx)
        except DiskFileDeviceUnavailable:
            return HTTPInsufficientStorage(drive=device, request=request)
        try:
            orig_metadata = disk_file.read_metadata()
        except DiskFileXattrNotSupported:
            return HTTPInsufficientStorage(drive=device, request=request)
        except (DiskFileNotExist, DiskFileQuarantined):
            orig_metadata = {}

        # Checks for If-None-Match
        if request.if_none_match is not None and orig_metadata:
            if '*' in request.if_none_match:
                # File exists already so return 412
                return HTTPPreconditionFailed(request=request)
            if orig_metadata.get('ETag') in request.if_none_match:
                # The current ETag matches, so return 412
                return HTTPPreconditionFailed(request=request)

        orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
        if orig_timestamp >= req_timestamp:
            return HTTPConflict(
                request=request,
                headers={'X-Backend-Timestamp': orig_timestamp.internal})
        orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
        upload_expiration = time.time() + self.max_upload_time
        etag = md5()
        elapsed_time = 0
        try:
            with disk_file.create(size=fsize) as writer:
                upload_size = 0

                def timeout_reader():
                    with ChunkReadTimeout(self.client_timeout):
                        return request.environ['wsgi.input'].read(
                            self.network_chunk_size)

                try:
                    for chunk in iter(lambda: timeout_reader(), ''):
                        start_time = time.time()
                        if start_time > upload_expiration:
                            self.logger.increment('PUT.timeouts')
                            return HTTPRequestTimeout(request=request)
                        etag.update(chunk)
                        upload_size = writer.write(chunk)
                        elapsed_time += time.time() - start_time
                except ChunkReadTimeout:
                    return HTTPRequestTimeout(request=request)
                if upload_size:
                    self.logger.transfer_rate('PUT.' + device + '.timing',
                                              elapsed_time, upload_size)
                if fsize is not None and fsize != upload_size:
                    return HTTPClientDisconnect(request=request)
                etag = etag.hexdigest()
                if 'etag' in request.headers and \
                        request.headers['etag'].lower() != etag:
                    return HTTPUnprocessableEntity(request=request)
                metadata = {
                    'X-Timestamp': request.timestamp.internal,
                    'Content-Type': request.headers['content-type'],
                    'ETag': etag,
                    'Content-Length': str(upload_size),
                }
                metadata.update(val for val in request.headers.iteritems()
                                if is_sys_or_user_meta('object', val[0]))
                headers_to_copy = (request.headers.get(
                    'X-Backend-Replication-Headers', '').split() +
                                   list(self.allowed_headers))
                for header_key in headers_to_copy:
                    if header_key in request.headers:
                        header_caps = header_key.title()
                        metadata[header_caps] = request.headers[header_key]
                writer.put(metadata)
        except (DiskFileXattrNotSupported, DiskFileNoSpace):
            return HTTPInsufficientStorage(drive=device, request=request)
        if orig_delete_at != new_delete_at:
            if new_delete_at:
                self.delete_at_update('PUT', new_delete_at, account, container,
                                      obj, request, device, policy_idx)
            if orig_delete_at:
                self.delete_at_update('DELETE', orig_delete_at, account,
                                      container, obj, request, device,
                                      policy_idx)
        self.container_update(
            'PUT', account, container, obj, request,
            HeaderKeyDict({
                'x-size': metadata['Content-Length'],
                'x-content-type': metadata['Content-Type'],
                'x-timestamp': metadata['X-Timestamp'],
                'x-etag': metadata['ETag']
            }), device, policy_idx)
        return HTTPCreated(request=request, etag=etag)
예제 #21
0
파일: server.py 프로젝트: eckeman/swift
 def DELETE(self, request):
     """Handle HTTP DELETE requests for the Swift Object Server."""
     device, partition, account, container, obj, policy_idx = \
         get_name_and_placement(request, 5, 5, True)
     req_timestamp = valid_timestamp(request)
     try:
         disk_file = self.get_diskfile(device,
                                       partition,
                                       account,
                                       container,
                                       obj,
                                       policy_idx=policy_idx)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     try:
         orig_metadata = disk_file.read_metadata()
     except DiskFileXattrNotSupported:
         return HTTPInsufficientStorage(drive=device, request=request)
     except DiskFileExpired as e:
         orig_timestamp = e.timestamp
         orig_metadata = e.metadata
         response_class = HTTPNotFound
     except DiskFileDeleted as e:
         orig_timestamp = e.timestamp
         orig_metadata = {}
         response_class = HTTPNotFound
     except (DiskFileNotExist, DiskFileQuarantined):
         orig_timestamp = 0
         orig_metadata = {}
         response_class = HTTPNotFound
     else:
         orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
         if orig_timestamp < req_timestamp:
             response_class = HTTPNoContent
         else:
             response_class = HTTPConflict
     response_timestamp = max(orig_timestamp, req_timestamp)
     orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
     try:
         req_if_delete_at_val = request.headers['x-if-delete-at']
         req_if_delete_at = int(req_if_delete_at_val)
     except KeyError:
         pass
     except ValueError:
         return HTTPBadRequest(request=request,
                               body='Bad X-If-Delete-At header value')
     else:
         # request includes x-if-delete-at; we must not place a tombstone
         # if we can not verify the x-if-delete-at time
         if not orig_timestamp:
             # no object found at all
             return HTTPNotFound()
         if orig_delete_at != req_if_delete_at:
             return HTTPPreconditionFailed(
                 request=request,
                 body='X-If-Delete-At and X-Delete-At do not match')
         else:
             # differentiate success from no object at all
             response_class = HTTPNoContent
     if orig_delete_at:
         self.delete_at_update('DELETE', orig_delete_at, account, container,
                               obj, request, device, policy_idx)
     if orig_timestamp < req_timestamp:
         disk_file.delete(req_timestamp)
         self.container_update(
             'DELETE', account, container, obj, request,
             HeaderKeyDict({'x-timestamp': req_timestamp.internal}), device,
             policy_idx)
     return response_class(
         request=request,
         headers={'X-Backend-Timestamp': response_timestamp.internal})
예제 #22
0
파일: obj.py 프로젝트: AymericDu/swift
    def _link_object(self, req):
        _, container, obj = req.headers['Oio-Copy-From'].split('/', 2)

        from_account = req.headers.get('X-Copy-From-Account',
                                       self.account_name)
        self.app.logger.info(
            "Creating link from %s/%s/%s to %s/%s/%s",
            # Existing
            from_account,
            container,
            obj,
            # New
            self.account_name,
            self.container_name,
            self.object_name)
        storage = self.app.storage

        if req.headers.get('Range'):
            raise Exception("Fast Copy with Range is unsupported")

            ranges = ranges_from_http_header(req.headers.get('Range'))
            if len(ranges) != 1:
                raise HTTPInternalServerError(
                    request=req, body="mutiple ranges unsupported")
            ranges = ranges[0]
        else:
            ranges = None

        headers = self._prepare_headers(req)
        metadata = self.load_object_metadata(headers)
        oio_headers = {REQID_HEADER: self.trans_id}
        oio_cache = req.environ.get('oio.cache')
        perfdata = req.environ.get('swift.perfdata')
        # FIXME(FVE): use object_show, cache in req.environ
        version = obj_version_from_env(req.environ)
        props = storage.object_get_properties(from_account,
                                              container,
                                              obj,
                                              headers=oio_headers,
                                              version=version,
                                              cache=oio_cache,
                                              perfdata=perfdata)
        if props['properties'].get(SLO, None):
            raise Exception("Fast Copy with SLO is unsupported")
        else:
            if ranges:
                raise HTTPInternalServerError(
                    request=req, body="no range supported with single object")

        try:
            # TODO check return code (values ?)
            link_meta = storage.object_link(from_account,
                                            container,
                                            obj,
                                            self.account_name,
                                            self.container_name,
                                            self.object_name,
                                            headers=oio_headers,
                                            properties=metadata,
                                            properties_directive='REPLACE',
                                            target_version=version,
                                            cache=oio_cache,
                                            perfdata=perfdata)
        # TODO(FVE): this exception catching block has to be refactored
        # TODO check which ones are ok or make non sense
        except exceptions.Conflict:
            raise HTTPConflict(request=req)
        except exceptions.PreconditionFailed:
            raise HTTPPreconditionFailed(request=req)
        except exceptions.SourceReadError:
            req.client_disconnect = True
            self.app.logger.warning(
                _('Client disconnected without sending last chunk'))
            self.app.logger.increment('client_disconnects')
            raise HTTPClientDisconnect(request=req)
        except exceptions.EtagMismatch:
            return HTTPUnprocessableEntity(request=req)
        except (exceptions.ServiceBusy, exceptions.OioTimeout,
                exceptions.DeadlineReached):
            raise
        except (exceptions.NoSuchContainer, exceptions.NotFound):
            raise HTTPNotFound(request=req)
        except exceptions.ClientException as err:
            # 481 = CODE_POLICY_NOT_SATISFIABLE
            if err.status == 481:
                raise exceptions.ServiceBusy()
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)
        except Exception:
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)

        resp = HTTPCreated(request=req, etag=link_meta['hash'])
        return resp
예제 #23
0
파일: server.py 프로젝트: bigbang009/swift
    def GET(self, req):
        """
        Handle HTTP GET request.

        The body of the response to a successful GET request contains a listing
        of either objects or shard ranges. The exact content of the listing is
        determined by a combination of request headers and query string
        parameters, as follows:

        * The type of the listing is determined by the
          ``X-Backend-Record-Type`` header. If this header has value ``shard``
          then the response body will be a list of shard ranges; if this header
          has value ``auto``, and the container state is ``sharding`` or
          ``sharded``, then the listing will be a list of shard ranges;
          otherwise the response body will be a list of objects.

        * Both shard range and object listings may be constrained to a name
          range by the ``marker`` and ``end_marker`` query string parameters.
          Object listings will only contain objects whose names are greater
          than any ``marker`` value and less than any ``end_marker`` value.
          Shard range listings will only contain shard ranges whose namespace
          is greater than or includes any ``marker`` value and is less than or
          includes any ``end_marker`` value.

        * Shard range listings may also be constrained by an ``includes`` query
          string parameter. If this parameter is present the listing will only
          contain shard ranges whose namespace includes the value of the
          parameter; any ``marker`` or ``end_marker`` parameters are ignored

        * The length of an object listing may be constrained by the ``limit``
          parameter. Object listings may also be constrained by ``prefix``,
          ``delimiter`` and ``path`` query string parameters.

        * Shard range listings will include deleted shard ranges if and only if
          the ``X-Backend-Include-Deleted`` header value is one of
          :attr:`swift.common.utils.TRUE_VALUES`. Object listings never
          include deleted objects.

        * Shard range listings may be constrained to include only shard ranges
          whose state is specified by a query string ``states`` parameter. If
          present, the ``states`` parameter should be a comma separated list of
          either the string or integer representation of
          :data:`~swift.common.utils.ShardRange.STATES`.

          Two alias values may be used in a ``states`` parameter value:
          ``listing`` will cause the listing to include all shard ranges in a
          state suitable for contributing to an object listing; ``updating``
          will cause the listing to include all shard ranges in a state
          suitable to accept an object update.

          If either of these aliases is used then the shard range listing will
          if necessary be extended with a synthesised 'filler' range in order
          to satisfy the requested name range when insufficient actual shard
          ranges are found. Any 'filler' shard range will cover the otherwise
          uncovered tail of the requested name range and will point back to the
          same container.

        * Listings are not normally returned from a deleted container. However,
          the ``X-Backend-Override-Deleted`` header may be used with a value in
          :attr:`swift.common.utils.TRUE_VALUES` to force a shard range
          listing to be returned from a deleted container whose DB file still
          exists.

        :param req: an instance of :class:`swift.common.swob.Request`
        :returns: an instance of :class:`swift.common.swob.Response`
        """
        drive, part, account, container, obj = get_obj_name_and_placement(req)
        path = get_param(req, 'path')
        prefix = get_param(req, 'prefix')
        delimiter = get_param(req, 'delimiter')
        marker = get_param(req, 'marker', '')
        end_marker = get_param(req, 'end_marker')
        limit = constraints.CONTAINER_LISTING_LIMIT
        given_limit = get_param(req, 'limit')
        reverse = config_true_value(get_param(req, 'reverse'))
        if given_limit and given_limit.isdigit():
            limit = int(given_limit)
            if limit > constraints.CONTAINER_LISTING_LIMIT:
                return HTTPPreconditionFailed(
                    request=req,
                    body='Maximum limit is %d' %
                    constraints.CONTAINER_LISTING_LIMIT)
        out_content_type = listing_formats.get_listing_content_type(req)
        try:
            check_drive(self.root, drive, self.mount_check)
        except ValueError:
            return HTTPInsufficientStorage(drive=drive, request=req)
        broker = self._get_container_broker(drive,
                                            part,
                                            account,
                                            container,
                                            pending_timeout=0.1,
                                            stale_reads_ok=True)
        info, is_deleted = broker.get_info_is_deleted()
        record_type = req.headers.get('x-backend-record-type', '').lower()
        if record_type == 'auto' and info.get('db_state') in (SHARDING,
                                                              SHARDED):
            record_type = 'shard'
        if record_type == 'shard':
            override_deleted = info and config_true_value(
                req.headers.get('x-backend-override-deleted', False))
            resp_headers = gen_resp_headers(info,
                                            is_deleted=is_deleted
                                            and not override_deleted)
            if is_deleted and not override_deleted:
                return HTTPNotFound(request=req, headers=resp_headers)
            resp_headers['X-Backend-Record-Type'] = 'shard'
            includes = get_param(req, 'includes')
            states = get_param(req, 'states')
            fill_gaps = False
            if states:
                states = list_from_csv(states)
                fill_gaps = any(('listing' in states, 'updating' in states))
                try:
                    states = broker.resolve_shard_range_states(states)
                except ValueError:
                    return HTTPBadRequest(request=req, body='Bad state')
            include_deleted = config_true_value(
                req.headers.get('x-backend-include-deleted', False))
            container_list = broker.get_shard_ranges(
                marker,
                end_marker,
                includes,
                reverse,
                states=states,
                include_deleted=include_deleted,
                fill_gaps=fill_gaps)
        else:
            resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
            if is_deleted:
                return HTTPNotFound(request=req, headers=resp_headers)
            resp_headers['X-Backend-Record-Type'] = 'object'
            # Use the retired db while container is in process of sharding,
            # otherwise use current db
            src_broker = broker.get_brokers()[0]
            container_list = src_broker.list_objects_iter(
                limit,
                marker,
                end_marker,
                prefix,
                delimiter,
                path,
                storage_policy_index=info['storage_policy_index'],
                reverse=reverse,
                allow_reserved=req.allow_reserved_names)
        return self.create_listing(req, out_content_type, info, resp_headers,
                                   broker.metadata, container_list, container)
예제 #24
0
    def handle_delete(self,
                      req,
                      objs_to_delete=None,
                      user_agent='BulkDelete',
                      swift_source='BD'):
        """
        :params req: a swob Request
        :raises HTTPException: on unhandled errors
        :returns: a swob Response
        """
        try:
            vrs, account, _junk = req.split_path(2, 3, True)
        except ValueError:
            return HTTPNotFound(request=req)

        incoming_format = req.headers.get('Content-Type')
        if incoming_format and not incoming_format.startswith('text/plain'):
            # For now only accept newline separated object names
            return HTTPNotAcceptable(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            return HTTPNotAcceptable(request=req)

        if objs_to_delete is None:
            objs_to_delete = self.get_objs_to_delete(req)
        failed_files = []
        success_count = not_found_count = 0
        failed_file_response_type = HTTPBadRequest
        for obj_to_delete in objs_to_delete:
            obj_to_delete = obj_to_delete.strip().lstrip('/')
            if not obj_to_delete:
                continue
            delete_path = '/'.join(['', vrs, account, obj_to_delete])
            if not check_utf8(delete_path):
                failed_files.append(
                    [quote(delete_path),
                     HTTPPreconditionFailed().status])
                continue
            new_env = req.environ.copy()
            new_env['PATH_INFO'] = delete_path
            del (new_env['wsgi.input'])
            new_env['CONTENT_LENGTH'] = 0
            new_env['HTTP_USER_AGENT'] = \
                '%s %s' % (req.environ.get('HTTP_USER_AGENT'), user_agent)
            new_env['swift.source'] = swift_source
            delete_obj_req = Request.blank(delete_path, new_env)
            resp = delete_obj_req.get_response(self.app)
            if resp.status_int // 100 == 2:
                success_count += 1
            elif resp.status_int == HTTP_NOT_FOUND:
                not_found_count += 1
            elif resp.status_int == HTTP_UNAUTHORIZED:
                return HTTPUnauthorized(request=req)
            else:
                if resp.status_int // 100 == 5:
                    failed_file_response_type = HTTPBadGateway
                failed_files.append([quote(delete_path), resp.status])

        resp_body = get_response_body(out_content_type, {
            'Number Deleted': success_count,
            'Number Not Found': not_found_count
        }, failed_files)
        if (success_count or not_found_count) and not failed_files:
            return HTTPOk(resp_body, content_type=out_content_type)
        if failed_files:
            return failed_file_response_type(resp_body,
                                             content_type=out_content_type)
        return HTTPBadRequest('Invalid bulk delete.')
예제 #25
0
    def handle_request(self, req):
        """
        Entry point for proxy server.
        Should return a WSGI-style callable (such as swob.Response).

        :param req: swob.Request object
        """
        try:
            self.logger.set_statsd_prefix('proxy-server')
            if req.content_length and req.content_length < 0:
                self.logger.increment('errors')
                return HTTPBadRequest(request=req,
                                      body='Invalid Content-Length')

            try:
                if not check_utf8(req.path_info):
                    self.logger.increment('errors')
                    return HTTPPreconditionFailed(
                        request=req, body='Invalid UTF8 or contains NULL')
            except UnicodeError:
                self.logger.increment('errors')
                return HTTPPreconditionFailed(
                    request=req, body='Invalid UTF8 or contains NULL')

            try:
                controller, path_parts = self.get_controller(req)
                p = req.path_info
                if isinstance(p, six.text_type):
                    p = p.encode('utf-8')
            except APIVersionError:
                self.logger.increment('errors')
                return HTTPBadRequest(request=req)
            except ValueError:
                self.logger.increment('errors')
                return HTTPNotFound(request=req)
            if not controller:
                self.logger.increment('errors')
                return HTTPPreconditionFailed(request=req, body='Bad URL')
            if self.deny_host_headers and \
                    req.host.split(':')[0] in self.deny_host_headers:
                return HTTPForbidden(request=req, body='Invalid host header')

            self.logger.set_statsd_prefix('proxy-server.' +
                                          controller.server_type.lower())
            controller = controller(self, **path_parts)
            if 'swift.trans_id' not in req.environ:
                # if this wasn't set by an earlier middleware, set it now
                trans_id_suffix = self.trans_id_suffix
                trans_id_extra = req.headers.get('x-trans-id-extra')
                if trans_id_extra:
                    trans_id_suffix += '-' + trans_id_extra[:32]
                trans_id = generate_trans_id(trans_id_suffix)
                req.environ['swift.trans_id'] = trans_id
                self.logger.txn_id = trans_id
            req.headers['x-trans-id'] = req.environ['swift.trans_id']
            controller.trans_id = req.environ['swift.trans_id']
            self.logger.client_ip = get_remote_client(req)
            try:
                handler = getattr(controller, req.method)
                getattr(handler, 'publicly_accessible')
            except AttributeError:
                allowed_methods = getattr(controller, 'allowed_methods', set())
                return HTTPMethodNotAllowed(
                    request=req, headers={'Allow': ', '.join(allowed_methods)})
            old_authorize = None
            if 'swift.authorize' in req.environ:
                # We call authorize before the handler, always. If authorized,
                # we remove the swift.authorize hook so isn't ever called
                # again. If not authorized, we return the denial unless the
                # controller's method indicates it'd like to gather more
                # information and try again later.
                resp = req.environ['swift.authorize'](req)
                if not resp and not req.headers.get('X-Copy-From-Account') \
                        and not req.headers.get('Destination-Account'):
                    # No resp means authorized, no delayed recheck required.
                    old_authorize = req.environ['swift.authorize']
                else:
                    # Response indicates denial, but we might delay the denial
                    # and recheck later. If not delayed, return the error now.
                    if not getattr(handler, 'delay_denial', None):
                        return resp
            # Save off original request method (GET, POST, etc.) in case it
            # gets mutated during handling.  This way logging can display the
            # method the client actually sent.
            req.environ['swift.orig_req_method'] = req.method
            try:
                if old_authorize:
                    req.environ.pop('swift.authorize', None)
                return handler(req)
            finally:
                if old_authorize:
                    req.environ['swift.authorize'] = old_authorize
        except HTTPException as error_response:
            return error_response
        except (Exception, Timeout):
            self.logger.exception(_('ERROR Unhandled exception in request'))
            return HTTPServerError(request=req)
예제 #26
0
    def PUT(self, req):
        """HTTP PUT request handler."""
        if req.if_none_match is not None and '*' not in req.if_none_match:
            # Sending an etag with if-none-match isn't currently supported
            return HTTPBadRequest(request=req, content_type='text/plain',
                                  body='If-None-Match only supports *')
        container_info = self.container_info(
            self.account_name, self.container_name, req)
        policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                       container_info['storage_policy'])
        obj_ring = self.app.get_object_ring(policy_index)
        # pass the policy index to storage nodes via req header
        req.headers['X-Backend-Storage-Policy-Index'] = policy_index
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not containers:
            return HTTPNotFound(request=req)
        try:
            ml = req.message_length()
        except ValueError as e:
            return HTTPBadRequest(request=req, content_type='text/plain',
                                  body=str(e))
        except AttributeError as e:
            return HTTPNotImplemented(request=req, content_type='text/plain',
                                      body=str(e))
        if ml is not None and ml > constraints.MAX_FILE_SIZE:
            return HTTPRequestEntityTooLarge(request=req)
        if 'x-delete-after' in req.headers:
            try:
                x_delete_after = int(req.headers['x-delete-after'])
            except ValueError:
                return HTTPBadRequest(request=req,
                                      content_type='text/plain',
                                      body='Non-integer X-Delete-After')
            req.headers['x-delete-at'] = normalize_delete_at_timestamp(
                time.time() + x_delete_after)
        partition, nodes = obj_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)
        # do a HEAD request for container sync and checking object versions
        if 'x-timestamp' in req.headers or \
                (object_versions and not
                 req.environ.get('swift_versioned_copy')):
            # make sure proxy-server uses the right policy index
            _headers = {'X-Backend-Storage-Policy-Index': policy_index,
                        'X-Newest': 'True'}
            hreq = Request.blank(req.path_info, headers=_headers,
                                 environ={'REQUEST_METHOD': 'HEAD'})
            hresp = self.GETorHEAD_base(
                hreq, _('Object'), obj_ring, partition,
                hreq.swift_entity_path)
        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req_timestamp = Timestamp(req.headers['X-Timestamp'])
                if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
                        hresp.environ['swift_x_timestamp'] >= req_timestamp:
                    return HTTPAccepted(request=req)
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
            req.headers['X-Timestamp'] = req_timestamp.internal
        else:
            req.headers['X-Timestamp'] = Timestamp(time.time()).internal
        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        detect_content_type = \
            config_true_value(req.headers.get('x-detect-content-type'))
        if detect_content_type or not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                'application/octet-stream'
            if detect_content_type:
                req.headers.pop('x-detect-content-type')
            else:
                content_type_manually_set = False

        error_response = check_object_creation(req, self.object_name) or \
            check_content_type(req)
        if error_response:
            return error_response
        if object_versions and not req.environ.get('swift_versioned_copy'):
            if hresp.status_int != HTTP_NOT_FOUND:
                # This is a version manifest and needs to be handled
                # differently. First copy the existing data to a new object,
                # then write the data from this request to the version manifest
                # object.
                lcontainer = object_versions.split('/')[0]
                prefix_len = '%03x' % len(self.object_name)
                lprefix = prefix_len + self.object_name + '/'
                ts_source = hresp.environ.get('swift_x_timestamp')
                if ts_source is None:
                    ts_source = time.mktime(time.strptime(
                                            hresp.headers['last-modified'],
                                            '%a, %d %b %Y %H:%M:%S GMT'))
                new_ts = Timestamp(ts_source).internal
                vers_obj_name = lprefix + new_ts
                copy_headers = {
                    'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
                copy_environ = {'REQUEST_METHOD': 'COPY',
                                'swift_versioned_copy': True
                                }
                copy_req = Request.blank(req.path_info, headers=copy_headers,
                                         environ=copy_environ)
                copy_resp = self.COPY(copy_req)
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

        reader = req.environ['wsgi.input'].read
        data_source = iter(lambda: reader(self.app.client_chunk_size), '')
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            if req.environ.get('swift.orig_req_method', req.method) != 'POST':
                req.environ.setdefault('swift.log_info', []).append(
                    'x-copy-from:%s' % source_header)
            src_container_name, src_obj_name = check_copy_from_header(req)
            ver, acct, _rest = req.split_path(2, 3, True)
            if isinstance(acct, unicode):
                acct = acct.encode('utf-8')
            source_header = '/%s/%s/%s/%s' % (ver, acct,
                                              src_container_name, src_obj_name)
            source_req = req.copy_get()
            # make sure the source request uses it's container_info
            source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            sink_req = Request.blank(req.path_info,
                                     environ=req.environ, headers=req.headers)
            source_resp = self.GET(source_req)
            # This gives middlewares a way to change the source; for example,
            # this lets you COPY a SLO manifest and have the new object be the
            # concatenation of the segments (like what a GET request gives
            # the client), not a copy of the manifest file.
            hook = req.environ.get(
                'swift.copy_hook',
                (lambda source_req, source_resp, sink_req: source_resp))
            source_resp = hook(source_req, source_resp, sink_req)

            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            data_source = iter(source_resp.app_iter)
            sink_req.content_length = source_resp.content_length
            if sink_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            if sink_req.content_length > constraints.MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=req)
            sink_req.etag = source_resp.etag
            # we no longer need the X-Copy-From header
            del sink_req.headers['X-Copy-From']
            if not content_type_manually_set:
                sink_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if not config_true_value(
                    sink_req.headers.get('x-fresh-metadata', 'false')):
                copy_headers_into(source_resp, sink_req)
                copy_headers_into(req, sink_req)
            # copy over x-static-large-object for POSTs and manifest copies
            if 'X-Static-Large-Object' in source_resp.headers and \
                    req.params.get('multipart-manifest') == 'get':
                sink_req.headers['X-Static-Large-Object'] = \
                    source_resp.headers['X-Static-Large-Object']

            req = sink_req

        if 'x-delete-at' in req.headers:
            try:
                x_delete_at = normalize_delete_at_timestamp(
                    int(req.headers['x-delete-at']))
                if int(x_delete_at) < time.time():
                    return HTTPBadRequest(
                        body='X-Delete-At in past', request=req,
                        content_type='text/plain')
            except ValueError:
                return HTTPBadRequest(request=req, content_type='text/plain',
                                      body='Non-integer X-Delete-At')
            req.environ.setdefault('swift.log_info', []).append(
                'x-delete-at:%s' % x_delete_at)
            delete_at_container = normalize_delete_at_timestamp(
                int(x_delete_at) /
                self.app.expiring_objects_container_divisor *
                self.app.expiring_objects_container_divisor)
            delete_at_part, delete_at_nodes = \
                self.app.container_ring.get_nodes(
                    self.app.expiring_objects_account, delete_at_container)
        else:
            delete_at_container = delete_at_part = delete_at_nodes = None

        node_iter = GreenthreadSafeIterator(
            self.iter_nodes_local_first(obj_ring, partition))
        pile = GreenPile(len(nodes))
        te = req.headers.get('transfer-encoding', '')
        chunked = ('chunked' in te)

        outgoing_headers = self._backend_requests(
            req, len(nodes), container_partition, containers,
            delete_at_container, delete_at_part, delete_at_nodes)

        for nheaders in outgoing_headers:
            # RFC2616:8.2.3 disallows 100-continue without a body
            if (req.content_length > 0) or chunked:
                nheaders['Expect'] = '100-continue'
            pile.spawn(self._connect_put_node, node_iter, partition,
                       req.swift_entity_path, nheaders,
                       self.app.logger.thread_locals)

        conns = [conn for conn in pile if conn]
        min_conns = quorum_size(len(nodes))

        if req.if_none_match is not None and '*' in req.if_none_match:
            statuses = [conn.resp.status for conn in conns if conn.resp]
            if HTTP_PRECONDITION_FAILED in statuses:
                # If we find any copy of the file, it shouldn't be uploaded
                self.app.logger.debug(
                    _('Object PUT returning 412, %(statuses)r'),
                    {'statuses': statuses})
                return HTTPPreconditionFailed(request=req)

        if len(conns) < min_conns:
            self.app.logger.error(
                _('Object PUT returning 503, %(conns)s/%(nodes)s '
                  'required connections'),
                {'conns': len(conns), 'nodes': min_conns})
            return HTTPServiceUnavailable(request=req)
        bytes_transferred = 0
        try:
            with ContextPool(len(nodes)) as pool:
                for conn in conns:
                    conn.failed = False
                    conn.queue = Queue(self.app.put_queue_depth)
                    pool.spawn(self._send_file, conn, req.path)
                while True:
                    with ChunkReadTimeout(self.app.client_timeout):
                        try:
                            chunk = next(data_source)
                        except StopIteration:
                            if chunked:
                                for conn in conns:
                                    conn.queue.put('0\r\n\r\n')
                            break
                    bytes_transferred += len(chunk)
                    if bytes_transferred > constraints.MAX_FILE_SIZE:
                        return HTTPRequestEntityTooLarge(request=req)
                    for conn in list(conns):
                        if not conn.failed:
                            conn.queue.put(
                                '%x\r\n%s\r\n' % (len(chunk), chunk)
                                if chunked else chunk)
                        else:
                            conns.remove(conn)
                    if len(conns) < min_conns:
                        self.app.logger.error(_(
                            'Object PUT exceptions during'
                            ' send, %(conns)s/%(nodes)s required connections'),
                            {'conns': len(conns), 'nodes': min_conns})
                        return HTTPServiceUnavailable(request=req)
                for conn in conns:
                    if conn.queue.unfinished_tasks:
                        conn.queue.join()
            conns = [conn for conn in conns if not conn.failed]
        except ChunkReadTimeout as err:
            self.app.logger.warn(
                _('ERROR Client read timeout (%ss)'), err.seconds)
            self.app.logger.increment('client_timeouts')
            return HTTPRequestTimeout(request=req)
        except (Exception, Timeout):
            self.app.logger.exception(
                _('ERROR Exception causing client disconnect'))
            return HTTPClientDisconnect(request=req)
        if req.content_length and bytes_transferred < req.content_length:
            req.client_disconnect = True
            self.app.logger.warn(
                _('Client disconnected without sending enough data'))
            self.app.logger.increment('client_disconnects')
            return HTTPClientDisconnect(request=req)

        statuses, reasons, bodies, etags = self._get_put_responses(req, conns,
                                                                   nodes)

        if len(etags) > 1:
            self.app.logger.error(
                _('Object servers returned %s mismatched etags'), len(etags))
            return HTTPServerError(request=req)
        etag = etags.pop() if len(etags) else None
        resp = self.best_response(req, statuses, reasons, bodies,
                                  _('Object PUT'), etag=etag)
        if source_header:
            resp.headers['X-Copied-From'] = quote(
                source_header.split('/', 3)[3])
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
        resp.last_modified = math.ceil(
            float(Timestamp(req.headers['X-Timestamp'])))
        return resp
예제 #27
0
파일: obj.py 프로젝트: AymericDu/swift
    def _store_object(self, req, data_source, headers):
        kwargs = {}
        content_type = req.headers.get('content-type', 'octet/stream')
        policy = None
        container_info = self.container_info(self.account_name,
                                             self.container_name, req)
        if 'X-Oio-Storage-Policy' in req.headers:
            policy = req.headers.get('X-Oio-Storage-Policy')
            if not self.app.POLICIES.get_by_name(policy):
                raise HTTPBadRequest(
                    "invalid policy '%s', must be in %s" %
                    (policy, self.app.POLICIES.by_name.keys()))
        else:
            try:
                policy_index = int(
                    req.headers.get('X-Backend-Storage-Policy-Index',
                                    container_info['storage_policy']))
            except TypeError:
                policy_index = 0
            if policy_index != 0:
                policy = self.app.POLICIES.get_by_index(policy_index).name
            else:
                content_length = int(req.headers.get('content-length', 0))
                policy = self._get_auto_policy_from_size(content_length)

        ct_props = {'properties': {}, 'system': {}}
        metadata = self.load_object_metadata(headers)
        oio_headers = {REQID_HEADER: self.trans_id}
        oio_cache = req.environ.get('oio.cache')
        perfdata = req.environ.get('swift.perfdata')
        # only send headers if needed
        if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
            oio_headers[FORCEVERSIONING_HEADER] = \
                headers.get(FORCEVERSIONING_HEADER)
        if req.environ.get('oio.force-version'):
            # In a case of MPU, it contains version of the UploadId
            # to be able to include version-id of MPU in S3 reponse
            kwargs['version'] = req.environ.get('oio.force-version')

        # In case a shard is being created, save the name of the S3 bucket
        # in a container property. This will be used when aggregating
        # container statistics to make bucket statistics.
        if BUCKET_NAME_HEADER in headers:
            bname = headers[BUCKET_NAME_HEADER]
            # FIXME(FVE): the segments container is not part of another bucket!
            # We should not have to strip this here.
            if bname and bname.endswith(MULTIUPLOAD_SUFFIX):
                bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
            ct_props['system'][BUCKET_NAME_PROP] = bname
        try:
            _chunks, _size, checksum, _meta = self._object_create(
                self.account_name,
                self.container_name,
                obj_name=self.object_name,
                file_or_path=data_source,
                mime_type=content_type,
                policy=policy,
                headers=oio_headers,
                etag=req.headers.get('etag', '').strip('"'),
                properties=metadata,
                container_properties=ct_props,
                cache=oio_cache,
                perfdata=perfdata,
                **kwargs)
            # TODO(FVE): when oio-sds supports it, do that in a callback
            # passed to object_create (or whatever upload method supports it)
            footer_md = self.load_object_metadata(self._get_footers(req))
            if footer_md:
                self.app.storage.object_set_properties(self.account_name,
                                                       self.container_name,
                                                       self.object_name,
                                                       version=_meta.get(
                                                           'version', None),
                                                       properties=footer_md,
                                                       headers=oio_headers,
                                                       cache=oio_cache,
                                                       perfdata=perfdata)
        except exceptions.Conflict:
            raise HTTPConflict(request=req)
        except exceptions.PreconditionFailed:
            raise HTTPPreconditionFailed(request=req)
        except SourceReadTimeout as err:
            self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
            self.app.logger.increment('client_timeouts')
            raise HTTPRequestTimeout(request=req)
        except exceptions.SourceReadError:
            req.client_disconnect = True
            self.app.logger.warning(
                _('Client disconnected without sending last chunk'))
            self.app.logger.increment('client_disconnects')
            raise HTTPClientDisconnect(request=req)
        except exceptions.EtagMismatch:
            return HTTPUnprocessableEntity(request=req)
        except (exceptions.ServiceBusy, exceptions.OioTimeout,
                exceptions.DeadlineReached):
            raise
        except exceptions.NoSuchContainer:
            raise HTTPNotFound(request=req)
        except exceptions.ClientException as err:
            # 481 = CODE_POLICY_NOT_SATISFIABLE
            if err.status == 481:
                raise exceptions.ServiceBusy()
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)
        except Exception:
            self.app.logger.exception(
                _('ERROR Exception transferring data %s'), {'path': req.path})
            raise HTTPInternalServerError(request=req)

        last_modified = int(_meta.get('mtime', math.ceil(time.time())))

        # FIXME(FVE): if \x10 character in object name, decode version
        # number and set it in the response headers, instead of the oio
        # version number.
        version_id = _meta.get('version', 'null')
        resp = HTTPCreated(request=req,
                           etag=checksum,
                           last_modified=last_modified,
                           headers={'x-object-sysmeta-version-id': version_id})
        return resp
예제 #28
0
    def DELETE(self, req):
        """HTTP DELETE request handler."""
        container_info = self.container_info(
            self.account_name, self.container_name, req)
        # pass the policy index to storage nodes via req header
        policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                       container_info['storage_policy'])
        obj_ring = self.app.get_object_ring(policy_index)
        # pass the policy index to storage nodes via req header
        req.headers['X-Backend-Storage-Policy-Index'] = policy_index
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if object_versions:
            # this is a version manifest and needs to be handled differently
            object_versions = unquote(object_versions)
            lcontainer = object_versions.split('/')[0]
            prefix_len = '%03x' % len(self.object_name)
            lprefix = prefix_len + self.object_name + '/'
            last_item = None
            try:
                for last_item in self._listing_iter(lcontainer, lprefix,
                                                    req.environ):
                    pass
            except ListingIterNotFound:
                # no worries, last_item is None
                pass
            except ListingIterNotAuthorized as err:
                return err.aresp
            except ListingIterError:
                return HTTPServerError(request=req)
            if last_item:
                # there are older versions so copy the previous version to the
                # current object and delete the previous version
                orig_container = self.container_name
                orig_obj = self.object_name
                self.container_name = lcontainer
                self.object_name = last_item['name'].encode('utf-8')
                copy_path = '/v1/' + self.account_name + '/' + \
                            self.container_name + '/' + self.object_name
                copy_headers = {'X-Newest': 'True',
                                'Destination': orig_container + '/' + orig_obj
                                }
                copy_environ = {'REQUEST_METHOD': 'COPY',
                                'swift_versioned_copy': True
                                }
                creq = Request.blank(copy_path, headers=copy_headers,
                                     environ=copy_environ)
                copy_resp = self.COPY(creq)
                if is_client_error(copy_resp.status_int):
                    # some user error, maybe permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)
                # reset these because the COPY changed them
                self.container_name = lcontainer
                self.object_name = last_item['name'].encode('utf-8')
                new_del_req = Request.blank(copy_path, environ=req.environ)
                container_info = self.container_info(
                    self.account_name, self.container_name, req)
                policy_idx = container_info['storage_policy']
                obj_ring = self.app.get_object_ring(policy_idx)
                # pass the policy index to storage nodes via req header
                new_del_req.headers['X-Backend-Storage-Policy-Index'] = \
                    policy_idx
                container_partition = container_info['partition']
                containers = container_info['nodes']
                new_del_req.acl = container_info['write_acl']
                new_del_req.path_info = copy_path
                req = new_del_req
                # remove 'X-If-Delete-At', since it is not for the older copy
                if 'X-If-Delete-At' in req.headers:
                    del req.headers['X-If-Delete-At']
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not containers:
            return HTTPNotFound(request=req)
        partition, nodes = obj_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)
        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req_timestamp = Timestamp(req.headers['X-Timestamp'])
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
            req.headers['X-Timestamp'] = req_timestamp.internal
        else:
            req.headers['X-Timestamp'] = Timestamp(time.time()).internal

        headers = self._backend_requests(
            req, len(nodes), container_partition, containers)
        resp = self.make_requests(req, obj_ring,
                                  partition, 'DELETE', req.swift_entity_path,
                                  headers)
        return resp
예제 #29
0
    def handle_extract(self, req, compress_type):
        """
        :params req: a swob Request
        :params compress_type: specifying the compression type of the tar.
                               Accepts '', 'gz, or 'bz2'
        :raises HTTPException: on unhandled errors
        :returns: a swob response to request
        """
        success_count = 0
        failed_files = []
        existing_containers = set()
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            return HTTPNotAcceptable(request=req)
        if req.content_length is None and \
                req.headers.get('transfer-encoding', '').lower() != 'chunked':
            return HTTPLengthRequired(request=req)
        try:
            vrs, account, extract_base = req.split_path(2, 3, True)
        except ValueError:
            return HTTPNotFound(request=req)
        extract_base = extract_base or ''
        extract_base = extract_base.rstrip('/')
        try:
            tar = tarfile.open(mode='r|' + compress_type,
                               fileobj=req.body_file)
            while True:
                tar_info = tar.next()
                if tar_info is None or \
                        len(failed_files) >= self.max_failed_extractions:
                    break
                if tar_info.isfile():
                    obj_path = tar_info.name
                    if obj_path.startswith('./'):
                        obj_path = obj_path[2:]
                    obj_path = obj_path.lstrip('/')
                    if extract_base:
                        obj_path = extract_base + '/' + obj_path
                    if '/' not in obj_path:
                        continue  # ignore base level file

                    destination = '/'.join(['', vrs, account, obj_path])
                    container = obj_path.split('/', 1)[0]
                    if not check_utf8(destination):
                        failed_files.append([
                            quote(destination[:MAX_PATH_LENGTH]),
                            HTTPPreconditionFailed().status
                        ])
                        continue
                    if tar_info.size > MAX_FILE_SIZE:
                        failed_files.append([
                            quote(destination[:MAX_PATH_LENGTH]),
                            HTTPRequestEntityTooLarge().status
                        ])
                        continue
                    if container not in existing_containers:
                        try:
                            self.create_container(
                                req, '/'.join(['', vrs, account, container]))
                            existing_containers.add(container)
                        except CreateContainerError, err:
                            if err.status_int == HTTP_UNAUTHORIZED:
                                return HTTPUnauthorized(request=req)
                            failed_files.append([
                                quote(destination[:MAX_PATH_LENGTH]),
                                err.status
                            ])
                            continue
                        except ValueError:
                            failed_files.append([
                                quote(destination[:MAX_PATH_LENGTH]),
                                HTTP_BAD_REQUEST
                            ])
                            continue
                        if len(existing_containers) > self.max_containers:
                            return HTTPBadRequest(
                                'More than %d base level containers in tar.' %
                                self.max_containers)
예제 #30
0
    def handle_obj_versions_put(self, req, object_versions, object_name,
                                policy_index):
        ret = None

        # do a HEAD request to check object versions
        _headers = {
            'X-Newest': 'True',
            'X-Backend-Storage-Policy-Index': policy_index,
            'x-auth-token': req.headers.get('x-auth-token')
        }

        # make a pre_auth request in case the user has write access
        # to container, but not READ. This was allowed in previous version
        # (i.e., before middleware) so keeping the same behavior here
        head_req = make_pre_authed_request(req.environ,
                                           path=req.path_info,
                                           headers=_headers,
                                           method='HEAD',
                                           swift_source='VW')
        hresp = head_req.get_response(self.app)

        is_dlo_manifest = 'X-Object-Manifest' in req.headers or \
                          'X-Object-Manifest' in hresp.headers

        # if there's an existing object, then copy it to
        # X-Versions-Location
        if is_success(hresp.status_int) and not is_dlo_manifest:
            lcontainer = object_versions.split('/')[0]
            prefix_len = '%03x' % len(object_name)
            lprefix = prefix_len + object_name + '/'
            ts_source = hresp.environ.get('swift_x_timestamp')
            if ts_source is None:
                ts_source = time.mktime(
                    time.strptime(hresp.headers['last-modified'],
                                  '%a, %d %b %Y %H:%M:%S GMT'))
            new_ts = Timestamp(ts_source).internal
            vers_obj_name = lprefix + new_ts
            copy_headers = {
                'Destination': '%s/%s' % (lcontainer, vers_obj_name),
                'x-auth-token': req.headers.get('x-auth-token')
            }

            # COPY implementation sets X-Newest to True when it internally
            # does a GET on source object. So, we don't have to explicity
            # set it in request headers here.
            copy_req = make_pre_authed_request(req.environ,
                                               path=req.path_info,
                                               headers=copy_headers,
                                               method='COPY',
                                               swift_source='VW')
            copy_resp = copy_req.get_response(self.app)

            if is_success(copy_resp.status_int):
                # success versioning previous existing object
                # return None and handle original request
                ret = None
            else:
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    ret = HTTPPreconditionFailed(request=req)
                else:
                    # could not copy the data, bail
                    ret = HTTPServiceUnavailable(request=req)

        else:
            if hresp.status_int == HTTP_NOT_FOUND or is_dlo_manifest:
                # nothing to version
                # return None and handle original request
                ret = None
            else:
                # if not HTTP_NOT_FOUND, return error immediately
                ret = hresp

        return ret