Esempio n. 1
0
def _validate_and_prep_request_headers(req):
    """
    Validate that the value from x-symlink-target header is well formatted
    and that the x-symlink-target-etag header (if present) does not contain
    problematic characters. We assume the caller ensures that
    x-symlink-target header is present in req.headers.

    :param req: HTTP request object
    :returns: a tuple, the full versioned path to the object (as a WSGI string)
              and the X-Symlink-Target-Etag header value which may be None
    :raise: HTTPPreconditionFailed if x-symlink-target value
            is not well formatted.
    :raise: HTTPBadRequest if the x-symlink-target value points to the request
            path.
    :raise: HTTPBadRequest if the x-symlink-target-etag value contains
            a semicolon, double-quote, or backslash.
    """
    # N.B. check_path_header doesn't assert the leading slash and
    # copy middleware may accept the format. In the symlink, API
    # says apparently to use "container/object" format so add the
    # validation first, here.
    error_body = 'X-Symlink-Target header must be of the form ' \
                 '<container name>/<object name>'
    if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'):
        raise HTTPPreconditionFailed(body=error_body,
                                     request=req,
                                     content_type='text/plain')

    # check container and object format
    container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body)
    req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj))

    # Check account format if it exists
    account = check_account_format(
        req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \
        if TGT_ACCT_SYMLINK_HDR in req.headers else None

    # Extract request path
    _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)

    if account:
        req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account)
    else:
        account = req_acc

    # Check if symlink targets the symlink itself or not
    if (account, container, obj) == (req_acc, req_cont, req_obj):
        raise HTTPBadRequest(body='Symlink cannot target itself',
                             request=req,
                             content_type='text/plain')
    etag = req.headers.get(TGT_ETAG_SYMLINK_HDR, None)
    if etag and any(c in etag for c in ';"\\'):
        # See cgi.parse_header for why the above chars are problematic
        raise HTTPBadRequest(body='Bad %s format' %
                             TGT_ETAG_SYMLINK_HDR.title(),
                             request=req,
                             content_type='text/plain')
    if not (etag or req.headers.get('Content-Type')):
        req.headers['Content-Type'] = 'application/symlink'
    return '/v1/%s/%s/%s' % (account, container, obj), etag
 def handle_COPY(self, req, start_response):
     if not req.headers.get('Destination'):
         return HTTPPreconditionFailed(request=req,
                                       body='Destination header required'
                                       )(req.environ, start_response)
     dest_account = self.account_name
     if 'Destination-Account' in req.headers:
         dest_account = req.headers.get('Destination-Account')
         dest_account = check_account_format(req, dest_account)
         req.headers['X-Copy-From-Account'] = self.account_name
         self.account_name = dest_account
         del req.headers['Destination-Account']
     dest_container, dest_object = _check_destination_header(req)
     source = '/%s/%s' % (self.container_name, self.object_name)
     self.container_name = dest_container
     self.object_name = dest_object
     # re-write the existing request as a PUT instead of creating a new one
     req.method = 'PUT'
     # As this the path info is updated with destination container,
     # the proxy server app will use the right object controller
     # implementation corresponding to the container's policy type.
     ver, _junk = req.split_path(1, 2, rest_with_last=True)
     req.path_info = '/%s/%s/%s/%s' % \
                     (ver, dest_account, dest_container, dest_object)
     req.headers['Content-Length'] = 0
     req.headers['X-Copy-From'] = quote(source)
     del req.headers['Destination']
     return self.handle_PUT(req, start_response)
Esempio n. 3
0
 def COPY(self, req):
     """HTTP COPY request handler."""
     if not req.headers.get('Destination'):
         return HTTPPreconditionFailed(request=req,
                                       body='Destination header required')
     dest_account = self.account_name
     if 'Destination-Account' in req.headers:
         dest_account = req.headers.get('Destination-Account')
         dest_account = check_account_format(req, dest_account)
         req.headers['X-Copy-From-Account'] = self.account_name
         self.account_name = dest_account
         del req.headers['Destination-Account']
     dest_container, dest_object = check_destination_header(req)
     source = '/%s/%s' % (self.container_name, self.object_name)
     self.container_name = dest_container
     self.object_name = dest_object
     # re-write the existing request as a PUT instead of creating a new one
     # since this one is already attached to the posthooklogger
     req.method = 'PUT'
     req.path_info = '/v1/%s/%s/%s' % \
                     (dest_account, dest_container, dest_object)
     req.headers['Content-Length'] = 0
     req.headers['X-Copy-From'] = quote(source)
     del req.headers['Destination']
     return self.PUT(req)
Esempio n. 4
0
 def COPY(self, req):
     """HTTP COPY request handler."""
     if not req.headers.get('Destination'):
         return HTTPPreconditionFailed(request=req,
                                       body='Destination header required')
     dest_account = self.account_name
     if 'Destination-Account' in req.headers:
         dest_account = req.headers.get('Destination-Account')
         dest_account = check_account_format(req, dest_account)
         req.headers['X-Copy-From-Account'] = self.account_name
         self.account_name = dest_account
         del req.headers['Destination-Account']
     dest_container, dest_object = check_destination_header(req)
     source = '/%s/%s' % (self.container_name, self.object_name)
     self.container_name = dest_container
     self.object_name = dest_object
     # re-write the existing request as a PUT instead of creating a new one
     # since this one is already attached to the posthooklogger
     req.method = 'PUT'
     req.path_info = '/v1/%s/%s/%s' % \
                     (dest_account, dest_container, dest_object)
     req.headers['Content-Length'] = 0
     req.headers['X-Copy-From'] = quote(source)
     del req.headers['Destination']
     return self.PUT(req)
Esempio n. 5
0
    def object_request(self, req, version, account, container, obj,
                       allow_versioned_writes):
        account_name = unquote(account)
        container_name = unquote(container)
        object_name = unquote(obj)
        container_info = None
        resp = None
        is_enabled = config_true_value(allow_versioned_writes)
        if req.method in ('PUT', 'DELETE'):
            container_info = get_container_info(
                req.environ, self.app)
        elif req.method == 'COPY' and 'Destination' in req.headers:
            if 'Destination-Account' in req.headers:
                account_name = req.headers.get('Destination-Account')
                account_name = check_account_format(req, account_name)
            container_name, object_name = check_destination_header(req)
            req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
                version, account_name, container_name, object_name)
            container_info = get_container_info(
                req.environ, self.app)

        if not container_info:
            return self.app

        # To maintain backwards compatibility, container version
        # location could be stored as sysmeta or not, need to check both.
        # If stored as sysmeta, check if middleware is enabled. If sysmeta
        # is not set, but versions property is set in container_info, then
        # for backwards compatibility feature is enabled.
        object_versions = container_info.get(
            'sysmeta', {}).get('versions-location')
        if object_versions and isinstance(object_versions, six.text_type):
            object_versions = object_versions.encode('utf-8')
        elif not object_versions:
            object_versions = container_info.get('versions')
            # if allow_versioned_writes is not set in the configuration files
            # but 'versions' is configured, enable feature to maintain
            # backwards compatibility
            if not allow_versioned_writes and object_versions:
                is_enabled = True

        if is_enabled and object_versions:
            object_versions = unquote(object_versions)
            vw_ctx = VersionedWritesContext(self.app, self.logger)
            if req.method in ('PUT', 'COPY'):
                policy_idx = req.headers.get(
                    'X-Backend-Storage-Policy-Index',
                    container_info['storage_policy'])
                resp = vw_ctx.handle_obj_versions_put(
                    req, object_versions, object_name, policy_idx)
            else:  # handle DELETE
                resp = vw_ctx.handle_obj_versions_delete(
                    req, object_versions, account_name,
                    container_name, object_name)

        if resp:
            return resp
        else:
            return self.app
Esempio n. 6
0
    def object_request(self, req, version, account, container, obj,
                       allow_versioned_writes):
        account_name = unquote(account)
        container_name = unquote(container)
        object_name = unquote(obj)
        container_info = None
        resp = None
        is_enabled = config_true_value(allow_versioned_writes)
        if req.method in ('PUT', 'DELETE'):
            container_info = get_container_info(
                req.environ, self.app)
        elif req.method == 'COPY' and 'Destination' in req.headers:
            if 'Destination-Account' in req.headers:
                account_name = req.headers.get('Destination-Account')
                account_name = check_account_format(req, account_name)
            container_name, object_name = check_destination_header(req)
            req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
                version, account_name, container_name, object_name)
            container_info = get_container_info(
                req.environ, self.app)

        if not container_info:
            return self.app

        # To maintain backwards compatibility, container version
        # location could be stored as sysmeta or not, need to check both.
        # If stored as sysmeta, check if middleware is enabled. If sysmeta
        # is not set, but versions property is set in container_info, then
        # for backwards compatibility feature is enabled.
        object_versions = container_info.get(
            'sysmeta', {}).get('versions-location')
        if object_versions and isinstance(object_versions, unicode):
            object_versions = object_versions.encode('utf-8')
        elif not object_versions:
            object_versions = container_info.get('versions')
            # if allow_versioned_writes is not set in the configuration files
            # but 'versions' is configured, enable feature to maintain
            # backwards compatibility
            if not allow_versioned_writes and object_versions:
                is_enabled = True

        if is_enabled and object_versions:
            object_versions = unquote(object_versions)
            vw_ctx = VersionedWritesContext(self.app, self.logger)
            if req.method in ('PUT', 'COPY'):
                policy_idx = req.headers.get(
                    'X-Backend-Storage-Policy-Index',
                    container_info['storage_policy'])
                resp = vw_ctx.handle_obj_versions_put(
                    req, object_versions, object_name, policy_idx)
            else:  # handle DELETE
                resp = vw_ctx.handle_obj_versions_delete(
                    req, object_versions, account_name,
                    container_name, object_name)

        if resp:
            return resp
        else:
            return self.app
Esempio n. 7
0
def _check_symlink_header(req):
    """
    Validate that the value from x-symlink-target header is
    well formatted. We assume the caller ensures that
    x-symlink-target header is present in req.headers.

    :param req: HTTP request object
    :raise: HTTPPreconditionFailed if x-symlink-target value
            is not well formatted.
    :raise: HTTPBadRequest if the x-symlink-target value points to the request
            path.
    """
    # N.B. check_path_header doesn't assert the leading slash and
    # copy middleware may accept the format. In the symlink, API
    # says apparently to use "container/object" format so add the
    # validation first, here.
    error_body = 'X-Symlink-Target header must be of the form ' \
                 '<container name>/<object name>'
    try:
        if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'):
            raise HTTPPreconditionFailed(body=error_body,
                                         request=req,
                                         content_type='text/plain')
    except TypeError:
        raise HTTPPreconditionFailed(body=error_body,
                                     request=req,
                                     content_type='text/plain')

    # check container and object format
    container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body)
    req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj))

    # Check account format if it exists
    try:
        account = check_account_format(
            req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \
            if TGT_ACCT_SYMLINK_HDR in req.headers else None
    except TypeError:
        raise HTTPPreconditionFailed(
            body='Account name cannot contain slashes',
            request=req,
            content_type='text/plain')

    # Extract request path
    _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)

    if account:
        req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account)
    else:
        account = req_acc

    # Check if symlink targets the symlink itself or not
    if (account, container, obj) == (req_acc, req_cont, req_obj):
        raise HTTPBadRequest(body='Symlink cannot target itself',
                             request=req,
                             content_type='text/plain')
Esempio n. 8
0
    def object_request(self, req, api_version, account, container, obj,
                       allow_versioned_writes):
        account_name = unquote(account)
        container_name = unquote(container)
        object_name = unquote(obj)
        container_info = None
        resp = None
        is_enabled = config_true_value(allow_versioned_writes)
        if req.method in ('PUT', 'DELETE'):
            container_info = get_container_info(req.environ, self.app)
        elif req.method == 'COPY' and 'Destination' in req.headers:
            if 'Destination-Account' in req.headers:
                account_name = req.headers.get('Destination-Account')
                account_name = check_account_format(req, account_name)
            container_name, object_name = check_destination_header(req)
            req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
                api_version, account_name, container_name, object_name)
            container_info = get_container_info(req.environ, self.app)

        if not container_info:
            return self.app

        # To maintain backwards compatibility, container version
        # location could be stored as sysmeta or not, need to check both.
        # If stored as sysmeta, check if middleware is enabled. If sysmeta
        # is not set, but versions property is set in container_info, then
        # for backwards compatibility feature is enabled.
        versions_cont = container_info.get('sysmeta',
                                           {}).get('versions-location')
        if not versions_cont:
            versions_cont = container_info.get('versions')
            # if allow_versioned_writes is not set in the configuration files
            # but 'versions' is configured, enable feature to maintain
            # backwards compatibility
            if not allow_versioned_writes and versions_cont:
                is_enabled = True

        if is_enabled and versions_cont:
            versions_cont = unquote(versions_cont).split('/')[0]
            vw_ctx = VersionedWritesContext(self.app, self.logger)
            if req.method in ('PUT', 'COPY'):
                resp = vw_ctx.handle_obj_versions_put(req, versions_cont,
                                                      api_version,
                                                      account_name,
                                                      object_name)
            else:  # handle DELETE
                resp = vw_ctx.handle_obj_versions_delete(
                    req, versions_cont, api_version, account_name,
                    container_name, object_name)

        if resp:
            return resp
        else:
            return self.app
Esempio n. 9
0
def _check_symlink_header(req):
    """
    Validate that the value from x-symlink-target header is
    well formatted. We assume the caller ensures that
    x-symlink-target header is present in req.headers.

    :param req: HTTP request object
    :raise: HTTPPreconditionFailed if x-symlink-target value
            is not well formatted.
    :raise: HTTPBadRequest if the x-symlink-target value points to the request
            path.
    """
    # N.B. check_path_header doesn't assert the leading slash and
    # copy middleware may accept the format. In the symlink, API
    # says apparently to use "container/object" format so add the
    # validation first, here.
    if unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'):
        raise HTTPPreconditionFailed(
            body='X-Symlink-Target header must be of the '
                 'form <container name>/<object name>',
            request=req, content_type='text/plain')

    # check container and object format
    container, obj = check_path_header(
        req, TGT_OBJ_SYMLINK_HDR, 2,
        'X-Symlink-Target header must be of the '
        'form <container name>/<object name>')

    # Check account format if it exists
    account = check_account_format(
        req, unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \
        if TGT_ACCT_SYMLINK_HDR in req.headers else None

    # Extract request path
    _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)

    if not account:
        account = req_acc

    # Check if symlink targets the symlink itself or not
    if (account, container, obj) == (req_acc, req_cont, req_obj):
        raise HTTPBadRequest(
            body='Symlink cannot target itself',
            request=req, content_type='text/plain')
Esempio n. 10
0
    def __call__(self, env, start_response):
        req = Request(env)

        if req.method not in self.write_methods:
            return self.app(env, start_response)

        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method == 'COPY' and 'Destination-Account' in req.headers:
            dest_account = req.headers.get('Destination-Account')
            account = check_account_format(req, dest_account)

        if self.account_read_only(req, account):
            msg = 'Writes are disabled for this account.'
            return HTTPMethodNotAllowed(body=msg)(env, start_response)

        return self.app(env, start_response)
Esempio n. 11
0
    def __call__(self, env, start_response):
        req = Request(env)

        if req.method not in self.write_methods:
            return self.app(env, start_response)

        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method == 'COPY' and 'Destination-Account' in req.headers:
            dest_account = req.headers.get('Destination-Account')
            account = check_account_format(req, dest_account)

        if self.account_read_only(req, account):
            msg = 'Writes are disabled for this account.'
            return HTTPMethodNotAllowed(body=msg)(env, start_response)

        return self.app(env, start_response)
Esempio n. 12
0
    def PUT(self, req):
        """HTTP PUT request handler."""
        if req.if_none_match is not None and '*' not in req.if_none_match:
            # Sending an etag with if-none-match isn't currently supported
            return HTTPBadRequest(request=req, content_type='text/plain',
                                  body='If-None-Match only supports *')
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp

        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        detect_content_type = \
            config_true_value(req.headers.get('x-detect-content-type'))
        if detect_content_type or not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                                          'application/octet-stream'
            if detect_content_type:
                req.headers.pop('x-detect-content-type')
            else:
                content_type_manually_set = False

        error_response = check_object_creation(req, self.object_name) or \
                         check_content_type(req)
        if error_response:
            return error_response

        req.headers['X-Timestamp'] = Timestamp(time.time()).internal

        stream = req.environ['wsgi.input']
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            if req.environ.get('swift.orig_req_method', req.method) != 'POST':
                req.environ.setdefault('swift.log_info', []).append(
                    'x-copy-from:%s' % source_header)
            ver, acct, _rest = req.split_path(2, 3, True)
            src_account_name = req.headers.get('X-Copy-From-Account', None)
            if src_account_name:
                src_account_name = check_account_format(req, src_account_name)
            else:
                src_account_name = acct
            src_container_name, src_obj_name = check_copy_from_header(req)
            source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
                                              src_container_name, src_obj_name)
            source_req = req.copy_get()

            # make sure the source request uses it's container_info
            source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            orig_account_name = self.account_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            self.account_name = src_account_name
            sink_req = Request.blank(req.path_info,
                                     environ=req.environ, headers=req.headers)
            source_resp = self.GET(source_req)

            # This gives middlewares a way to change the source; for example,
            # this lets you COPY a SLO manifest and have the new object be the
            # concatenation of the segments (like what a GET request gives
            # the client), not a copy of the manifest file.
            hook = req.environ.get(
                'swift.copy_hook',
                (lambda source_req, source_resp, sink_req: source_resp))
            source_resp = hook(source_req, source_resp, sink_req)

            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            self.account_name = orig_account_name
            stream = IterO(source_resp.app_iter)
            sink_req.content_length = source_resp.content_length
            if sink_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            if sink_req.content_length > constraints.MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=req)
            sink_req.etag = source_resp.etag

            # we no longer need the X-Copy-From header
            del sink_req.headers['X-Copy-From']
            if 'X-Copy-From-Account' in sink_req.headers:
                del sink_req.headers['X-Copy-From-Account']
            if not content_type_manually_set:
                sink_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if config_true_value(
                    sink_req.headers.get('x-fresh-metadata', 'false')):
                # post-as-copy: ignore new sysmeta, copy existing sysmeta
                condition = lambda k: is_sys_meta('object', k)
                remove_items(sink_req.headers, condition)
                copy_header_subset(source_resp, sink_req, condition)
            else:
                # copy/update existing sysmeta and user meta
                copy_headers_into(source_resp, sink_req)
                copy_headers_into(req, sink_req)

            # copy over x-static-large-object for POSTs and manifest copies
            if 'X-Static-Large-Object' in source_resp.headers and \
                            req.params.get('multipart-manifest') == 'get':
                sink_req.headers['X-Static-Large-Object'] = \
                    source_resp.headers['X-Static-Large-Object']

            req = sink_req

        content_length = req.content_length
        content_type = req.headers.get('content-type', 'octet/stream')
        storage = self.app.storage

        if content_length is None:
            content_length = 0
        try:
            chunks, size, checksum = storage.object_create(self.account_name, self.container_name,
                                  obj_name=self.object_name,
                                  file_or_path=stream,
                                  content_length=content_length,
                                  content_type=content_type)
        except exceptions.NoSuchContainer:
            return HTTPNotFound(request=req)
        except exceptions.ClientReadTimeout:
            return HTTPRequestTimeout(request=req)
        resp = HTTPCreated(request=req, etag=checksum)
        if source_header:
            acct, path = source_header.split('/', 3)[2:4]
            resp.headers['X-Copied-From-Account'] = quote(acct)
            resp.headers['X-Copied-From'] = quote(path)
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
        resp.last_modified = math.ceil(
            float(Timestamp(req.headers['X-Timestamp'])))
        return resp
Esempio n. 13
0
    def PUT(self, req):
        """HTTP PUT request handler."""
        if req.if_none_match is not None and '*' not in req.if_none_match:
            # Sending an etag with if-none-match isn't currently supported
            return HTTPBadRequest(request=req, content_type='text/plain',
                                  body='If-None-Match only supports *')
        container_info = self.container_info(
            self.account_name, self.container_name, req)
        policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                       container_info['storage_policy'])
        obj_ring = self.app.get_object_ring(policy_index)

        # pass the policy index to storage nodes via req header
        req.headers['X-Backend-Storage-Policy-Index'] = policy_index
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp

        if not containers:
            return HTTPNotFound(request=req)

        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        detect_content_type = \
            config_true_value(req.headers.get('x-detect-content-type'))
        if detect_content_type or not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                'application/octet-stream'
            if detect_content_type:
                req.headers.pop('x-detect-content-type')
            else:
                content_type_manually_set = False

        error_response = check_object_creation(req, self.object_name) or \
            check_content_type(req)
        if error_response:
            return error_response

        partition, nodes = obj_ring.get_nodes(
            self.account_name, self.container_name, self.object_name)

        # do a HEAD request for container sync and checking object versions
        if 'x-timestamp' in req.headers or \
                (object_versions and not
                 req.environ.get('swift_versioned_copy')):
            # make sure proxy-server uses the right policy index
            _headers = {'X-Backend-Storage-Policy-Index': policy_index,
                        'X-Newest': 'True'}
            hreq = Request.blank(req.path_info, headers=_headers,
                                 environ={'REQUEST_METHOD': 'HEAD'})
            hresp = self.GETorHEAD_base(
                hreq, _('Object'), obj_ring, partition,
                hreq.swift_entity_path)

        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req_timestamp = Timestamp(req.headers['X-Timestamp'])
                if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
                        hresp.environ['swift_x_timestamp'] >= req_timestamp:
                    return HTTPAccepted(request=req)
            except ValueError:
                return HTTPBadRequest(
                    request=req, content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                         'was %r' % req.headers['x-timestamp'])
            req.headers['X-Timestamp'] = req_timestamp.internal
        else:
            req.headers['X-Timestamp'] = Timestamp(time.time()).internal

        if object_versions and not req.environ.get('swift_versioned_copy'):
            if hresp.status_int != HTTP_NOT_FOUND:
                # This is a version manifest and needs to be handled
                # differently. First copy the existing data to a new object,
                # then write the data from this request to the version manifest
                # object.
                lcontainer = object_versions.split('/')[0]
                prefix_len = '%03x' % len(self.object_name)
                lprefix = prefix_len + self.object_name + '/'
                ts_source = hresp.environ.get('swift_x_timestamp')
                if ts_source is None:
                    ts_source = time.mktime(time.strptime(
                                            hresp.headers['last-modified'],
                                            '%a, %d %b %Y %H:%M:%S GMT'))
                new_ts = Timestamp(ts_source).internal
                vers_obj_name = lprefix + new_ts
                copy_headers = {
                    'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
                copy_environ = {'REQUEST_METHOD': 'COPY',
                                'swift_versioned_copy': True
                                }
                copy_req = Request.blank(req.path_info, headers=copy_headers,
                                         environ=copy_environ)
                copy_resp = self.COPY(copy_req)
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

        reader = req.environ['wsgi.input'].read
        data_source = iter(lambda: reader(self.app.client_chunk_size), '')
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            if req.environ.get('swift.orig_req_method', req.method) != 'POST':
                req.environ.setdefault('swift.log_info', []).append(
                    'x-copy-from:%s' % source_header)
            ver, acct, _rest = req.split_path(2, 3, True)
            src_account_name = req.headers.get('X-Copy-From-Account', None)
            if src_account_name:
                src_account_name = check_account_format(req, src_account_name)
            else:
                src_account_name = acct
            src_container_name, src_obj_name = check_copy_from_header(req)
            source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
                            src_container_name, src_obj_name)
            source_req = req.copy_get()

            # make sure the source request uses it's container_info
            source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            orig_account_name = self.account_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            self.account_name = src_account_name
            sink_req = Request.blank(req.path_info,
                                     environ=req.environ, headers=req.headers)
            source_resp = self.GET(source_req)

            # This gives middlewares a way to change the source; for example,
            # this lets you COPY a SLO manifest and have the new object be the
            # concatenation of the segments (like what a GET request gives
            # the client), not a copy of the manifest file.
            hook = req.environ.get(
                'swift.copy_hook',
                (lambda source_req, source_resp, sink_req: source_resp))
            source_resp = hook(source_req, source_resp, sink_req)

            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            self.account_name = orig_account_name
            data_source = iter(source_resp.app_iter)
            sink_req.content_length = source_resp.content_length
            if sink_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            if sink_req.content_length > constraints.MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=req)
            sink_req.etag = source_resp.etag

            # we no longer need the X-Copy-From header
            del sink_req.headers['X-Copy-From']
            if 'X-Copy-From-Account' in sink_req.headers:
                del sink_req.headers['X-Copy-From-Account']
            if not content_type_manually_set:
                sink_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if config_true_value(
                    sink_req.headers.get('x-fresh-metadata', 'false')):
                # post-as-copy: ignore new sysmeta, copy existing sysmeta
                condition = lambda k: is_sys_meta('object', k)
                remove_items(sink_req.headers, condition)
                copy_header_subset(source_resp, sink_req, condition)
            else:
                # copy/update existing sysmeta and user meta
                copy_headers_into(source_resp, sink_req)
                copy_headers_into(req, sink_req)

            # copy over x-static-large-object for POSTs and manifest copies
            if 'X-Static-Large-Object' in source_resp.headers and \
                    req.params.get('multipart-manifest') == 'get':
                sink_req.headers['X-Static-Large-Object'] = \
                    source_resp.headers['X-Static-Large-Object']

            req = sink_req

        req, delete_at_container, delete_at_part, \
            delete_at_nodes = self._config_obj_expiration(req)

        node_iter = GreenthreadSafeIterator(
            self.iter_nodes_local_first(obj_ring, partition))
        pile = GreenPile(len(nodes))
        te = req.headers.get('transfer-encoding', '')
        chunked = ('chunked' in te)

        outgoing_headers = self._backend_requests(
            req, len(nodes), container_partition, containers,
            delete_at_container, delete_at_part, delete_at_nodes)

        for nheaders in outgoing_headers:
            # RFC2616:8.2.3 disallows 100-continue without a body
            if (req.content_length > 0) or chunked:
                nheaders['Expect'] = '100-continue'
            pile.spawn(self._connect_put_node, node_iter, partition,
                       req.swift_entity_path, nheaders,
                       self.app.logger.thread_locals)

        conns = [conn for conn in pile if conn]
        min_conns = quorum_size(len(nodes))

        if req.if_none_match is not None and '*' in req.if_none_match:
            statuses = [conn.resp.status for conn in conns if conn.resp]
            if HTTP_PRECONDITION_FAILED in statuses:
                # If we find any copy of the file, it shouldn't be uploaded
                self.app.logger.debug(
                    _('Object PUT returning 412, %(statuses)r'),
                    {'statuses': statuses})
                return HTTPPreconditionFailed(request=req)

        if len(conns) < min_conns:
            self.app.logger.error(
                _('Object PUT returning 503, %(conns)s/%(nodes)s '
                  'required connections'),
                {'conns': len(conns), 'nodes': min_conns})
            return HTTPServiceUnavailable(request=req)
        bytes_transferred = 0
        try:
            with ContextPool(len(nodes)) as pool:
                for conn in conns:
                    conn.failed = False
                    conn.queue = Queue(self.app.put_queue_depth)
                    pool.spawn(self._send_file, conn, req.path)
                while True:
                    with ChunkReadTimeout(self.app.client_timeout):
                        try:
                            chunk = next(data_source)
                        except StopIteration:
                            if chunked:
                                for conn in conns:
                                    conn.queue.put('0\r\n\r\n')
                            break
                    bytes_transferred += len(chunk)
                    if bytes_transferred > constraints.MAX_FILE_SIZE:
                        return HTTPRequestEntityTooLarge(request=req)
                    for conn in list(conns):
                        if not conn.failed:
                            conn.queue.put(
                                '%x\r\n%s\r\n' % (len(chunk), chunk)
                                if chunked else chunk)
                        else:
                            conns.remove(conn)
                    if len(conns) < min_conns:
                        self.app.logger.error(_(
                            'Object PUT exceptions during'
                            ' send, %(conns)s/%(nodes)s required connections'),
                            {'conns': len(conns), 'nodes': min_conns})
                        return HTTPServiceUnavailable(request=req)
                for conn in conns:
                    if conn.queue.unfinished_tasks:
                        conn.queue.join()
            conns = [conn for conn in conns if not conn.failed]
        except ChunkReadTimeout as err:
            self.app.logger.warn(
                _('ERROR Client read timeout (%ss)'), err.seconds)
            self.app.logger.increment('client_timeouts')
            return HTTPRequestTimeout(request=req)
        except (Exception, Timeout):
            self.app.logger.exception(
                _('ERROR Exception causing client disconnect'))
            return HTTPClientDisconnect(request=req)
        if req.content_length and bytes_transferred < req.content_length:
            req.client_disconnect = True
            self.app.logger.warn(
                _('Client disconnected without sending enough data'))
            self.app.logger.increment('client_disconnects')
            return HTTPClientDisconnect(request=req)

        statuses, reasons, bodies, etags = self._get_put_responses(req, conns,
                                                                   nodes)

        if len(etags) > 1:
            self.app.logger.error(
                _('Object servers returned %s mismatched etags'), len(etags))
            return HTTPServerError(request=req)
        etag = etags.pop() if len(etags) else None
        resp = self.best_response(req, statuses, reasons, bodies,
                                  _('Object PUT'), etag=etag)
        if source_header:
            acct, path = source_header.split('/', 3)[2:4]
            resp.headers['X-Copied-From-Account'] = quote(acct)
            resp.headers['X-Copied-From'] = quote(path)
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
        resp.last_modified = math.ceil(
            float(Timestamp(req.headers['X-Timestamp'])))
        return resp
Esempio n. 14
0
    def handle_PUT(self, req, start_response):
        if req.content_length:
            return HTTPBadRequest(body='Copy requests require a zero byte '
                                  'body',
                                  request=req,
                                  content_type='text/plain')(req.environ,
                                                             start_response)

        # Form the path of source object to be fetched
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get('X-Copy-From-Account')
        if src_account_name:
            src_account_name = check_account_format(req, src_account_name)
        else:
            src_account_name = acct
        src_container_name, src_obj_name = _check_copy_from_header(req)
        source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
                                        src_container_name, src_obj_name)

        if req.environ.get('swift.orig_req_method', req.method) != 'POST':
            self.logger.info("Copying object from %s to %s" %
                             (source_path, req.path))

        # GET the source object, bail out on error
        ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
        source_resp = self._get_source_object(ssc_ctx, source_path, req)
        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            return source_resp(source_resp.environ, start_response)

        # Create a new Request object based on the original request instance.
        # This will preserve original request environ including headers.
        sink_req = Request.blank(req.path_info, environ=req.environ)

        def is_object_sysmeta(k):
            return is_sys_meta('object', k)

        if config_true_value(req.headers.get('x-fresh-metadata', 'false')):
            # x-fresh-metadata only applies to copy, not post-as-copy: ignore
            # existing user metadata, update existing sysmeta with new
            copy_header_subset(source_resp, sink_req, is_object_sysmeta)
            copy_header_subset(req, sink_req, is_object_sysmeta)
        else:
            # First copy existing sysmeta, user meta and other headers from the
            # source to the sink, apart from headers that are conditionally
            # copied below and timestamps.
            exclude_headers = ('x-static-large-object', 'x-object-manifest',
                               'etag', 'content-type', 'x-timestamp',
                               'x-backend-timestamp')
            copy_header_subset(source_resp, sink_req,
                               lambda k: k.lower() not in exclude_headers)
            # now update with original req headers
            sink_req.headers.update(req.headers)

        params = sink_req.params
        if params.get('multipart-manifest') == 'get':
            if 'X-Static-Large-Object' in source_resp.headers:
                params['multipart-manifest'] = 'put'
            if 'X-Object-Manifest' in source_resp.headers:
                del params['multipart-manifest']
                sink_req.headers['X-Object-Manifest'] = \
                    source_resp.headers['X-Object-Manifest']
            sink_req.params = params

        # Set swift.source, data source, content length and etag
        # for the PUT request
        sink_req.environ['swift.source'] = 'SSC'
        sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        if (source_resp.status_int == HTTP_OK
                and 'X-Static-Large-Object' not in source_resp.headers
                and ('X-Object-Manifest' not in source_resp.headers
                     or req.params.get('multipart-manifest') == 'get')):
            # copy source etag so that copied content is verified, unless:
            #  - not a 200 OK response: source etag may not match the actual
            #    content, for example with a 206 Partial Content response to a
            #    ranged request
            #  - SLO manifest: etag cannot be specified in manifest PUT; SLO
            #    generates its own etag value which may differ from source
            #  - SLO: etag in SLO response is not hash of actual content
            #  - DLO: etag in DLO response is not hash of actual content
            sink_req.headers['Etag'] = source_resp.etag
        else:
            # since we're not copying the source etag, make sure that any
            # container update override values are not copied.
            remove_items(
                sink_req.headers, lambda k: k.startswith(
                    'X-Object-Sysmeta-Container-Update-Override-'))

        # We no longer need these headers
        sink_req.headers.pop('X-Copy-From', None)
        sink_req.headers.pop('X-Copy-From-Account', None)

        # If the copy request does not explicitly override content-type,
        # use the one present in the source object.
        if not req.headers.get('content-type'):
            sink_req.headers['Content-Type'] = \
                source_resp.headers['Content-Type']

        # Create response headers for PUT response
        resp_headers = self._create_response_headers(source_path, source_resp,
                                                     sink_req)

        put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
        close_if_possible(source_resp.app_iter)
        return put_resp
Esempio n. 15
0
File: copy.py Progetto: clayg/swift
    def handle_PUT(self, req, start_response):
        if req.content_length:
            return HTTPBadRequest(body='Copy requests require a zero byte '
                                  'body', request=req,
                                  content_type='text/plain')(req.environ,
                                                             start_response)

        # Form the path of source object to be fetched
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get('X-Copy-From-Account')
        if src_account_name:
            src_account_name = check_account_format(req, src_account_name)
        else:
            src_account_name = acct
        src_container_name, src_obj_name = _check_copy_from_header(req)
        source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
                                        src_container_name, src_obj_name)

        if req.environ.get('swift.orig_req_method', req.method) != 'POST':
            self.logger.info("Copying object from %s to %s" %
                             (source_path, req.path))

        # GET the source object, bail out on error
        ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
        source_resp = self._get_source_object(ssc_ctx, source_path, req)
        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            return source_resp(source_resp.environ, start_response)

        # Create a new Request object based on the original request instance.
        # This will preserve original request environ including headers.
        sink_req = Request.blank(req.path_info, environ=req.environ)

        def is_object_sysmeta(k):
            return is_sys_meta('object', k)

        if 'swift.post_as_copy' in sink_req.environ:
            # Post-as-copy: ignore new sysmeta, copy existing sysmeta
            remove_items(sink_req.headers, is_object_sysmeta)
            copy_header_subset(source_resp, sink_req, is_object_sysmeta)
        elif config_true_value(req.headers.get('x-fresh-metadata', 'false')):
            # x-fresh-metadata only applies to copy, not post-as-copy: ignore
            # existing user metadata, update existing sysmeta with new
            copy_header_subset(source_resp, sink_req, is_object_sysmeta)
            copy_header_subset(req, sink_req, is_object_sysmeta)
        else:
            # First copy existing sysmeta, user meta and other headers from the
            # source to the sink, apart from headers that are conditionally
            # copied below and timestamps.
            exclude_headers = ('x-static-large-object', 'x-object-manifest',
                               'etag', 'content-type', 'x-timestamp',
                               'x-backend-timestamp')
            copy_header_subset(source_resp, sink_req,
                               lambda k: k.lower() not in exclude_headers)
            # now update with original req headers
            sink_req.headers.update(req.headers)

        params = sink_req.params
        if params.get('multipart-manifest') == 'get':
            if 'X-Static-Large-Object' in source_resp.headers:
                params['multipart-manifest'] = 'put'
            if 'X-Object-Manifest' in source_resp.headers:
                del params['multipart-manifest']
                if 'swift.post_as_copy' not in sink_req.environ:
                    sink_req.headers['X-Object-Manifest'] = \
                        source_resp.headers['X-Object-Manifest']
            sink_req.params = params

        # Set swift.source, data source, content length and etag
        # for the PUT request
        sink_req.environ['swift.source'] = 'SSC'
        sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        if (source_resp.status_int == HTTP_OK and
                'X-Static-Large-Object' not in source_resp.headers and
                ('X-Object-Manifest' not in source_resp.headers or
                 req.params.get('multipart-manifest') == 'get')):
            # copy source etag so that copied content is verified, unless:
            #  - not a 200 OK response: source etag may not match the actual
            #    content, for example with a 206 Partial Content response to a
            #    ranged request
            #  - SLO manifest: etag cannot be specified in manifest PUT; SLO
            #    generates its own etag value which may differ from source
            #  - SLO: etag in SLO response is not hash of actual content
            #  - DLO: etag in DLO response is not hash of actual content
            sink_req.headers['Etag'] = source_resp.etag
        else:
            # since we're not copying the source etag, make sure that any
            # container update override values are not copied.
            remove_items(sink_req.headers, lambda k: k.startswith(
                'X-Object-Sysmeta-Container-Update-Override-'))

        # We no longer need these headers
        sink_req.headers.pop('X-Copy-From', None)
        sink_req.headers.pop('X-Copy-From-Account', None)

        # If the copy request does not explicitly override content-type,
        # use the one present in the source object.
        if not req.headers.get('content-type'):
            sink_req.headers['Content-Type'] = \
                source_resp.headers['Content-Type']

        # Create response headers for PUT response
        resp_headers = self._create_response_headers(source_path,
                                                     source_resp, sink_req)

        put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
        close_if_possible(source_resp.app_iter)
        return put_resp
Esempio n. 16
0
    def PUT(self, req):
        """HTTP PUT request handler."""
        if req.if_none_match is not None and '*' not in req.if_none_match:
            # Sending an etag with if-none-match isn't currently supported
            return HTTPBadRequest(request=req,
                                  content_type='text/plain',
                                  body='If-None-Match only supports *')
        container_info = self.container_info(self.account_name,
                                             self.container_name, req)
        policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                       container_info['storage_policy'])
        obj_ring = self.app.get_object_ring(policy_index)

        # pass the policy index to storage nodes via req header
        req.headers['X-Backend-Storage-Policy-Index'] = policy_index
        container_partition = container_info['partition']
        containers = container_info['nodes']
        req.acl = container_info['write_acl']
        req.environ['swift_sync_key'] = container_info['sync_key']
        object_versions = container_info['versions']
        if 'swift.authorize' in req.environ:
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp

        if not containers:
            return HTTPNotFound(request=req)

        # Sometimes the 'content-type' header exists, but is set to None.
        content_type_manually_set = True
        detect_content_type = \
            config_true_value(req.headers.get('x-detect-content-type'))
        if detect_content_type or not req.headers.get('content-type'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            req.headers['Content-Type'] = guessed_type or \
                'application/octet-stream'
            if detect_content_type:
                req.headers.pop('x-detect-content-type')
            else:
                content_type_manually_set = False

        error_response = check_object_creation(req, self.object_name) or \
            check_content_type(req)
        if error_response:
            return error_response

        partition, nodes = obj_ring.get_nodes(self.account_name,
                                              self.container_name,
                                              self.object_name)
        ####################################  CHANGED_CODE  ############################################################
        # Change the nodes list to contain only one dictionary item instead of the original 3 returned by the ring.
        d = dict()
        # d[partition] = nodes[1:]
        # f.write(str(d)+"\n")
        # f.close()
        print("===Original Nodes===")
        print(nodes)
        temp_nodes = []
        flag = 0
        f = open("/home/hduser/swift/swift/proxy/controllers/spindowndevices",
                 "r")
        sdlist = f.read().split("\n")
        print("===Spun down devices===:", sdlist)
        f.close()

        upnodes = [item for item in nodes if item['device'] not in sdlist]
        downnodes = [item for item in nodes if item['device'] in sdlist]
        temp_nodes = upnodes
        if (len(downnodes) > 0):
            d = ast.literal_eval(
                open("/home/hduser/swift/swift/proxy/controllers/nodes.txt",
                     "r").read())
            # d_temp=pickle.load("/home/hduser/swift/proxy/controllers/nodes.p","rb")
            # print("===Current dict===:",d)
            for item in downnodes:
                if (partition in d):
                    d[partition].append(item)
                    # print("===Modified dict===:",d)
                else:
                    d[partition] = [item]
                    # print("===Modified dict===:",d)
        # pickle.dump(d,open("/home/hduser/nodes.p","wb"))
        # print("Before writing:",d)
        fo = open("/home/hduser/swift/swift/proxy/controllers/nodes.txt", "w")
        fo.write(str(d) + "\n")
        fo.close()
        # pickle.dump(d,open("/home/hduser/swift/swift/proxy/controllers/nodes.p","wb"))
        ## Old method, IGNORE
        # for item in nodes:
        #     device = item['device']
        #     if(device not in sdlist):
        #     # if(os.path.ismount("path"))
        #         temp_nodes.append(item)
        #         flag = 1
        #         break
        #     else:
        #         pickle.dump(d,open("/home/hduser/nodes.p","wb"))
        #         # d = pickle.load(open("/home/hduser/nodes.p","rb"))
        #         import ast
        #         d = ast.literal_eval(open("/home/hduser/nodes.txt","r").read())
        #         print("===Current dict===:",d)
        #         if(partition in d):
        #             print("In IF")
        #             d[partition].append(item)
        #             print("===Modified dict===:",d)
        #         else:
        #             print("In ELSE")
        #             d[partition] = [item]
        #             print("===Modified dict===:",d)
        #         pickle.dump(d,open("/home/hduser/nodes.p","wb"))
        #         fo = open("/home/hduser/nodes.txt","w")
        #         fo.write(str(d)+"\n")

        # Code to spin up a device if none are running already.
        if (len(upnodes) == 0):
            dev = nodes[0]['device']
            print("===ALL NODES DOWN===")
            print("===Mounting device===", dev)
            os.system("mount /dev/" + str(dev))

        print('===In controller PUT===:')
        print("===Partition===", partition)
        nodes = temp_nodes

        print('===In controller PUT===:')
        print("===Partition===", partition)
        nodes = temp_nodes
        print("===Nodes===:", nodes)

        check_ssd()
        ############################################  CHANGED_CODE  ########################################################

        # do a HEAD request for checking object versions
        if object_versions and not req.environ.get('swift_versioned_copy'):
            # make sure proxy-server uses the right policy index
            _headers = {
                'X-Backend-Storage-Policy-Index': policy_index,
                'X-Newest': 'True'
            }
            hreq = Request.blank(req.path_info,
                                 headers=_headers,
                                 environ={'REQUEST_METHOD': 'HEAD'})
            hresp = self.GETorHEAD_base(hreq, _('Object'), obj_ring, partition,
                                        hreq.swift_entity_path)

        # Used by container sync feature
        if 'x-timestamp' in req.headers:
            try:
                req_timestamp = Timestamp(req.headers['X-Timestamp'])
            except ValueError:
                return HTTPBadRequest(
                    request=req,
                    content_type='text/plain',
                    body='X-Timestamp should be a UNIX timestamp float value; '
                    'was %r' % req.headers['x-timestamp'])
            req.headers['X-Timestamp'] = req_timestamp.internal
        else:
            req.headers['X-Timestamp'] = Timestamp(time.time()).internal

        if object_versions and not req.environ.get('swift_versioned_copy'):
            is_manifest = 'X-Object-Manifest' in req.headers or \
                          'X-Object-Manifest' in hresp.headers
            if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
                # This is a version manifest and needs to be handled
                # differently. First copy the existing data to a new object,
                # then write the data from this request to the version manifest
                # object.
                lcontainer = object_versions.split('/')[0]
                prefix_len = '%03x' % len(self.object_name)
                lprefix = prefix_len + self.object_name + '/'
                ts_source = hresp.environ.get('swift_x_timestamp')
                if ts_source is None:
                    ts_source = time.mktime(
                        time.strptime(hresp.headers['last-modified'],
                                      '%a, %d %b %Y %H:%M:%S GMT'))
                new_ts = Timestamp(ts_source).internal
                vers_obj_name = lprefix + new_ts
                copy_headers = {
                    'Destination': '%s/%s' % (lcontainer, vers_obj_name)
                }
                copy_environ = {
                    'REQUEST_METHOD': 'COPY',
                    'swift_versioned_copy': True
                }
                copy_req = Request.blank(req.path_info,
                                         headers=copy_headers,
                                         environ=copy_environ)
                copy_resp = self.COPY(copy_req)
                if is_client_error(copy_resp.status_int):
                    # missing container or bad permissions
                    return HTTPPreconditionFailed(request=req)
                elif not is_success(copy_resp.status_int):
                    # could not copy the data, bail
                    return HTTPServiceUnavailable(request=req)

        reader = req.environ['wsgi.input'].read
        data_source = iter(lambda: reader(self.app.client_chunk_size), '')
        source_header = req.headers.get('X-Copy-From')
        source_resp = None
        if source_header:
            if req.environ.get('swift.orig_req_method', req.method) != 'POST':
                req.environ.setdefault('swift.log_info', []).append(
                    'x-copy-from:%s' % source_header)
            ver, acct, _rest = req.split_path(2, 3, True)
            src_account_name = req.headers.get('X-Copy-From-Account', None)
            if src_account_name:
                src_account_name = check_account_format(req, src_account_name)
            else:
                src_account_name = acct
            src_container_name, src_obj_name = check_copy_from_header(req)
            source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
                                              src_container_name, src_obj_name)
            source_req = req.copy_get()

            # make sure the source request uses it's container_info
            source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
            source_req.path_info = source_header
            source_req.headers['X-Newest'] = 'true'
            orig_obj_name = self.object_name
            orig_container_name = self.container_name
            orig_account_name = self.account_name
            self.object_name = src_obj_name
            self.container_name = src_container_name
            self.account_name = src_account_name
            sink_req = Request.blank(req.path_info,
                                     environ=req.environ,
                                     headers=req.headers)
            source_resp = self.GET(source_req)

            # This gives middlewares a way to change the source; for example,
            # this lets you COPY a SLO manifest and have the new object be the
            # concatenation of the segments (like what a GET request gives
            # the client), not a copy of the manifest file.
            hook = req.environ.get(
                'swift.copy_hook',
                (lambda source_req, source_resp, sink_req: source_resp))
            source_resp = hook(source_req, source_resp, sink_req)

            if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
                return source_resp
            self.object_name = orig_obj_name
            self.container_name = orig_container_name
            self.account_name = orig_account_name
            data_source = iter(source_resp.app_iter)
            sink_req.content_length = source_resp.content_length
            if sink_req.content_length is None:
                # This indicates a transfer-encoding: chunked source object,
                # which currently only happens because there are more than
                # CONTAINER_LISTING_LIMIT segments in a segmented object. In
                # this case, we're going to refuse to do the server-side copy.
                return HTTPRequestEntityTooLarge(request=req)
            if sink_req.content_length > constraints.MAX_FILE_SIZE:
                return HTTPRequestEntityTooLarge(request=req)
            sink_req.etag = source_resp.etag

            # we no longer need the X-Copy-From header
            del sink_req.headers['X-Copy-From']
            if 'X-Copy-From-Account' in sink_req.headers:
                del sink_req.headers['X-Copy-From-Account']
            if not content_type_manually_set:
                sink_req.headers['Content-Type'] = \
                    source_resp.headers['Content-Type']
            if config_true_value(
                    sink_req.headers.get('x-fresh-metadata', 'false')):
                # post-as-copy: ignore new sysmeta, copy existing sysmeta
                condition = lambda k: is_sys_meta('object', k)
                remove_items(sink_req.headers, condition)
                copy_header_subset(source_resp, sink_req, condition)
            else:
                # copy/update existing sysmeta and user meta
                copy_headers_into(source_resp, sink_req)
                copy_headers_into(req, sink_req)

            # copy over x-static-large-object for POSTs and manifest copies
            if 'X-Static-Large-Object' in source_resp.headers and \
                    req.params.get('multipart-manifest') == 'get':
                sink_req.headers['X-Static-Large-Object'] = \
                    source_resp.headers['X-Static-Large-Object']

            req = sink_req

        req, delete_at_container, delete_at_part, \
            delete_at_nodes = self._config_obj_expiration(req)

        node_iter = GreenthreadSafeIterator(
            self.iter_nodes_local_first(obj_ring, partition))
        pile = GreenPile(len(nodes))
        te = req.headers.get('transfer-encoding', '')
        chunked = ('chunked' in te)

        outgoing_headers = self._backend_requests(
            req, len(nodes), container_partition, containers,
            delete_at_container, delete_at_part, delete_at_nodes)

        for nheaders in outgoing_headers:
            # RFC2616:8.2.3 disallows 100-continue without a body
            if (req.content_length > 0) or chunked:
                nheaders['Expect'] = '100-continue'

#################################  CHANGED_CODE  ###################################################################
# Replaced node_iter by nodes in the following line to make sure that a new list with different order isnt used.
# Change from node_iter to nodes to make sure it writes to the same device.
# Without this, it gets a new list of nodes from the ring in a different order and connects to the first one.

            pile.spawn(self._connect_put_node, nodes, partition,
                       req.swift_entity_path, nheaders,
                       self.app.logger.thread_locals)

#################################  CHANGED_CODE ###################################################################

        conns = [conn for conn in pile if conn]
        min_conns = quorum_size(len(nodes))

        if req.if_none_match is not None and '*' in req.if_none_match:
            statuses = [conn.resp.status for conn in conns if conn.resp]
            if HTTP_PRECONDITION_FAILED in statuses:
                # If we find any copy of the file, it shouldn't be uploaded
                self.app.logger.debug(
                    _('Object PUT returning 412, %(statuses)r'),
                    {'statuses': statuses})
                return HTTPPreconditionFailed(request=req)

        if any(conn for conn in conns
               if conn.resp and conn.resp.status == HTTP_CONFLICT):
            timestamps = [
                HeaderKeyDict(
                    conn.resp.getheaders()).get('X-Backend-Timestamp')
                for conn in conns if conn.resp
            ]
            self.app.logger.debug(
                _('Object PUT returning 202 for 409: '
                  '%(req_timestamp)s <= %(timestamps)r'), {
                      'req_timestamp': req.timestamp.internal,
                      'timestamps': ', '.join(timestamps)
                  })
            return HTTPAccepted(request=req)

        if len(conns) < min_conns:
            self.app.logger.error(
                _('Object PUT returning 503, %(conns)s/%(nodes)s '
                  'required connections'), {
                      'conns': len(conns),
                      'nodes': min_conns
                  })
            return HTTPServiceUnavailable(request=req)
        bytes_transferred = 0
        try:
            with ContextPool(len(nodes)) as pool:
                for conn in conns:
                    conn.failed = False
                    conn.queue = Queue(self.app.put_queue_depth)
                    pool.spawn(self._send_file, conn, req.path)
                while True:
                    with ChunkReadTimeout(self.app.client_timeout):
                        try:
                            chunk = next(data_source)
                        except StopIteration:
                            if chunked:
                                for conn in conns:
                                    conn.queue.put('0\r\n\r\n')
                            break
                    bytes_transferred += len(chunk)
                    if bytes_transferred > constraints.MAX_FILE_SIZE:
                        return HTTPRequestEntityTooLarge(request=req)
                    for conn in list(conns):
                        if not conn.failed:
                            conn.queue.put('%x\r\n%s\r\n' %
                                           (len(chunk),
                                            chunk) if chunked else chunk)
                        else:
                            conns.remove(conn)
                    if len(conns) < min_conns:
                        self.app.logger.error(
                            _('Object PUT exceptions during'
                              ' send, %(conns)s/%(nodes)s required connections'
                              ), {
                                  'conns': len(conns),
                                  'nodes': min_conns
                              })
                        return HTTPServiceUnavailable(request=req)
                for conn in conns:
                    if conn.queue.unfinished_tasks:
                        conn.queue.join()
            conns = [conn for conn in conns if not conn.failed]
        except ChunkReadTimeout as err:
            self.app.logger.warn(_('ERROR Client read timeout (%ss)'),
                                 err.seconds)
            self.app.logger.increment('client_timeouts')
            return HTTPRequestTimeout(request=req)
        except (Exception, Timeout):
            self.app.logger.exception(
                _('ERROR Exception causing client disconnect'))
            return HTTPClientDisconnect(request=req)
        if req.content_length and bytes_transferred < req.content_length:
            req.client_disconnect = True
            self.app.logger.warn(
                _('Client disconnected without sending enough data'))
            self.app.logger.increment('client_disconnects')
            return HTTPClientDisconnect(request=req)

        statuses, reasons, bodies, etags = self._get_put_responses(
            req, conns, nodes)

        if len(etags) > 1:
            self.app.logger.error(
                _('Object servers returned %s mismatched etags'), len(etags))
            return HTTPServerError(request=req)
        etag = etags.pop() if len(etags) else None
        resp = self.best_response(req,
                                  statuses,
                                  reasons,
                                  bodies,
                                  _('Object PUT'),
                                  etag=etag)
        if source_header:
            acct, path = source_header.split('/', 3)[2:4]
            resp.headers['X-Copied-From-Account'] = quote(acct)
            resp.headers['X-Copied-From'] = quote(path)
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
        resp.last_modified = math.ceil(
            float(Timestamp(req.headers['X-Timestamp'])))
        return resp
Esempio n. 17
0
    def handle_PUT(self, req, start_response):
        if req.content_length:
            return HTTPBadRequest(
                body="Copy requests require a zero byte " "body", request=req, content_type="text/plain"
            )(req.environ, start_response)

        # Form the path of source object to be fetched
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get("X-Copy-From-Account")
        if src_account_name:
            src_account_name = check_account_format(req, src_account_name)
        else:
            src_account_name = acct
        src_container_name, src_obj_name = _check_copy_from_header(req)
        source_path = "/%s/%s/%s/%s" % (ver, src_account_name, src_container_name, src_obj_name)

        if req.environ.get("swift.orig_req_method", req.method) != "POST":
            self.logger.info("Copying object from %s to %s" % (source_path, req.path))

        # GET the source object, bail out on error
        ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
        source_resp = self._get_source_object(ssc_ctx, source_path, req)
        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            close_if_possible(source_resp.app_iter)
            return source_resp(source_resp.environ, start_response)

        # Create a new Request object based on the original req instance.
        # This will preserve env and headers.
        sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers)

        params = sink_req.params
        if params.get("multipart-manifest") == "get":
            if "X-Static-Large-Object" in source_resp.headers:
                params["multipart-manifest"] = "put"
            if "X-Object-Manifest" in source_resp.headers:
                del params["multipart-manifest"]
                sink_req.headers["X-Object-Manifest"] = source_resp.headers["X-Object-Manifest"]
            sink_req.params = params

        # Set data source, content length and etag for the PUT request
        sink_req.environ["wsgi.input"] = FileLikeIter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        if (
            source_resp.status_int == HTTP_OK
            and "X-Static-Large-Object" not in source_resp.headers
            and ("X-Object-Manifest" not in source_resp.headers or req.params.get("multipart-manifest") == "get")
        ):
            # copy source etag so that copied content is verified, unless:
            #  - not a 200 OK response: source etag may not match the actual
            #    content, for example with a 206 Partial Content response to a
            #    ranged request
            #  - SLO manifest: etag cannot be specified in manifest PUT; SLO
            #    generates its own etag value which may differ from source
            #  - SLO: etag in SLO response is not hash of actual content
            #  - DLO: etag in DLO response is not hash of actual content
            sink_req.headers["Etag"] = source_resp.etag
        else:
            # since we're not copying the source etag, make sure that any
            # container update override values are not copied.
            remove_items(source_resp.headers, lambda k: k.startswith("X-Object-Sysmeta-Container-Update-Override-"))

        # We no longer need these headers
        sink_req.headers.pop("X-Copy-From", None)
        sink_req.headers.pop("X-Copy-From-Account", None)
        # If the copy request does not explicitly override content-type,
        # use the one present in the source object.
        if not req.headers.get("content-type"):
            sink_req.headers["Content-Type"] = source_resp.headers["Content-Type"]

        fresh_meta_flag = config_true_value(sink_req.headers.get("x-fresh-metadata", "false"))

        if fresh_meta_flag or "swift.post_as_copy" in sink_req.environ:
            # Post-as-copy: ignore new sysmeta, copy existing sysmeta
            condition = lambda k: is_sys_meta("object", k)
            remove_items(sink_req.headers, condition)
            copy_header_subset(source_resp, sink_req, condition)
        else:
            # Copy/update existing sysmeta, transient-sysmeta and user meta
            _copy_headers(source_resp.headers, sink_req.headers)
            # Copy/update new metadata provided in request if any
            _copy_headers(req.headers, sink_req.headers)

        # Create response headers for PUT response
        resp_headers = self._create_response_headers(source_path, source_resp, sink_req)

        put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
        close_if_possible(source_resp.app_iter)
        return put_resp
Esempio n. 18
0
    def __call__(self, req):
        try:
            (version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app

        # verify new quota headers are properly formatted
        if not obj and req.method in ('PUT', 'POST'):
            val = req.headers.get('X-Container-Meta-Quota-Bytes')
            if val and not val.isdigit():
                return HTTPBadRequest(body='Invalid bytes quota.')
            val = req.headers.get('X-Container-Meta-Quota-Count')
            if val and not val.isdigit():
                return HTTPBadRequest(body='Invalid count quota.')

        # check user uploads against quotas
        elif obj and req.method in ('PUT', 'COPY'):
            container_info = None
            if req.method == 'PUT':
                container_info = get_container_info(
                    req.environ, self.app, swift_source='CQ')
            if req.method == 'COPY' and 'Destination' in req.headers:
                dest_account = account
                if 'Destination-Account' in req.headers:
                    dest_account = req.headers.get('Destination-Account')
                    dest_account = check_account_format(req, dest_account)
                dest_container, dest_object = check_destination_header(req)
                path_info = req.environ['PATH_INFO']
                req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
                    version, dest_account, dest_container, dest_object)
                try:
                    container_info = get_container_info(
                        req.environ, self.app, swift_source='CQ')
                finally:
                    req.environ['PATH_INFO'] = path_info
            if not container_info or not is_success(container_info['status']):
                # this will hopefully 404 later
                return self.app

            if 'quota-bytes' in container_info.get('meta', {}) and \
                    'bytes' in container_info and \
                    container_info['meta']['quota-bytes'].isdigit():
                content_length = (req.content_length or 0)
                if 'x-copy-from' in req.headers or req.method == 'COPY':
                    if 'x-copy-from' in req.headers:
                        container, obj = check_copy_from_header(req)
                    path = '/%s/%s/%s/%s' % (version, account,
                                             container, obj)
                    object_info = get_object_info(req.environ, self.app, path)
                    if not object_info or not object_info['length']:
                        content_length = 0
                    else:
                        content_length = int(object_info['length'])
                new_size = int(container_info['bytes']) + content_length
                if int(container_info['meta']['quota-bytes']) < new_size:
                    return self.bad_response(req, container_info)

            if 'quota-count' in container_info.get('meta', {}) and \
                    'object_count' in container_info and \
                    container_info['meta']['quota-count'].isdigit():
                new_count = int(container_info['object_count']) + 1
                if int(container_info['meta']['quota-count']) < new_count:
                    return self.bad_response(req, container_info)

        return self.app
Esempio n. 19
0
File: obj.py Progetto: igusher/swift
    def _handle_copy_request(self, req):
        """
        This method handles copying objects based on values set in the headers
        'X-Copy-From' and 'X-Copy-From-Account'

        This method was added as part of the refactoring of the PUT method and
        the functionality is expected to be moved to middleware
        """
        if req.environ.get('swift.orig_req_method', req.method) != 'POST':
            req.environ.setdefault('swift.log_info', []).append(
                'x-copy-from:%s' % req.headers['X-Copy-From'])
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get('X-Copy-From-Account', None)
        if src_account_name:
            src_account_name = check_account_format(req, src_account_name)
        else:
            src_account_name = acct
        src_container_name, src_obj_name = check_copy_from_header(req)
        source_header = '/%s/%s/%s/%s' % (
            ver, src_account_name, src_container_name, src_obj_name)
        source_req = req.copy_get()

        # make sure the source request uses it's container_info
        source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
        source_req.path_info = source_header
        source_req.headers['X-Newest'] = 'true'

        orig_obj_name = self.object_name
        orig_container_name = self.container_name
        orig_account_name = self.account_name
        sink_req = Request.blank(req.path_info,
                                 environ=req.environ, headers=req.headers)

        self.object_name = src_obj_name
        self.container_name = src_container_name
        self.account_name = src_account_name
        source_resp = self.GET(source_req)

        # This gives middlewares a way to change the source; for example,
        # this lets you COPY a SLO manifest and have the new object be the
        # concatenation of the segments (like what a GET request gives
        # the client), not a copy of the manifest file.
        hook = req.environ.get(
            'swift.copy_hook',
            (lambda source_req, source_resp, sink_req: source_resp))
        source_resp = hook(source_req, source_resp, sink_req)

        # reset names
        self.object_name = orig_obj_name
        self.container_name = orig_container_name
        self.account_name = orig_account_name

        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            # this is a bit of ugly code, but I'm willing to live with it
            # until copy request handling moves to middleware
            return source_resp, None, None, None
        if source_resp.content_length is None:
            # This indicates a transfer-encoding: chunked source object,
            # which currently only happens because there are more than
            # CONTAINER_LISTING_LIMIT segments in a segmented object. In
            # this case, we're going to refuse to do the server-side copy.
            raise HTTPRequestEntityTooLarge(request=req)
        if source_resp.content_length > constraints.MAX_FILE_SIZE:
            raise HTTPRequestEntityTooLarge(request=req)

        data_source = iter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        sink_req.etag = source_resp.etag

        # we no longer need the X-Copy-From header
        del sink_req.headers['X-Copy-From']
        if 'X-Copy-From-Account' in sink_req.headers:
            del sink_req.headers['X-Copy-From-Account']
        if not req.content_type_manually_set:
            sink_req.headers['Content-Type'] = \
                source_resp.headers['Content-Type']
        if config_true_value(
                sink_req.headers.get('x-fresh-metadata', 'false')):
            # post-as-copy: ignore new sysmeta, copy existing sysmeta
            condition = lambda k: is_sys_meta('object', k)
            remove_items(sink_req.headers, condition)
            copy_header_subset(source_resp, sink_req, condition)
        else:
            # copy/update existing sysmeta and user meta
            copy_headers_into(source_resp, sink_req)
            copy_headers_into(req, sink_req)

        # copy over x-static-large-object for POSTs and manifest copies
        if 'X-Static-Large-Object' in source_resp.headers and \
                req.params.get('multipart-manifest') == 'get':
            sink_req.headers['X-Static-Large-Object'] = \
                source_resp.headers['X-Static-Large-Object']

        req = sink_req

        def update_response(req, resp):
            acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4]
            resp.headers['X-Copied-From-Account'] = quote(acct)
            resp.headers['X-Copied-From'] = quote(path)
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
            return resp

        # this is a bit of ugly code, but I'm willing to live with it
        # until copy request handling moves to middleware
        return None, req, data_source, update_response
Esempio n. 20
0
    def handle_PUT(self, req, start_response):
        if req.content_length:
            return HTTPBadRequest(body='Copy requests require a zero byte '
                                  'body', request=req,
                                  content_type='text/plain')(req.environ,
                                                             start_response)

        # Form the path of source object to be fetched
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get('X-Copy-From-Account')
        if src_account_name:
            src_account_name = check_account_format(req, src_account_name)
        else:
            src_account_name = acct
        src_container_name, src_obj_name = _check_copy_from_header(req)
        source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
                                        src_container_name, src_obj_name)

        if req.environ.get('swift.orig_req_method', req.method) != 'POST':
            self.logger.info("Copying object from %s to %s" %
                             (source_path, req.path))

        # GET the source object, bail out on error
        ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
        source_resp = self._get_source_object(ssc_ctx, source_path, req)
        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            close_if_possible(source_resp.app_iter)
            return source_resp(source_resp.environ, start_response)

        # Create a new Request object based on the original req instance.
        # This will preserve env and headers.
        sink_req = Request.blank(req.path_info,
                                 environ=req.environ, headers=req.headers)

        params = sink_req.params
        if params.get('multipart-manifest') == 'get':
            if 'X-Static-Large-Object' in source_resp.headers:
                params['multipart-manifest'] = 'put'
            if 'X-Object-Manifest' in source_resp.headers:
                del params['multipart-manifest']
                sink_req.headers['X-Object-Manifest'] = \
                    source_resp.headers['X-Object-Manifest']
            sink_req.params = params

        # Set data source, content length and etag for the PUT request
        sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        sink_req.etag = source_resp.etag

        # We no longer need these headers
        sink_req.headers.pop('X-Copy-From', None)
        sink_req.headers.pop('X-Copy-From-Account', None)
        # If the copy request does not explicitly override content-type,
        # use the one present in the source object.
        if not req.headers.get('content-type'):
            sink_req.headers['Content-Type'] = \
                source_resp.headers['Content-Type']

        fresh_meta_flag = config_true_value(
            sink_req.headers.get('x-fresh-metadata', 'false'))

        if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ:
            # Post-as-copy: ignore new sysmeta, copy existing sysmeta
            condition = lambda k: is_sys_meta('object', k)
            remove_items(sink_req.headers, condition)
            copy_header_subset(source_resp, sink_req, condition)
        else:
            # Copy/update existing sysmeta and user meta
            _copy_headers_into(source_resp, sink_req)
            # Copy/update new metadata provided in request if any
            _copy_headers_into(req, sink_req)

        # Create response headers for PUT response
        resp_headers = self._create_response_headers(source_path,
                                                     source_resp, sink_req)

        put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
        close_if_possible(source_resp.app_iter)
        return put_resp
Esempio n. 21
0
    def _handle_copy_request(self, req):
        """
        This method handles copying objects based on values set in the headers
        'X-Copy-From' and 'X-Copy-From-Account'

        This method was added as part of the refactoring of the PUT method and
        the functionality is expected to be moved to middleware
        """
        if req.environ.get('swift.orig_req_method', req.method) != 'POST':
            req.environ.setdefault('swift.log_info', []).append(
                'x-copy-from:%s' % req.headers['X-Copy-From'])
        ver, acct, _rest = req.split_path(2, 3, True)
        src_account_name = req.headers.get('X-Copy-From-Account', None)
        if src_account_name:
            src_account_name = check_account_format(req, src_account_name)
        else:
            src_account_name = acct
        src_container_name, src_obj_name = check_copy_from_header(req)
        source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
                                          src_container_name, src_obj_name)
        source_req = req.copy_get()

        # make sure the source request uses it's container_info
        source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
        source_req.path_info = source_header
        source_req.headers['X-Newest'] = 'true'

        orig_obj_name = self.object_name
        orig_container_name = self.container_name
        orig_account_name = self.account_name
        sink_req = Request.blank(req.path_info,
                                 environ=req.environ,
                                 headers=req.headers)

        self.object_name = src_obj_name
        self.container_name = src_container_name
        self.account_name = src_account_name
        source_resp = self.GET(source_req)

        # This gives middlewares a way to change the source; for example,
        # this lets you COPY a SLO manifest and have the new object be the
        # concatenation of the segments (like what a GET request gives
        # the client), not a copy of the manifest file.
        hook = req.environ.get(
            'swift.copy_hook',
            (lambda source_req, source_resp, sink_req: source_resp))
        source_resp = hook(source_req, source_resp, sink_req)

        # reset names
        self.object_name = orig_obj_name
        self.container_name = orig_container_name
        self.account_name = orig_account_name

        if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
            # this is a bit of ugly code, but I'm willing to live with it
            # until copy request handling moves to middleware
            return source_resp, None, None, None
        if source_resp.content_length is None:
            # This indicates a transfer-encoding: chunked source object,
            # which currently only happens because there are more than
            # CONTAINER_LISTING_LIMIT segments in a segmented object. In
            # this case, we're going to refuse to do the server-side copy.
            raise HTTPRequestEntityTooLarge(request=req)
        if source_resp.content_length > constraints.MAX_FILE_SIZE:
            raise HTTPRequestEntityTooLarge(request=req)

        data_source = iter(source_resp.app_iter)
        sink_req.content_length = source_resp.content_length
        sink_req.etag = source_resp.etag

        # we no longer need the X-Copy-From header
        del sink_req.headers['X-Copy-From']
        if 'X-Copy-From-Account' in sink_req.headers:
            del sink_req.headers['X-Copy-From-Account']
        if not req.content_type_manually_set:
            sink_req.headers['Content-Type'] = \
                source_resp.headers['Content-Type']
        if config_true_value(sink_req.headers.get('x-fresh-metadata',
                                                  'false')):
            # post-as-copy: ignore new sysmeta, copy existing sysmeta
            condition = lambda k: is_sys_meta('object', k)
            remove_items(sink_req.headers, condition)
            copy_header_subset(source_resp, sink_req, condition)
        else:
            # copy/update existing sysmeta and user meta
            copy_headers_into(source_resp, sink_req)
            copy_headers_into(req, sink_req)

        # copy over x-static-large-object for POSTs and manifest copies
        if 'X-Static-Large-Object' in source_resp.headers and \
                req.params.get('multipart-manifest') == 'get':
            sink_req.headers['X-Static-Large-Object'] = \
                source_resp.headers['X-Static-Large-Object']

        req = sink_req

        def update_response(req, resp):
            acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4]
            resp.headers['X-Copied-From-Account'] = quote(acct)
            resp.headers['X-Copied-From'] = quote(path)
            if 'last-modified' in source_resp.headers:
                resp.headers['X-Copied-From-Last-Modified'] = \
                    source_resp.headers['last-modified']
            copy_headers_into(req, resp)
            return resp

        # this is a bit of ugly code, but I'm willing to live with it
        # until copy request handling moves to middleware
        return None, req, data_source, update_response