コード例 #1
0
    def test_check_object_creation(self):
        req = Mock()
        req.headers = dict()

        valid_object_names = ["a/b/c/d",
                              '/'.join(("1@3%&*0-", "};+=]|")),
                              '/'.join(('a' * 255, 'b' * 255, 'c' * 221))]
        for o in valid_object_names:
            self.assertFalse(cnt.check_object_creation(req, o))

        invalid_object_names = ["a/./b",
                                "a/b/../d",
                                "a//b",
                                "a/c//",
                                '/'.join(('a' * 256, 'b' * 255, 'c' * 221)),
                                '/'.join(('a' * 255, 'b' * 255, 'c' * 222))]
        for o in invalid_object_names:
            self.assertTrue(cnt.check_object_creation(req, o))

        # Check for creation of directory marker objects that ends with slash
        with patch.dict(req.headers, {'content-type':
                                      'application/directory'}):
            self.assertFalse(cnt.check_object_creation(req, "a/b/c/d/"))

        # Check creation of objects ending with slash having any other content
        # type than application/directory is not allowed
        for content_type in ('text/plain', 'text/html', 'image/jpg',
                             'application/octet-stream', 'blah/blah'):
            with patch.dict(req.headers, {'content-type':
                                          content_type}):
                self.assertTrue(cnt.check_object_creation(req, "a/b/c/d/"))
コード例 #2
0
    def PUT(self, request):
        """Handle HTTP PUT requests for the Swift on File object server"""
        try:
            device, partition, account, container, obj, policy = \
                get_name_and_placement(request, 5, 5, True)

            req_timestamp = valid_timestamp(request)

            # check swiftonhpss constraints first
            error_response = check_object_creation(request, obj)
            if error_response:
                return error_response

            # (HPSS) Shameless copy-paste from ObjectController.PUT and
            # modification, because we have to do certain things like pass in
            # purgelock and class-of-service information that Swift won't know
            # to do and need to do it in a very specific order.
            new_delete_at = int(request.headers.get('X-Delete-At') or 0)
            if new_delete_at and new_delete_at < time.time():
                return HTTPBadRequest(body='X-Delete-At in past',
                                      request=request,
                                      context_type='text/plain')

            try:
                fsize = request.message_length()
            except ValueError as e:
                return HTTPBadRequest(body=str(e),
                                      request=request,
                                      content_type='text/plain')

            # Try to get DiskFile
            try:
                disk_file = self.get_diskfile(device, partition, account,
                                              container, obj, policy=policy)
            except DiskFileDeviceUnavailable:
                return HTTPInsufficientStorage(drive=device, request=request)

            try:
                orig_metadata = disk_file.read_metadata()
            except (DiskFileNotExist, DiskFileQuarantined):
                orig_metadata = {}

            # Check for If-None-Match in request
            if request.if_none_match and orig_metadata:
                if '*' in request.if_none_match:
                    # File exists already, return 412
                    return HTTPPreconditionFailed(request=request)
                if orig_metadata.get('ETag') in request.if_none_match:
                    # The current ETag matches, return 412
                    return HTTPPreconditionFailed(request=request)

            orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))

            if orig_timestamp >= req_timestamp:
                return HTTPConflict(
                    request=request,
                    headers={'X-Backend-Timestamp': orig_timestamp.internal})
            orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
            upload_expiration = time.time() + self.max_upload_time

            etag = md5()
            elapsed_time = 0

            # (HPSS) Check for HPSS-specific metadata headers
            cos = request.headers.get('X-Hpss-Class-Of-Service-Id', None)
            purgelock = config_true_value(
                request.headers.get('X-Hpss-Purgelock-Status', 'false'))

            try:
                # Feed DiskFile our HPSS-specific stuff
                with disk_file.create(size=fsize, cos=cos) as writer:
                    upload_size = 0

                    def timeout_reader():
                        with ChunkReadTimeout(self.client_timeout):
                            return request.environ['wsgi.input'].read(
                                self.network_chunk_size)

                    try:
                        for chunk in iter(lambda: timeout_reader(), ''):
                            start_time = time.time()
                            if start_time > upload_expiration:
                                self.logger.increment('PUT.timeouts')
                                return HTTPRequestTimeout(request=request)
                            etag.update(chunk)
                            upload_size = writer.write(chunk)
                            elapsed_time += time.time() - start_time
                    except ChunkReadTimeout:
                        return HTTPRequestTimeout(request=request)
                    if upload_size:
                        self.logger.transfer_rate('PUT.%s.timing' % device,
                                                  elapsed_time, upload_size)
                    if fsize and fsize != upload_size:
                        return HTTPClientDisconnect(request=request)
                    etag = etag.hexdigest()

                    if 'etag' in request.headers \
                            and request.headers['etag'].lower() != etag:
                        return HTTPUnprocessableEntity(request=request)

                    # Update object metadata
                    metadata = {'X-Timestamp': request.timestamp.internal,
                                'Content-Type': request.headers['content-type'],
                                'ETag': etag,
                                'Content-Length': str(upload_size),
                                }
                    meta_headers = {header: request.headers[header] for header
                                    in request.headers
                                    if is_sys_or_user_meta('object', header)}
                    metadata.update(meta_headers)
                    backend_headers = \
                        request.headers.get('X-Backend-Replication-Headers')
                    for header_key in (backend_headers or self.allowed_headers):
                        if header_key in request.headers:
                            header_caps = header_key.title()
                            metadata[header_caps] = request.headers[header_key]

                    # (HPSS) Write the file, with added options
                    writer.put(metadata, purgelock=purgelock)

            except DiskFileNoSpace:
                return HTTPInsufficientStorage(drive=device, request=request)
            except SwiftOnFileSystemIOError as e:
                return HTTPServiceUnavailable(request=request)

            # FIXME: this stuff really should be handled in DiskFile somehow?
            # we set the hpss checksum in here, so both systems have valid
            # and current checksum metadata

            # (HPSS) Set checksum on file ourselves, if hpssfs won't do it
            # for us.
            data_file = disk_file._data_file
            try:
                xattr.setxattr(data_file, 'system.hpss.hash',
                               "md5:%s" % etag)
            except IOError:
                logging.debug("Could not write ETag to system.hpss.hash,"
                              " trying user.hash.checksum")
                try:
                    xattr.setxattr(data_file,
                                   'user.hash.checksum', etag)
                    xattr.setxattr(data_file,
                                   'user.hash.algorithm', 'md5')
                    xattr.setxattr(data_file,
                                   'user.hash.state', 'Valid')
                    xattr.setxattr(data_file,
                                   'user.hash.filesize', str(upload_size))
                    xattr.setxattr(data_file,
                                   'user.hash.app', 'swiftonhpss')
                except IOError as err:
                    raise SwiftOnFileSystemIOError(
                        err.errno,
                        'Could not write MD5 checksum to HPSS filesystem: '
                        '%s' % err.strerror)

            # Update container metadata
            if orig_delete_at != new_delete_at:
                if new_delete_at:
                    self.delete_at_update('PUT', new_delete_at, account,
                                          container, obj, request, device,
                                          policy)
                if orig_delete_at:
                    self.delete_at_update('DELETE', orig_delete_at, account,
                                          container, obj, request, device,
                                          policy)
            self.container_update('PUT', account, container, obj, request,
                                  HeaderKeyDict(
                                      {'x-size':
                                           metadata['Content-Length'],
                                       'x-content-type':
                                           metadata['Content-Type'],
                                       'x-timestamp':
                                           metadata['X-Timestamp'],
                                       'x-etag':
                                           metadata['ETag']}),
                                  device, policy)
            # Create convenience symlink
            try:
                self._object_symlink(request, disk_file._data_file, device,
                                     account)
            except SwiftOnFileSystemOSError:
                logging.debug('could not make account symlink')
                return HTTPServiceUnavailable(request=request)
            return HTTPCreated(request=request, etag=etag)

        except (AlreadyExistsAsFile, AlreadyExistsAsDir):
            device = \
                split_and_validate_path(request, 1, 5, True)
            return HTTPConflict(drive=device, request=request)
コード例 #3
0
ファイル: server.py プロジェクト: openstack/swiftonhpss
    def PUT(self, request):
        """Handle HTTP PUT requests for the Swift on File object server"""

        try:
            device, partition, account, container, obj, policy = \
                get_name_and_placement(request, 5, 5, True)

            req_timestamp = valid_timestamp(request)

            # check swiftonhpss constraints first
            error_response = check_object_creation(request, obj)
            if error_response:
                return error_response

            # (HPSS) Shameless copy-paste from ObjectController.PUT and
            # modification, because we have to do certain things like pass in
            # purgelock and class-of-service information that Swift won't know
            # to do and need to do it in a very specific order.
            new_delete_at = int(request.headers.get('X-Delete-At') or 0)
            if new_delete_at and new_delete_at < time.time():
                return HTTPBadRequest(body='X-Delete-At in past',
                                      request=request,
                                      context_type='text/plain')

            try:
                fsize = request.message_length()
            except ValueError as e:
                return HTTPBadRequest(body=str(e),
                                      request=request,
                                      content_type='text/plain')

            self.logger.debug("DiskFile @ %s/%s/%s/%s" %
                              (device, account, container, obj))

            # Try to get DiskFile
            try:
                disk_file = self.get_diskfile(device, partition, account,
                                              container, obj, policy=policy,
                                              uid=int(self.hpss_uid),
                                              gid=int(self.hpss_gid))
            except DiskFileDeviceUnavailable:
                return HTTPInsufficientStorage(drive=device, request=request)

            try:
                orig_metadata = disk_file.read_metadata()
            except (DiskFileNotExist, DiskFileQuarantined):
                orig_metadata = {}

            # Check for If-None-Match in request
            if request.if_none_match and orig_metadata:
                if '*' in request.if_none_match:
                    # File exists already, return 412
                    return HTTPPreconditionFailed(request=request)
                if orig_metadata.get('ETag') in request.if_none_match:
                    # The current ETag matches, return 412
                    return HTTPPreconditionFailed(request=request)

            orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))

            if orig_timestamp >= req_timestamp:
                return HTTPConflict(
                    request=request,
                    headers={'X-Backend-Timestamp': orig_timestamp.internal})
            orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
            upload_expiration = time.time() + self.max_upload_time

            self.logger.debug("Receiving and writing object")

            etag = md5()
            elapsed_time = 0

            hints = {'cos': request.headers.get('X-Hpss-Class-Of-Service-Id',
                                                self.default_cos_id),
                     'purgelock':
                         request.headers.get('X-Hpss-Purgelock-Status', False)}

            if request.headers['content-type'] == 'application/directory':
                # TODO: handle directories different
                pass

            try:
                # Feed DiskFile our HPSS-specific stuff
                with disk_file.create(hpss_hints=hints) as writer:
                    upload_size = 0

                    def timeout_reader():
                        with ChunkReadTimeout(self.client_timeout):
                            return request.environ['wsgi.input'].read(
                                self.network_chunk_size)

                    try:
                        for chunk in iter(lambda: timeout_reader(), ''):
                            start_time = time.time()
                            if start_time > upload_expiration:
                                self.logger.increment('PUT.timeouts')
                                return HTTPRequestTimeout(request=request)
                            etag.update(chunk)
                            upload_size = writer.write(chunk)
                            elapsed_time += time.time() - start_time
                    except ChunkReadTimeout:
                        return HTTPRequestTimeout(request=request)
                    if upload_size:
                        self.logger.transfer_rate('PUT.%s.timing' % device,
                                                  elapsed_time, upload_size)
                    if fsize and fsize != upload_size:
                        return HTTPClientDisconnect(request=request)
                    etag = etag.hexdigest()

                    if 'etag' in request.headers \
                            and request.headers['etag'].lower() != etag:
                        return HTTPUnprocessableEntity(request=request)

                    self.logger.debug("Writing object metadata")

                    # Update object metadata
                    content_type = request.headers['content-type']
                    metadata = {'X-Timestamp': request.timestamp.internal,
                                'Content-Type': content_type,
                                'ETag': etag,
                                'Content-Length': str(upload_size),
                                }
                    meta_headers = {header: request.headers[header] for header
                                    in request.headers
                                    if is_sys_or_user_meta('object', header)}
                    metadata.update(meta_headers)
                    backend_headers = \
                        request.headers.get('X-Backend-Replication-Headers')
                    for header_key in (backend_headers or
                                       self.allowed_headers):
                        if header_key in request.headers:
                            header_caps = header_key.title()
                            metadata[header_caps] = request.headers[header_key]

                    self.logger.debug("Finalizing object")

                    # (HPSS) Write the file, with added options
                    writer.put(metadata)

            except DiskFileNoSpace:
                return HTTPInsufficientStorage(drive=device, request=request)
            except SwiftOnFileSystemIOError as e:
                self.logger.error(e)
                return HTTPServiceUnavailable(request=request)

            self.logger.debug("Writing container metadata")

            # Update container metadata
            if orig_delete_at != new_delete_at:
                if new_delete_at:
                    self.delete_at_update('PUT', new_delete_at, account,
                                          container, obj, request, device,
                                          policy)
                if orig_delete_at:
                    self.delete_at_update('DELETE', orig_delete_at, account,
                                          container, obj, request, device,
                                          policy)
            container_headers = {'x-size': metadata['Content-Length'],
                                 'x-content-type': metadata['Content-Type'],
                                 'x-timestamp': metadata['X-Timestamp'],
                                 'x-etag': metadata['ETag']}
            self.container_update('PUT', account, container, obj, request,
                                  HeaderKeyDict(container_headers),
                                  device, policy)

            self.logger.debug("Done!")
            # Create convenience symlink
            try:
                self._project_symlink(request, disk_file, account)
            except SwiftOnFileSystemOSError:
                logging.debug('could not make account symlink')
                return HTTPServiceUnavailable(request=request)

            return HTTPCreated(request=request, etag=etag)

        except (AlreadyExistsAsFile, AlreadyExistsAsDir):
            device = \
                split_and_validate_path(request, 1, 5, True)
            return HTTPConflict(drive=device, request=request)