def test_do_open(self):
        _fd, tmpfile = mkstemp()
        try:
            f = fs.do_open(tmpfile, 'r')
            try:
                f.write('test')
            except IOError as err:
                pass
            else:
                self.fail("IOError expected")
            finally:
                f.close()

            fd = fs.do_open(tmpfile, os.O_RDONLY)
            try:
                os.write(fd, 'test')
            except OSError as err:
                pass
            else:
                self.fail("OSError expected")
            finally:
                os.close(fd)
        finally:
            os.close(_fd)
            os.remove(tmpfile)
Beispiel #2
0
 def test_do_open_err(self):
     try:
         fs.do_open(os.path.join('/tmp', str(random.random())), 'r')
     except IOError:
         pass
     else:
         self.fail("IOError expected")
 def test_do_open_err_int_mode(self):
     try:
         fs.do_open(os.path.join('/tmp', str(random.random())),
                    os.O_RDONLY)
     except GlusterFileSystemOSError:
         pass
     else:
         self.fail("GlusterFileSystemOSError expected")
Beispiel #4
0
def _get_etag(path_or_fd):
    """
    FIXME: It would be great to have a translator that returns the md5sum() of
    the file as an xattr that can be simply fetched.

    Since we don't have that we should yield after each chunk read and
    computed so that we don't consume the worker thread.
    """
    etag = ''
    if isinstance(path_or_fd, int):
        # We are given a file descriptor, so this is an invocation from the
        # DiskFile.open() method.
        fd = path_or_fd
        dup_fd = do_dup(fd)
        try:
            etag = _read_for_etag(dup_fd)
            do_lseek(fd, 0, os.SEEK_SET)
        finally:
            do_close(dup_fd)
    else:
        # We are given a path to the object when the DiskDir.list_objects_iter
        # method invokes us.
        path = path_or_fd
        fd = do_open(path, os.O_RDONLY)
        try:
            etag = _read_for_etag(fd)
        finally:
            do_close(fd)

    return etag
Beispiel #5
0
def _get_etag(path_or_fd):
    """
    FIXME: It would be great to have a translator that returns the md5sum() of
    the file as an xattr that can be simply fetched.

    Since we don't have that we should yield after each chunk read and
    computed so that we don't consume the worker thread.
    """
    etag = ''
    if isinstance(path_or_fd, int):
        # We are given a file descriptor, so this is an invocation from the
        # DiskFile.open() method.
        fd = path_or_fd
        dup_fd = do_dup(fd)
        try:
            etag = _read_for_etag(dup_fd)
            do_lseek(fd, 0, os.SEEK_SET)
        finally:
            do_close(dup_fd)
    else:
        # We are given a path to the object when the DiskDir.list_objects_iter
        # method invokes us.
        path = path_or_fd
        fd = do_open(path, os.O_RDONLY)
        try:
            etag = _read_for_etag(fd)
        finally:
            do_close(fd)

    return etag
Beispiel #6
0
    def mkstemp(self):
        """Contextmanager to make a temporary file."""

        # Creating intermidiate directories and corresponding metadata.
        # For optimization, check if the subdirectory already exists,
        # if exists, then it means that it also has its metadata.
        # Not checking for container, since the container should already
        # exist for the call to come here.
        if not os_path.exists(self.datadir):
            path = self._container_path
            subdir_list = self._obj_path.split(os.path.sep)
            for i in range(len(subdir_list)):
                path = os.path.join(path, subdir_list[i])
                if not os_path.exists(path):
                    self._create_dir_object(path)

        tmpfile = '.' + self._obj + '.' + md5(self._obj +
                  str(random.random())).hexdigest()

        self.tmppath = os.path.join(self.datadir, tmpfile)
        fd = do_open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL)
        try:
            yield fd
        finally:
            try:
                do_close(fd)
            except OSError:
                pass
            tmppath, self.tmppath = self.tmppath, None
            do_unlink(tmppath)
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        else:
            stats = do_fstat(fd)
            if not stats:
                return
            self._is_dir = stat.S_ISDIR(stats.st_mode)
            obj_size = stats.st_size

        self._metadata = read_metadata(fd)
        if not validate_object(self._metadata):
            create_object_metadata(fd)
            self._metadata = read_metadata(fd)
        assert self._metadata is not None
        self._filter_metadata()

        if self._is_dir:
            do_close(fd)
            obj_size = 0
            self._fd = -1
        else:
            if self._is_object_expired(self._metadata):
                raise DiskFileExpired(metadata=self._metadata)
            self._fd = fd

        self._obj_size = obj_size
        return self
Beispiel #8
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        else:
            stats = do_fstat(fd)
            if not stats:
                return
            self._is_dir = stat.S_ISDIR(stats.st_mode)
            obj_size = stats.st_size

        self._metadata = read_metadata(fd)
        if not validate_object(self._metadata):
            create_object_metadata(fd)
            self._metadata = read_metadata(fd)
        assert self._metadata is not None
        self._filter_metadata()

        if self._is_dir:
            do_close(fd)
            obj_size = 0
            self._fd = -1
        else:
            if self._is_object_expired(self._metadata):
                raise DiskFileExpired(metadata=self._metadata)
            self._fd = fd

        self._obj_size = obj_size
        return self
Beispiel #9
0
 def test_do_open(self):
     try:
         fd, tmpfile = mkstemp()
         f = fs.do_open(tmpfile, 'r')
         try:
             f.write('test')
         except IOError as err:
             pass
         else:
             self.fail("IOError expected")
     finally:
         f.close()
         os.close(fd)
         os.remove(tmpfile)
Beispiel #10
0
    def writer(self, size=None):
        """
        Contextmanager to make a temporary file, optionally of a specified
        initial size.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.
        """
        data_file = os.path.join(self.put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        attempts = 1
        cur_thread = str(getcurrent())
        while True:
            postfix = md5(self._obj + _cur_host + _cur_pid + cur_thread
                          + str(random.random())).hexdigest()
            tmpfile = '.' + self._obj + '.' + postfix
            tmppath = os.path.join(self.put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno == errno.ENOSPC:
                    # Raise DiskFileNoSpace to be handled by upper layers
                    raise DiskFileNoSpace()
                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.mkstemp(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self.uid, self.gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely.
            dw = DiskWriter(self, fd, tmppath, self.threadpool)
            yield dw
        finally:
            try:
                if dw.fd:
                    do_close(dw.fd)
            except OSError:
                pass
            if dw.tmppath:
                do_unlink(dw.tmppath)
Beispiel #11
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        attempts = 1
        cur_thread = str(getcurrent())
        while True:
            postfix = md5(self._obj + _cur_host + _cur_pid + cur_thread +
                          str(random.random())).hexdigest()
            tmpfile = '.' + self._obj + '.' + postfix
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory' %
                                              (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError(
                        'DiskFile.mkstemp(): failed to'
                        ' successfully create a temporary file'
                        ' without running into a name conflict'
                        ' after %d of %d attempts for: %s' %
                        (attempts, MAX_OPEN_ATTEMPTS, data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn(
                        "DiskFile.mkstemp(): %s ... retrying in"
                        " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn(
                        "DiskFile.mkstemp(): %s ... retrying in"
                        " 0.1 secs", gerr)
                    attempts += 1
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self._uid, self._gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely since at the time of this writing FUSE does not
            # support it.
            dw = DiskFileWriter(fd, tmppath, self)
            yield dw
        finally:
            dw.close()
            if dw._tmppath:
                do_unlink(dw._tmppath)
Beispiel #12
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        fd = None
        attempts = 1
        while True:
            tmpfile = '.' + self._obj + '.' + uuid4().hex
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory'
                                              % (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.create(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # ENOENT
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    # Handle race:
                    # This can be the issue when memcache has cached that the
                    # container exists. If someone removes the container dir
                    # from filesystem, it's not reflected in memcache. So
                    # swift reports that the container exists and this code
                    # tries to create a file in a directory that does not
                    # exist. However, it's wrong to create the container here.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                    if attempts > 2:
                        # Ideally we would want to return 404 indicating that
                        # the container itself does not exist. Can't be done
                        # though as the caller won't catch DiskFileNotExist.
                        # We raise an exception with a meaningful name for
                        # correctness.
                        logging.warn("Container dir %s does not exist",
                                     self._container_path)
                        raise DiskFileContainerDoesNotExist
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            if not ((self._uid == DEFAULT_UID) and (self._gid == DEFAULT_GID)):
                # If both UID and GID is -1 (default values), it has no effect.
                # So don't do a fchown.
                # Further, at the time of this writing, UID and GID information
                # is not passed to DiskFile.
                do_fchown(fd, self._uid, self._gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely since at the time of this writing FUSE does not
            # support it.
            dw = DiskFileWriter(fd, tmppath, self)
            # It's now the responsibility of DiskFileWriter to close this fd.
            fd = None
            yield dw
        finally:
            if dw:
                dw.close()
                if dw._tmppath:
                    do_unlink(dw._tmppath)
Beispiel #13
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        try:
            if not self._stat:
                self._stat = do_fstat(self._fd)
            obj_size = self._stat.st_size

            if not self._metadata:
                self._metadata = read_metadata(self._fd)
            if not validate_object(self._metadata, self._stat):
                self._metadata = create_object_metadata(self._fd, self._stat,
                                                        self._metadata)
            assert self._metadata is not None
            self._filter_metadata()

            if stat.S_ISDIR(self._stat.st_mode):
                do_close(self._fd)
                obj_size = 0
                self._fd = -1
            else:
                if self._is_object_expired(self._metadata):
                    raise DiskFileExpired(metadata=self._metadata)
            self._obj_size = obj_size
        except (OSError, IOError, DiskFileExpired) as err:
            # Something went wrong. Context manager will not call
            # __exit__. So we close the fd manually here.
            self._close_fd()
            if hasattr(err, 'errno') and \
                    err.errno in (errno.ENOENT, errno.ESTALE):
                # Handle races: ENOENT/ESTALE can be raised by read_metadata()
                # call in GlusterFS if file gets deleted by another
                # client after do_open() succeeds
                logging.warn("open(%s) succeeded but one of the subsequent "
                             "syscalls failed with ENOENT/ESTALE. Raising "
                             "DiskFileNotExist." % (self._data_file))
                raise DiskFileNotExist
            else:
                # Re-raise the original exception after fd has been closed
                raise

        self._disk_file_open = True
        return self
Beispiel #14
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False,
                 disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
        self.disk_chunk_size = disk_chunk_size
        self.iter_hook = iter_hook
        # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/,
        # etc.
        obj = obj.strip(os.path.sep)
        if os.path.sep in obj:
            self._obj_path, self._obj = os.path.split(obj)
        else:
            self._obj_path = ''
            self._obj = obj

        if self._obj_path:
            self.name = os.path.join(container, self._obj_path)
        else:
            self.name = container
        # Absolute path for object directory.
        self.datadir = os.path.join(path, device, self.name)
        self.device_path = os.path.join(path, device)
        self._container_path = os.path.join(path, device, container)
        self._is_dir = False
        self.tmppath = None
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.uid = int(uid)
        self.gid = int(gid)
        self.suppress_file_closing = False

        # Don't store a value for data_file until we know it exists.
        self.data_file = None
        data_file = os.path.join(self.datadir, self._obj)
        if not os_path.exists(data_file):
            return

        self.data_file = os.path.join(data_file)
        self.metadata = read_metadata(data_file)
        if not self.metadata:
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        if not validate_object(self.metadata):
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        self.filter_metadata()

        if os_path.isdir(data_file):
            self._is_dir = True
        else:
            if keep_data_fp:
                # The caller has an assumption that the "fp" field of this
                # object is an file object if keep_data_fp is set. However,
                # this implementation of the DiskFile object does not need to
                # open the file for internal operations. So if the caller
                # requests it, we'll just open the file for them.
                self.fp = do_open(data_file, 'rb')
Beispiel #15
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False,
                 disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
        self.disk_chunk_size = disk_chunk_size
        self.iter_hook = iter_hook
        obj = obj.strip(os.path.sep)

        if os.path.sep in obj:
            self._obj_path, self._obj = os.path.split(obj)
        else:
            self._obj_path = ''
            self._obj = obj

        if self._obj_path:
            self.name = os.path.join(container, self._obj_path)
        else:
            self.name = container
        # Absolute path for object directory.
        self.datadir = os.path.join(path, device, self.name)
        self.device_path = os.path.join(path, device)
        self._container_path = os.path.join(path, device, container)
        if _use_put_mount:
            self.put_datadir = os.path.join(self.device_path + '_PUT',
                                            self.name)
        else:
            self.put_datadir = self.datadir
        self._is_dir = False
        self.tmppath = None
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.uid = int(uid)
        self.gid = int(gid)
        self.suppress_file_closing = False

        # Don't store a value for data_file until we know it exists.
        self.data_file = None
        data_file = os.path.join(self.put_datadir, self._obj)

        try:
            stats = do_stat(data_file)
        except OSError as err:
            if err.errno == errno.ENOTDIR:
                return
        else:
            if not stats:
                return

        self.data_file = data_file
        self._is_dir = stat.S_ISDIR(stats.st_mode)

        self.metadata = read_metadata(data_file)
        if not self.metadata:
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        if not validate_object(self.metadata):
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        self.filter_metadata()

        if not self._is_dir and keep_data_fp:
            # The caller has an assumption that the "fp" field of this
            # object is an file object if keep_data_fp is set. However,
            # this implementation of the DiskFile object does not need to
            # open the file for internal operations. So if the caller
            # requests it, we'll just open the file for them.
            self.fp = do_open(data_file, 'rb')
Beispiel #16
0
    def mkstemp(self, size=None):
        """
        Contextmanager to make a temporary file, optionally of a specified
        initial size.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.
        """
        data_file = os.path.join(self.put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        for i in range(0, 1000):
            tmpfile = '.' + self._obj + '.' + md5(self._obj +
                      str(random.random())).hexdigest()
            tmppath = os.path.join(self.put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    continue
                if gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    continue
                if gerr.errno != errno.ENOENT:
                    # FIXME: Other cases we should handle?
                    raise
                if not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    continue
                if i != 0:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    continue
                # It looks like the path to the object does not already exist
                self._create_dir_object(self._obj_path)
                continue
            else:
                break
        else:
            # We failed after 1,000 attempts to create the temporary file.
            raise DiskFileError('DiskFile.mkstemp(): failed to successfully'
                                ' create a temporary file without running'
                                ' into a name conflict after 1,000 attempts'
                                ' for: %s' % (data_file,))

        self.tmppath = tmppath

        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self.uid, self.gid)
            if _preallocate and size:
                # For XFS, fallocate() turns off speculative pre-allocation
                # until a write is issued either to the last block of the file
                # before the EOF or beyond the EOF. This means that we are
                # less likely to fragment free space with pre-allocated
                # extents that get truncated back to the known file size.
                # However, this call also turns holes into allocated but
                # unwritten extents, so that allocation occurs before the
                # write, not during XFS writeback. This effectively defeats
                # any allocation optimizations the filesystem can make at
                # writeback time.
                fallocate(fd, size)
            yield fd
        finally:
            try:
                do_close(fd)
            except OSError:
                pass
            if self.tmppath:
                tmppath, self.tmppath = self.tmppath, None
                do_unlink(tmppath)
Beispiel #17
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """

        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        fd = None
        attempts = 1
        while True:
            tmpfile = '.' + self._obj + '.' + uuid4().hex
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory'
                                              % (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.create(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # ENOENT
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    # Handle race:
                    # This can be the issue when memcache has cached that the
                    # container exists. If someone removes the container dir
                    # from filesystem, it's not reflected in memcache. So
                    # swift reports that the container exists and this code
                    # tries to create a file in a directory that does not
                    # exist. However, it's wrong to create the container here.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                    if attempts > 2:
                        # Ideally we would want to return 404 indicating that
                        # the container itself does not exist. Can't be done
                        # though as the caller won't catch DiskFileNotExist.
                        # We raise an exception with a meaningful name for
                        # correctness.
                        logging.warn("Container dir %s does not exist",
                                     self._container_path)
                        raise DiskFileContainerDoesNotExist
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            if not ((self._uid == DEFAULT_UID) and (self._gid == DEFAULT_GID)):
                # If both UID and GID is -1 (default values), it has no effect.
                # So don't do a fchown.
                # Further, at the time of this writing, UID and GID information
                # is not passed to DiskFile.
                do_fchown(fd, self._uid, self._gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely since at the time of this writing FUSE does not
            # support it.
            dw = DiskFileWriter(fd, tmppath, self)
            # It's now the responsibility of DiskFileWriter to close this fd.
            fd = None
            yield dw
        finally:
            if dw:
                dw.close()
                if dw._tmppath:
                    do_unlink(dw._tmppath)
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        attempts = 1
        cur_thread = str(getcurrent())
        while True:
            postfix = md5(self._obj + _cur_host + _cur_pid + cur_thread
                          + str(random.random())).hexdigest()
            tmpfile = '.' + self._obj + '.' + postfix
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory'
                                              % (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.mkstemp(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self._uid, self._gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely since at the time of this writing FUSE does not
            # support it.
            dw = DiskFileWriter(fd, tmppath, self)
            yield dw
        finally:
            dw.close()
            if dw._tmppath:
                do_unlink(dw._tmppath)
Beispiel #19
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        try:
            if not self._stat:
                self._stat = do_fstat(self._fd)
            obj_size = self._stat.st_size

            if not self._metadata:
                self._metadata = read_metadata(self._fd)
            if not validate_object(self._metadata, self._stat):
                self._metadata = create_object_metadata(self._fd, self._stat,
                                                        self._metadata)
            assert self._metadata is not None
            self._filter_metadata()

            if stat.S_ISDIR(self._stat.st_mode):
                do_close(self._fd)
                obj_size = 0
                self._fd = -1
            else:
                if self._is_object_expired(self._metadata):
                    raise DiskFileExpired(metadata=self._metadata)
            self._obj_size = obj_size
        except (OSError, IOError, DiskFileExpired) as err:
            # Something went wrong. Context manager will not call
            # __exit__. So we close the fd manually here.
            self._close_fd()
            if hasattr(err, 'errno') and \
                    err.errno in (errno.ENOENT, errno.ESTALE):
                # Handle races: ENOENT/ESTALE can be raised by read_metadata()
                # call in GlusterFS if file gets deleted by another
                # client after do_open() succeeds
                logging.warn("open(%s) succeeded but one of the subsequent "
                             "syscalls failed with ENOENT/ESTALE. Raising "
                             "DiskFileNotExist." % (self._data_file))
                raise DiskFileNotExist
            else:
                # Re-raise the original exception after fd has been closed
                raise

        self._disk_file_open = True
        return self