Ejemplo n.º 1
0
    def _unlinkold(self):
        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains no objects.
            # If it does contain objects, then just remove the object
            # metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted when the
            # container or parent directory is deleted.
            #
            # FIXME: Ideally we should use an atomic metadata update operation
            metadata = read_metadata(self._data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self._data_file, metadata)
            rmobjdir(self._data_file)
        else:
            # Delete file object
            do_unlink(self._data_file)

        # Garbage collection of non-object directories.  Now that we
        # deleted the file, determine if the current directory and any
        # parent directory may be deleted.
        dirname = os.path.dirname(self._data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been found, we can stop
                # garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)
Ejemplo n.º 2
0
    def _unlinkold(self):
        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains no objects.
            # If it does contain objects, then just remove the object
            # metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted when the
            # container or parent directory is deleted.
            #
            # FIXME: Ideally we should use an atomic metadata update operation
            metadata = read_metadata(self._data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self._data_file, metadata)
            rmobjdir(self._data_file)
        else:
            # Delete file object
            do_unlink(self._data_file)

        # Garbage collection of non-object directories.  Now that we
        # deleted the file, determine if the current directory and any
        # parent directory may be deleted.
        dirname = os.path.dirname(self._data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been found, we can stop
                # garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)
Ejemplo n.º 3
0
 def test_do_unlink(self):
     fd, tmpfile = mkstemp()
     try:
         assert fs.do_unlink(tmpfile) is None
         assert not os.path.exists(tmpfile)
         res = fs.do_unlink(os.path.join('/tmp', str(random.random())))
         assert res is None
     finally:
         os.close(fd)
Ejemplo n.º 4
0
 def test_do_unlink(self):
     fd, tmpfile = mkstemp()
     try:
         assert fs.do_unlink(tmpfile) is None
         assert not os.path.exists(tmpfile)
         res = fs.do_unlink(os.path.join('/tmp', str(random.random())))
         assert res is None
     finally:
         os.close(fd)
Ejemplo n.º 5
0
 def test_do_unlink_err(self):
     tmpdir = mkdtemp()
     try:
         fs.do_unlink(tmpdir)
     except GlusterFileSystemOSError:
         pass
     else:
         self.fail('GlusterFileSystemOSError expected')
     finally:
         os.rmdir(tmpdir)
Ejemplo n.º 6
0
 def test_do_unlink_err(self):
     tmpdir = mkdtemp()
     try:
         fs.do_unlink(tmpdir)
     except GlusterFileSystemOSError:
         pass
     else:
         self.fail('GlusterFileSystemOSError expected')
     finally:
         os.rmdir(tmpdir)
Ejemplo n.º 7
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        # Create /account/container directory structure on mount point root
        try:
            os.makedirs(self._container_path)
        except OSError as err:
            if err.errno != errno.EEXIST:
                raise

        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        attempts = 1
        cur_thread = str(getcurrent())
        while True:
            postfix = md5(self._obj + _cur_host + _cur_pid + cur_thread
                          + str(random.random())).hexdigest()
            tmpfile = '.' + self._obj + '.' + postfix
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except SwiftOnFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory'
                                              % (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.mkstemp(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self._uid, self._gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely since at the time of this writing FUSE does not
            # support it.
            dw = DiskFileWriter(fd, tmppath, self)
            yield dw
        finally:
            dw.close()
            if dw._tmppath:
                do_unlink(dw._tmppath)
Ejemplo n.º 8
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        # Create /account/container directory structure on mount point root
        try:
            os.makedirs(self._container_path)
        except OSError as err:
            if err.errno != errno.EEXIST:
                raise

        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        attempts = 1
        while True:
            # To know more about why following temp file naming convention is
            # used, please read this GlusterFS doc:
            # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations  # noqa
            tmpfile = '.' + self._obj + '.' + uuid4().hex
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except SwiftOnFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory' %
                                              (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError(
                        'DiskFile.mkstemp(): failed to'
                        ' successfully create a temporary file'
                        ' without running into a name conflict'
                        ' after %d of %d attempts for: %s' %
                        (attempts, MAX_OPEN_ATTEMPTS, data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn(
                        "DiskFile.mkstemp(): %s ... retrying in"
                        " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn(
                        "DiskFile.mkstemp(): %s ... retrying in"
                        " 0.1 secs", gerr)
                    attempts += 1
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self._uid, self._gid)
            # NOTE: we do not perform the fallocate() call at all. We ignore
            # it completely since at the time of this writing FUSE does not
            # support it.
            dw = DiskFileWriter(fd, tmppath, self)
            yield dw
        finally:
            dw.close()
            if dw._tmppath:
                do_unlink(dw._tmppath)
Ejemplo n.º 9
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        # Create /account/container directory structure on mount point root
        try:
            os.makedirs(self._container_path)
        except OSError as err:
            if err.errno != errno.EEXIST:
                raise

        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        fd = None
        attempts = 1
        while True:
            # To know more about why following temp file naming convention is
            # used, please read this GlusterFS doc:
            # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations  # noqa
            tmpfile = '.' + self._obj + '.' + uuid4().hex
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except SwiftOnFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory' %
                                              (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError(
                        'DiskFile.create(): failed to'
                        ' successfully create a temporary file'
                        ' without running into a name conflict'
                        ' after %d of %d attempts for: %s' %
                        (attempts, MAX_OPEN_ATTEMPTS, data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn(
                        "DiskFile.create(): %s ... retrying in"
                        " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # ENOENT
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    # Handle race:
                    # This can be the issue when memcache has cached that the
                    # container exists. If someone removes the container dir
                    # from filesystem, it's not reflected in memcache. So
                    # swift reports that the container exists and this code
                    # tries to create a file in a directory that does not
                    # exist. However, it's wrong to create the container here.
                    _random_sleep()
                    logging.warn(
                        "DiskFile.create(): %s ... retrying in"
                        " 0.1 secs", gerr)
                    attempts += 1
                    if attempts > 2:
                        # Ideally we would want to return 404 indicating that
                        # the container itself does not exist. Can't be done
                        # though as the caller won't catch DiskFileNotExist.
                        # We raise an exception with a meaningful name for
                        # correctness.
                        logging.warn("Container dir %s does not exist",
                                     self._container_path)
                        raise DiskFileContainerDoesNotExist
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            if size is not None and size > 0:
                try:
                    fallocate(fd, size)
                except OSError as err:
                    if err.errno in (errno.ENOSPC, errno.EDQUOT):
                        raise DiskFileNoSpace()
                    raise
            # Ensure it is properly owned before we make it available.
            if not ((self._uid == DEFAULT_UID) and (self._gid == DEFAULT_GID)):
                # If both UID and GID is -1 (default values), it has no effect.
                # So don't do a fchown.
                # Further, at the time of this writing, UID and GID information
                # is not passed to DiskFile.
                do_fchown(fd, self._uid, self._gid)
            dw = DiskFileWriter(fd, tmppath, self)
            # It's now the responsibility of DiskFileWriter to close this fd.
            fd = None
            yield dw
        finally:
            if dw:
                dw.close()
                if dw._tmppath:
                    do_unlink(dw._tmppath)
Ejemplo n.º 10
0
    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        # Create /account/container directory structure on mount point root
        try:
            os.makedirs(self._container_path)
        except OSError as err:
            if err.errno != errno.EEXIST:
                raise

        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        fd = None
        attempts = 1
        while True:
            # To know more about why following temp file naming convention is
            # used, please read this GlusterFS doc:
            # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations  # noqa
            tmpfile = '.' + self._obj + '.' + uuid4().hex
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except SwiftOnFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory'
                                              % (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.create(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # ENOENT
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    # Handle race:
                    # This can be the issue when memcache has cached that the
                    # container exists. If someone removes the container dir
                    # from filesystem, it's not reflected in memcache. So
                    # swift reports that the container exists and this code
                    # tries to create a file in a directory that does not
                    # exist. However, it's wrong to create the container here.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                    if attempts > 2:
                        # Ideally we would want to return 404 indicating that
                        # the container itself does not exist. Can't be done
                        # though as the caller won't catch DiskFileNotExist.
                        # We raise an exception with a meaningful name for
                        # correctness.
                        logging.warn("Container dir %s does not exist",
                                     self._container_path)
                        raise DiskFileContainerDoesNotExist
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.create(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            if size is not None and size > 0:
                try:
                    fallocate(fd, size)
                except OSError as err:
                    if err.errno in (errno.ENOSPC, errno.EDQUOT):
                        raise DiskFileNoSpace()
                    raise
            # Ensure it is properly owned before we make it available.
            if not ((self._uid == DEFAULT_UID) and (self._gid == DEFAULT_GID)):
                # If both UID and GID is -1 (default values), it has no effect.
                # So don't do a fchown.
                # Further, at the time of this writing, UID and GID information
                # is not passed to DiskFile.
                do_fchown(fd, self._uid, self._gid)
            dw = DiskFileWriter(fd, tmppath, self)
            # It's now the responsibility of DiskFileWriter to close this fd.
            fd = None
            yield dw
        finally:
            if dw:
                dw.close()
                if dw._tmppath:
                    do_unlink(dw._tmppath)