def test_read_metadata_err(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = serialize_metadata(expected_d)
     _xattr_get_err[xkey] = errno.EOPNOTSUPP
     try:
         utils.read_metadata(path)
     except IOError as e:
         assert e.errno == errno.EOPNOTSUPP
         assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
     else:
         self.fail("Expected an IOError exception on get")
Esempio n. 2
0
    def read_metadata(self):
        """
        Return the metadata for an object without opening the object's file on
        disk.

        :returns: metadata dictionary for an object
        :raises DiskFileError: this implementation will raise the same
                            errors as the `open()` method.
        """
        # FIXME: pull a lot of this and the copy of it from open() out to
        # another function

        # Do not actually open the file, in order to duck hpssfs checksum
        # validation and resulting timeouts
        # This means we do a few things DiskFile.open() does.
        try:
            if not self._stat:
                self._stat = do_stat(self._data_file)
            self._is_dir = stat.S_ISDIR(self._stat.st_mode)
            self._metadata = read_metadata(self._data_file)
        except SwiftOnFileSystemOSError:
            raise DiskFileNotExist
        if not self._validate_object_metadata():
            self._create_object_metadata(self._data_file)
        self._filter_metadata()
        return self._metadata
Esempio n. 3
0
    def delete(self, timestamp):
        """
        Delete the object.

        This implementation creates a tombstone file using the given
        timestamp, and removes any older versions of the object file. Any
        file that has an older timestamp than timestamp will be deleted.

        .. note::

            An implementation is free to use or ignore the timestamp
            parameter.

        :param timestamp: timestamp to compare with each file
        :raises DiskFileError: this implementation will raise the same
                            errors as the `create()` method.
        """
        try:
            metadata = read_metadata(self._data_file)
        except (IOError, OSError) as err:
            if err.errno != errno.ENOENT:
                raise
        else:
            if metadata[X_TIMESTAMP] >= timestamp:
                return

        self._unlinkold()

        self._metadata = None
        self._data_file = None
Esempio n. 4
0
    def _unlinkold(self):
        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains no objects.
            # If it does contain objects, then just remove the object
            # metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted when the
            # container or parent directory is deleted.
            #
            # FIXME: Ideally we should use an atomic metadata update operation
            metadata = read_metadata(self._data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self._data_file, metadata)
            rmobjdir(self._data_file)
        else:
            # Delete file object
            do_unlink(self._data_file)

        # Garbage collection of non-object directories.  Now that we
        # deleted the file, determine if the current directory and any
        # parent directory may be deleted.
        dirname = os.path.dirname(self._data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been found, we can stop
                # garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)
 def test_read_metadata(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = serialize_metadata(expected_d)
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
 def test_read_metadata_multiple_one_missing(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y' * 150000}
     expected_p = serialize_metadata(expected_d)
     for i in range(0, 2):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert len(expected_p) <= utils.MAX_XATTR_SIZE
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
 def test_read_metadata_multiple(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y' * 150000}
     expected_p = serialize_metadata(expected_d)
     for i in range(0, 3):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert not expected_p
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 4, "%r" % _xattr_op_cnt
Esempio n. 8
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            self._logger.debug("DiskFile: Opening %s" % self._data_file)
            self._stat = do_stat(self._data_file)
            self._is_dir = stat.S_ISDIR(self._stat.st_mode)
            if not self._is_dir:
                self._fd = do_open(self._data_file, hpss.O_RDONLY)
                self._obj_size = self._stat.st_size
            else:
                self._fd = -1
                self._obj_size = 0
        except SwiftOnFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        try:
            self._logger.debug("DiskFile: Reading metadata")
            self._metadata = read_metadata(self._data_file)
            if not self._validate_object_metadata():
                self._create_object_metadata(self._data_file)

            assert self._metadata is not None
            self._filter_metadata()

            if not self._is_dir:
                if self._is_object_expired(self._metadata):
                    raise DiskFileExpired(metadata=self._metadata)

        except (OSError, IOError, DiskFileExpired) as err:
            # Something went wrong. Context manager will not call
            # __exit__. So we close the fd manually here.
            self._close_fd()
            if hasattr(err, 'errno') and err.errno == errno.ENOENT:
                # Handle races: ENOENT can be raised by read_metadata()
                # call in GlusterFS if file gets deleted by another
                # client after do_open() succeeds
                logging.warn("open(%s) succeeded but one of the subsequent "
                             "syscalls failed with ENOENT. Raising "
                             "DiskFileNotExist." % (self._data_file))
                raise DiskFileNotExist
            else:
                # Re-raise the original exception after fd has been closed
                raise

        return self
 def _clear_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, obj),
                          metadata)
Esempio n. 10
0
 def _set_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
                          metadata)
Esempio n. 11
0
 def test_read_metadata_notfound(self):
     path = "/tmp/foo/r"
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt