示例#1
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False, disk_chunk_size=65536,
                 uid=DEFAULT_UID, gid=DEFAULT_GID):
        self.disk_chunk_size = disk_chunk_size
        device = account
        #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj

        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
        #Absolute path for obj directory.
        self.datadir = os.path.join(path, device, self.name)
        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        if not os.path.exists(self.datadir + '/' + self.obj):
            return

        self.data_file = os.path.join(self.datadir, self.obj)
        self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not self.metadata:
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)

        if not validate_object(self.metadata):
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' +
                                        self.obj)

        self.filter_metadata()

        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        else:
            stats = do_fstat(fd)
            if not stats:
                return
            self._is_dir = stat.S_ISDIR(stats.st_mode)
            obj_size = stats.st_size

        self._metadata = read_metadata(fd)
        if not validate_object(self._metadata):
            create_object_metadata(fd)
            self._metadata = read_metadata(fd)
        assert self._metadata is not None
        self._filter_metadata()

        if self._is_dir:
            do_close(fd)
            obj_size = 0
            self._fd = -1
        else:
            if self._is_object_expired(self._metadata):
                raise DiskFileExpired(metadata=self._metadata)
            self._fd = fd

        self._obj_size = obj_size
        return self
示例#3
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        else:
            stats = do_fstat(fd)
            if not stats:
                return
            self._is_dir = stat.S_ISDIR(stats.st_mode)
            obj_size = stats.st_size

        self._metadata = read_metadata(fd)
        if not validate_object(self._metadata):
            create_object_metadata(fd)
            self._metadata = read_metadata(fd)
        assert self._metadata is not None
        self._filter_metadata()

        if self._is_dir:
            do_close(fd)
            obj_size = 0
            self._fd = -1
        else:
            if self._is_object_expired(self._metadata):
                raise DiskFileExpired(metadata=self._metadata)
            self._fd = fd

        self._obj_size = obj_size
        return self
示例#4
0
 def test_read_metadata_err(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     _xattr_get_err[xkey] = errno.EOPNOTSUPP
     try:
         utils.read_metadata(path)
     except IOError as e:
         assert e.errno == errno.EOPNOTSUPP
         assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
     else:
         self.fail("Expected an IOError exception on get")
示例#5
0
 def test_read_metadata_err(self):
     path = "/tmp/foo/r"
     expected_d = {"a": "y"}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     _xattr_get_err[xkey] = errno.EOPNOTSUPP
     try:
         utils.read_metadata(path)
     except IOError as e:
         assert e.errno == errno.EOPNOTSUPP
         assert _xattr_op_cnt["get"] == 1, "%r" % _xattr_op_cnt
     else:
         self.fail("Expected an IOError exception on get")
示例#6
0
 def test_read_metadata_err(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = serialize_metadata(expected_d)
     _xattr_get_err[xkey] = errno.EOPNOTSUPP
     try:
         utils.read_metadata(path)
     except IOError as e:
         assert e.errno == errno.EOPNOTSUPP
         assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
     else:
         self.fail("Expected an IOError exception on get")
    def _unlinkold(self):
        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains no objects.
            # If it does contain objects, then just remove the object
            # metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted when the
            # container or parent directory is deleted.
            #
            # FIXME: Ideally we should use an atomic metadata update operation
            metadata = read_metadata(self._data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self._data_file, metadata)
            rmobjdir(self._data_file)
        else:
            # Delete file object
            do_unlink(self._data_file)

        # Garbage collection of non-object directories.  Now that we
        # deleted the file, determine if the current directory and any
        # parent directory may be deleted.
        dirname = os.path.dirname(self._data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been found, we can stop
                # garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)
示例#8
0
    def _unlinkold(self):
        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains no objects.
            # If it does contain objects, then just remove the object
            # metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted when the
            # container or parent directory is deleted.
            #
            # FIXME: Ideally we should use an atomic metadata update operation
            metadata = read_metadata(self._data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self._data_file, metadata)
            rmobjdir(self._data_file)
        else:
            # Delete file object
            do_unlink(self._data_file)

        # Garbage collection of non-object directories.  Now that we
        # deleted the file, determine if the current directory and any
        # parent directory may be deleted.
        dirname = os.path.dirname(self._data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been found, we can stop
                # garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)
示例#9
0
    def list_objects_iter(self):
        
        self.update_object_count()
        objects, object_count, bytes_used = self.object_info

        if objects:
            objects.sort()

        container_list = []
        if objects:
            for obj in objects:
                list_item = []
                list_item.append(obj)
                obj_path = os.path.join(self.datadir, obj)
                metadata = read_metadata(obj_path)
                if not metadata or not validate_object(metadata):
                    metadata = create_object_metadata(obj_path)
                if metadata:
#                    list_item.append(metadata[X_TIMESTAMP])
                    list_item.append(int(metadata[X_CONTENT_LENGTH]))
#                    list_item.append(metadata[X_CONTENT_TYPE])
                    list_item.append(metadata[X_ETAG])
                container_list.append(list_item)

        return container_list
示例#10
0
    def delete(self, timestamp):
        """
        Delete the object.

        This implementation creates a tombstone file using the given
        timestamp, and removes any older versions of the object file. Any
        file that has an older timestamp than timestamp will be deleted.

        .. note::

            An implementation is free to use or ignore the timestamp
            parameter.

        :param timestamp: timestamp to compare with each file
        :raises DiskFileError: this implementation will raise the same
                            errors as the `create()` method.
        """
        try:
            metadata = read_metadata(self._data_file)
        except (IOError, OSError) as err:
            if err.errno != errno.ENOENT:
                raise
        else:
            if metadata[X_TIMESTAMP] >= timestamp:
                return

        self._threadpool.run_in_thread(self._unlinkold)

        self._metadata = None
        self._data_file = None
示例#11
0
    def _keep_sys_metadata(self, metadata):
        """
        Make sure system metadata is not lost when writing new user metadata

        This method will read the existing metadata and check for system
        metadata. If there are any, it should be appended to the metadata obj
        the user is trying to write.
        """
        # If metadata has been previously fetched, use that.
        # Stale metadata (outdated size/etag) would've been updated when
        # metadata is fetched for the first time.
        orig_metadata = self._metadata or read_metadata(self._data_file)

        sys_keys = [X_CONTENT_TYPE, X_ETAG, 'name', X_CONTENT_LENGTH,
                    X_OBJECT_TYPE, X_TYPE]

        for key in sys_keys:
            if key in orig_metadata:
                metadata[key] = orig_metadata[key]

        if X_OBJECT_TYPE not in orig_metadata:
            if metadata[X_CONTENT_TYPE].lower() == DIR_TYPE:
                metadata[X_OBJECT_TYPE] = DIR_OBJECT
            else:
                metadata[X_OBJECT_TYPE] = FILE

        if X_TYPE not in orig_metadata:
            metadata[X_TYPE] = OBJECT

        return metadata
示例#12
0
    def delete(self, timestamp):
        """
        Delete the object.

        This implementation creates a tombstone file using the given
        timestamp, and removes any older versions of the object file. Any
        file that has an older timestamp than timestamp will be deleted.

        .. note::

            An implementation is free to use or ignore the timestamp
            parameter.

        :param timestamp: timestamp to compare with each file
        :raises DiskFileError: this implementation will raise the same
                            errors as the `create()` method.
        """
        try:
            metadata = read_metadata(self._data_file)
        except (IOError, OSError) as err:
            if err.errno != errno.ENOENT:
                raise
        else:
            if metadata[X_TIMESTAMP] >= timestamp:
                return

        self._threadpool.run_in_thread(self._unlinkold)

        self._metadata = None
        self._data_file = None
示例#13
0
    def _keep_sys_metadata(self, metadata):
        """
        Make sure system metadata is not lost when writing new user metadata

        This method will read the existing metadata and check for system
        metadata. If there are any, it should be appended to the metadata obj
        the user is trying to write.
        """
        # If metadata has been previously fetched, use that.
        # Stale metadata (outdated size/etag) would've been updated when
        # metadata is fetched for the first time.
        orig_metadata = self._metadata or read_metadata(self._data_file)

        sys_keys = [X_CONTENT_TYPE, X_ETAG, 'name', X_CONTENT_LENGTH,
                    X_OBJECT_TYPE, X_TYPE]

        for key in sys_keys:
            if key in orig_metadata and key not in metadata:
                metadata[key] = orig_metadata[key]

        if X_OBJECT_TYPE not in orig_metadata:
            if metadata[X_CONTENT_TYPE].lower() == DIR_TYPE:
                metadata[X_OBJECT_TYPE] = DIR_OBJECT
            else:
                metadata[X_OBJECT_TYPE] = FILE

        if X_TYPE not in orig_metadata:
            metadata[X_TYPE] = OBJECT

        return metadata
示例#14
0
 def test_read_metadata(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = serialize_metadata(expected_d)
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
示例#15
0
 def test_read_metadata(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' }
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
示例#16
0
 def test_read_metadata(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = serialize_metadata(expected_d)
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d,
                                                             res_d)
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
示例#17
0
 def test_read_metadata_multiple_one_missing(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y' * 150000}
     expected_p = serialize_metadata(expected_d)
     for i in range(0, 2):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert len(expected_p) <= utils.MAX_XATTR_SIZE
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
示例#18
0
 def test_read_metadata_multiple(self):
     path = "/tmp/foo/r"
     expected_d = {"a": "y" * 150000}
     expected_p = serialize_metadata(expected_d)
     for i in range(0, 3):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ""))
         _xattrs[xkey] = expected_p[: utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE :]
     assert not expected_p
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt["get"] == 4, "%r" % _xattr_op_cnt
示例#19
0
 def test_read_metadata_multiple(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' * 150000 }
     expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     for i in range(0,3):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert not expected_p
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
示例#20
0
 def test_read_metadata_multiple_one_missing(self):
     path = "/tmp/foo/r"
     expected_d = {'a': 'y' * 150000}
     expected_p = serialize_metadata(expected_d)
     for i in range(0, 2):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert len(expected_p) <= utils.MAX_XATTR_SIZE
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
示例#21
0
    def list_objects_iter(self, limit, marker, end_marker,
                          prefix, delimiter, path):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
    
        if path:
            prefix = path = path.rstrip('/') + '/'
            delimiter = '/'
        if delimiter and not prefix:
            prefix = ''

        self.update_object_count()

        objects, object_count, bytes_used = self.object_info

        if objects:
            objects.sort()

        if objects and prefix:
            objects = self.filter_prefix(objects, prefix)

        if objects and delimiter:
            objects = self.filter_delimiter(objects, delimiter, prefix)

        if objects and marker:
            objects = self.filter_marker(objects, marker)

        if objects and end_marker:
            objects = self.filter_end_marker(objects, end_marker)

        if objects and limit:
            if len(objects) > limit:
                objects = self.filter_limit(objects, limit)

        container_list = []
        if objects:
            for obj in objects:
                list_item = []
                list_item.append(obj)
                obj_path = os.path.join(self.datadir, obj)
                metadata = read_metadata(obj_path)
                if not metadata or not validate_object(metadata):
                    metadata = create_object_metadata(obj_path)
                if metadata:
#                    list_item.append(metadata[X_TIMESTAMP])
                    list_item.append(int(metadata[X_CONTENT_LENGTH]))
#                    list_item.append(metadata[X_CONTENT_TYPE])
                    list_item.append(metadata[X_ETAG])
                container_list.append(list_item)

        return container_list
示例#22
0
 def test_read_metadata_multiple_one_missing(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' * 150000 }
     expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     for i in range(0,2):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert len(expected_p) <= utils.MAX_XATTR_SIZE
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
     assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys())
示例#23
0
    def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
                          path):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        if path:
            prefix = path = path.rstrip('/') + '/'
            delimiter = '/'
        if delimiter and not prefix:
            prefix = ''

        self.update_object_count()

        objects, object_count, bytes_used = self.object_info

        if objects:
            objects.sort()

        if objects and prefix:
            objects = self.filter_prefix(objects, prefix)

        if objects and delimiter:
            objects = self.filter_delimiter(objects, delimiter, prefix)

        if objects and marker:
            objects = self.filter_marker(objects, marker)

        if objects and end_marker:
            objects = self.filter_end_marker(objects, end_marker)

        if objects and limit:
            if len(objects) > limit:
                objects = self.filter_limit(objects, limit)

        container_list = []
        if objects:
            for obj in objects:
                list_item = []
                list_item.append(obj)
                obj_path = os.path.join(self.datadir, obj)
                metadata = read_metadata(obj_path)
                if not metadata or not validate_object(metadata):
                    metadata = create_object_metadata(obj_path)
                if metadata:
                    list_item.append(metadata[X_TIMESTAMP])
                    list_item.append(int(metadata[X_CONTENT_LENGTH]))
                    list_item.append(metadata[X_CONTENT_TYPE])
                    list_item.append(metadata[X_ETAG])
                container_list.append(list_item)

        return container_list
示例#24
0
    def unlinkold(self, timestamp):
        """
        Remove any older versions of the object file.  Any file that has an
        older timestamp than timestamp will be deleted.

        :param timestamp: timestamp to compare with each file
        """
        if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp:
            return

        assert self.data_file, \
            "Have metadata, %r, but no data_file" % self.metadata

        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains
            # no objects.  If it does contain objects, then just
            # remove the object metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted
            # when the container or parent directory is deleted.
            metadata = read_metadata(self.data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self.data_file, metadata)
            rmobjdir(self.data_file)

        else:
            # Delete file object
            do_unlink(self.data_file)

        # Garbage collection of non-object directories.
        # Now that we deleted the file, determine
        # if the current directory and any parent
        # directory may be deleted.
        dirname = os.path.dirname(self.data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not
            # objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been
                # found, we can stop garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)

        self.metadata = {}
        self.data_file = None
示例#25
0
    def read_metadata(self):
        """
        Return the metadata for an object without requiring the caller to open
        the object first.

        This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
        metadata = disk_file.read_metadata()

        The operations performed here is very similar to those made in open().
        This is to avoid opening and closing of file (two syscalls over
        network). IOW, this optimization addresses the case where the fd
        returned by open() isn't going to be used i.e the file is not read (GET
        or metadata recalculation)

        :returns: metadata dictionary for an object
        :raises DiskFileError: this implementation will raise the same
                            errors as the `open()` method.
        """
        try:
            self._metadata = read_metadata(self._data_file)
        except (OSError, IOError) as err:
            if err.errno in (errno.ENOENT, errno.ESTALE):
                self._disk_file_does_not_exist = True
                raise DiskFileNotExist
            raise err

        if self._metadata and self._is_object_expired(self._metadata):
            raise DiskFileExpired(metadata=self._metadata)

        try:
            self._stat = do_stat(self._data_file)
        except (OSError, IOError) as err:
            if err.errno in (errno.ENOENT, errno.ESTALE):
                self._disk_file_does_not_exist = True
                raise DiskFileNotExist
            raise err

        if not validate_object(self._metadata, self._stat):
            # Metadata is stale/invalid. So open the object for reading
            # to update Etag and other metadata.
            with self.open():
                return self._metadata
        else:
            # Metadata is valid. Don't have to open the file.
            self._filter_metadata()
            return self._metadata
示例#26
0
    def read_metadata(self):
        """
        Return the metadata for an object without requiring the caller to open
        the object first.

        This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
        metadata = disk_file.read_metadata()

        The operations performed here is very similar to those made in open().
        This is to avoid opening and closing of file (two syscalls over
        network). IOW, this optimization addresses the case where the fd
        returned by open() isn't going to be used i.e the file is not read (GET
        or metadata recalculation)

        :returns: metadata dictionary for an object
        :raises DiskFileError: this implementation will raise the same
                            errors as the `open()` method.
        """
        try:
            self._metadata = read_metadata(self._data_file)
        except (OSError, IOError) as err:
            if err.errno in (errno.ENOENT, errno.ESTALE):
                self._disk_file_does_not_exist = True
                raise DiskFileNotExist
            raise err

        if self._metadata and self._is_object_expired(self._metadata):
            raise DiskFileExpired(metadata=self._metadata)

        try:
            self._stat = do_stat(self._data_file)
        except (OSError, IOError) as err:
            if err.errno in (errno.ENOENT, errno.ESTALE):
                self._disk_file_does_not_exist = True
                raise DiskFileNotExist
            raise err

        if not validate_object(self._metadata, self._stat):
            # Metadata is stale/invalid. So open the object for reading
            # to update Etag and other metadata.
            with self.open():
                return self._metadata
        else:
            # Metadata is valid. Don't have to open the file.
            self._filter_metadata()
            return self._metadata
示例#27
0
def _read_metadata(dd):
    """ Filter read metadata so that it always returns a tuple that includes
        some kind of timestamp. With 1.4.8 of the Swift integration the
        timestamps were not stored. Here we fabricate timestamps for volumes
        where the existing data has no timestamp (that is, stored data is not
        a tuple), allowing us a measure of backward compatibility.

        FIXME: At this time it does not appear that the timestamps on each
        metadata are used for much, so this should not hurt anything.
    """
    metadata_i = read_metadata(dd)
    metadata = {}
    timestamp = 0
    for key, value in metadata_i.iteritems():
        if not isinstance(value, tuple):
            value = (value, timestamp)
        metadata[key] = value
    return metadata
示例#28
0
def _read_metadata(dd):
    """ Filter read metadata so that it always returns a tuple that includes
        some kind of timestamp. With 1.4.8 of the Swift integration the
        timestamps were not stored. Here we fabricate timestamps for volumes
        where the existing data has no timestamp (that is, stored data is not
        a tuple), allowing us a measure of backward compatibility.

        FIXME: At this time it does not appear that the timestamps on each
        metadata are used for much, so this should not hurt anything.
    """
    metadata_i = read_metadata(dd)
    metadata = {}
    timestamp = 0
    for key, value in metadata_i.iteritems():
        if not isinstance(value, tuple):
            value = (value, timestamp)
        metadata[key] = value
    return metadata
示例#29
0
    def delete(self, timestamp):
        """
        Delete the object.

        This implementation creates a tombstone file using the given
        timestamp, and removes any older versions of the object file. Any
        file that has an older timestamp than timestamp will be deleted.

        .. note::

            An implementation is free to use or ignore the timestamp
            parameter.

        :param timestamp: timestamp to compare with each file
        :raises DiskFileError: this implementation will raise the same
                            errors as the `create()` method.
        """
        if self._disk_file_does_not_exist:
            # It was determined during disk_file.read_metadata() call that
            # the file does not exist. Don't proceed with deletion on a
            # non-existing path.
            return

        try:
            metadata = self._metadata or read_metadata(self._data_file)
        except (IOError, OSError) as err:
            if err.errno in (errno.ESTALE, errno.ENOENT):
                return
            else:
                raise
        else:
            if metadata[X_TIMESTAMP] >= timestamp:
                return

        self._threadpool.run_in_thread(self._unlinkold)

        self._metadata = None
        self._data_file = None
示例#30
0
    def delete(self, timestamp):
        """
        Delete the object.

        This implementation creates a tombstone file using the given
        timestamp, and removes any older versions of the object file. Any
        file that has an older timestamp than timestamp will be deleted.

        .. note::

            An implementation is free to use or ignore the timestamp
            parameter.

        :param timestamp: timestamp to compare with each file
        :raises DiskFileError: this implementation will raise the same
                            errors as the `create()` method.
        """
        if self._disk_file_does_not_exist:
            # It was determined during disk_file.read_metadata() call that
            # the file does not exist. Don't proceed with deletion on a
            # non-existing path.
            return

        try:
            metadata = self._metadata or read_metadata(self._data_file)
        except (IOError, OSError) as err:
            if err.errno in (errno.ESTALE, errno.ENOENT):
                return
            else:
                raise
        else:
            if metadata[X_TIMESTAMP] >= timestamp:
                return

        self._threadpool.run_in_thread(self._unlinkold)

        self._metadata = None
        self._data_file = None
示例#31
0
    def list_objects_iter(self,
                          limit,
                          marker,
                          end_marker,
                          prefix,
                          delimiter,
                          path=None):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        assert limit >= 0
        assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)

        if path is not None:
            if path:
                prefix = path = path.rstrip('/') + '/'
            else:
                prefix = path
            delimiter = '/'
        elif delimiter and not prefix:
            prefix = ''

        container_list = []

        objects = self._update_object_count()
        if objects:
            objects.sort()
        else:
            return container_list

        if end_marker:
            objects = filter_end_marker(objects, end_marker)

        if marker and marker >= prefix:
            objects = filter_marker(objects, marker)
        elif prefix:
            objects = filter_prefix_as_marker(objects, prefix)

        if prefix is None:
            # No prefix, we don't need to apply the other arguments, we just
            # return what we have.
            pass
        else:
            # We have a non-None (for all intents and purposes it is a string)
            # prefix.
            if not delimiter:
                if not prefix:
                    # We have nothing more to do
                    pass
                else:
                    objects = filter_prefix(objects, prefix)
            else:
                objects = filter_delimiter(objects, delimiter, prefix, marker,
                                           path)

        count = 0
        for obj in objects:
            obj_path = os.path.join(self.datadir, obj)
            metadata = read_metadata(obj_path)
            if not metadata or not validate_object(metadata):
                if delimiter == '/' and obj_path[-1] == delimiter:
                    clean_obj_path = obj_path[:-1]
                else:
                    clean_obj_path = obj_path
                try:
                    metadata = create_object_metadata(clean_obj_path)
                except OSError as e:
                    # FIXME - total hack to get upstream swift ported unit
                    # test cases working for now.
                    if e.errno != errno.ENOENT:
                        raise
            if not Glusterfs._implicit_dir_objects and metadata \
                    and metadata[X_CONTENT_TYPE] == DIR_TYPE \
                    and not dir_is_object(metadata):
                continue
            list_item = []
            list_item.append(obj)
            if metadata:
                list_item.append(metadata[X_TIMESTAMP])
                list_item.append(int(metadata[X_CONTENT_LENGTH]))
                list_item.append(metadata[X_CONTENT_TYPE])
                list_item.append(metadata[X_ETAG])
            container_list.append(list_item)
            count += 1
            if count >= limit:
                break

        return container_list
示例#32
0
    def list_objects_iter(self, limit, marker, end_marker,
                          prefix, delimiter, path=None,
                          storage_policy_index=0,
                          out_content_type=None, reverse=False):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        assert limit >= 0
        assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
        if path is not None:
            if path:
                prefix = path = path.rstrip('/') + '/'
            else:
                prefix = path
            delimiter = '/'
        elif delimiter and not prefix:
            prefix = ''

        container_list = []

        if self.account == 'gsexpiring':
            objects = list_objects_gsexpiring_container(self.datadir)
        else:
            objects = self._update_object_count()
        if objects:
            objects.sort()
        else:
            # No objects in container , return empty list
            return container_list

        if marker and end_marker and reverse:
            marker, end_marker = end_marker, marker

        if end_marker:
            objects = filter_end_marker(objects, end_marker)

        if marker and marker >= prefix:
            objects = filter_marker(objects, marker)
        elif prefix:
            objects = filter_prefix_as_marker(objects, prefix)

        if prefix is None:
            # No prefix, we don't need to apply the other arguments, we just
            # return what we have.
            pass
        else:
            # We have a non-None (for all intents and purposes it is a string)
            # prefix.
            if not delimiter:
                if not prefix:
                    # We have nothing more to do
                    pass
                else:
                    objects = filter_prefix(objects, prefix)
            else:
                objects = filter_delimiter(objects, delimiter, prefix, marker,
                                           path)

        if out_content_type == 'text/plain' or \
                self.account == 'gsexpiring':
            # When out_content_type == 'text/plain':
            #
            # The client is only asking for a plain list of objects and NOT
            # asking for any extended information about objects such as
            # bytes used or etag.
            #
            # When self.account == 'gsexpiring':
            #
            # This is a JSON request sent by the object expirer to list
            # tracker objects in a container in gsexpiring volume.
            # When out_content_type is 'application/json', the caller
            # expects each record entry to have the following ordered
            # fields: (name, timestamp, size, content_type, etag)
            for obj in objects:
                container_list.append((obj, '0', 0, 'text/plain', ''))
                if len(container_list) >= limit:
                        break
            if reverse:
                container_list.reverse()
            return container_list

        count = 0
        for obj in objects:
            obj_path = os.path.join(self.datadir, obj)
            try:
                metadata = read_metadata(obj_path)
            except GlusterFileSystemIOError as err:
                if err.errno in (errno.ENOENT, errno.ESTALE):
                    # obj might have been deleted by another process
                    # since the objects list was originally built
                    continue
                else:
                    raise err
            if not metadata or not validate_object(metadata):
                if delimiter == '/' and obj_path[-1] == delimiter:
                    clean_obj_path = obj_path[:-1]
                else:
                    clean_obj_path = obj_path
                try:
                    metadata = create_object_metadata(clean_obj_path)
                except OSError as e:
                    # FIXME - total hack to get upstream swift ported unit
                    # test cases working for now.
                    if e.errno not in (errno.ENOENT, errno.ESTALE):
                        raise
            if not Glusterfs._implicit_dir_objects and metadata \
                    and metadata[X_CONTENT_TYPE] == DIR_TYPE \
                    and not dir_is_object(metadata):
                continue
            list_item = []

            list_item.append(obj)

            if metadata:
                list_item.append(metadata[X_TIMESTAMP])
                list_item.append(int(metadata[X_CONTENT_LENGTH]))
                list_item.append(metadata[X_CONTENT_TYPE])
                list_item.append(metadata[X_ETAG])
            container_list.append(list_item)
            count += 1
            if count >= limit:
                break
        if reverse:
            container_list.reverse()
        return container_list
示例#33
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False,
                 disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
        self.disk_chunk_size = disk_chunk_size
        self.iter_hook = iter_hook
        obj = obj.strip(os.path.sep)

        if os.path.sep in obj:
            self._obj_path, self._obj = os.path.split(obj)
        else:
            self._obj_path = ''
            self._obj = obj

        if self._obj_path:
            self.name = os.path.join(container, self._obj_path)
        else:
            self.name = container
        # Absolute path for object directory.
        self.datadir = os.path.join(path, device, self.name)
        self.device_path = os.path.join(path, device)
        self._container_path = os.path.join(path, device, container)
        if _use_put_mount:
            self.put_datadir = os.path.join(self.device_path + '_PUT',
                                            self.name)
        else:
            self.put_datadir = self.datadir
        self._is_dir = False
        self.tmppath = None
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.uid = int(uid)
        self.gid = int(gid)
        self.suppress_file_closing = False

        # Don't store a value for data_file until we know it exists.
        self.data_file = None
        data_file = os.path.join(self.put_datadir, self._obj)

        try:
            stats = do_stat(data_file)
        except OSError as err:
            if err.errno == errno.ENOTDIR:
                return
        else:
            if not stats:
                return

        self.data_file = data_file
        self._is_dir = stat.S_ISDIR(stats.st_mode)

        self.metadata = read_metadata(data_file)
        if not self.metadata:
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        if not validate_object(self.metadata):
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        self.filter_metadata()

        if not self._is_dir and keep_data_fp:
            # The caller has an assumption that the "fp" field of this
            # object is an file object if keep_data_fp is set. However,
            # this implementation of the DiskFile object does not need to
            # open the file for internal operations. So if the caller
            # requests it, we'll just open the file for them.
            self.fp = do_open(data_file, 'rb')
示例#34
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False,
                 disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
        self.disk_chunk_size = disk_chunk_size
        self.iter_hook = iter_hook
        # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/,
        # etc.
        obj = obj.strip(os.path.sep)
        if os.path.sep in obj:
            self._obj_path, self._obj = os.path.split(obj)
        else:
            self._obj_path = ''
            self._obj = obj

        if self._obj_path:
            self.name = os.path.join(container, self._obj_path)
        else:
            self.name = container
        # Absolute path for object directory.
        self.datadir = os.path.join(path, device, self.name)
        self.device_path = os.path.join(path, device)
        self._container_path = os.path.join(path, device, container)
        self._is_dir = False
        self.tmppath = None
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.uid = int(uid)
        self.gid = int(gid)

        # Don't store a value for data_file until we know it exists.
        self.data_file = None
        data_file = os.path.join(self.datadir, self._obj)
        if not os.path.exists(data_file):
            return

        self.data_file = os.path.join(data_file)
        self.metadata = read_metadata(data_file)
        if not self.metadata:
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        if not validate_object(self.metadata):
            create_object_metadata(data_file)
            self.metadata = read_metadata(data_file)

        self.filter_metadata()

        if os.path.isdir(data_file):
            self._is_dir = True
        else:
            if keep_data_fp:
                # The caller has an assumption that the "fp" field of this
                # object is an file object if keep_data_fp is set. However,
                # this implementation of the DiskFile object does not need to
                # open the file for internal operations. So if the caller
                # requests it, we'll just open the file for them.
                self.fp = do_open(data_file, 'rb')
示例#35
0
 def test_read_metadata_notfound(self):
     path = "/tmp/foo/r"
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
示例#36
0
 def _set_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
                          metadata)
示例#37
0
    def get_container_timestamp(self, container):
        cont_path = os.path.join(self.datadir, container)
        metadata = read_metadata(cont_path)

        return int(metadata.get(X_PUT_TIMESTAMP, ('0', 0))[0]) or None
示例#38
0
    def list_objects_iter(self, limit, marker, end_marker,
                          prefix, delimiter, path=None):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        assert limit >= 0
        assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)

        if path is not None:
            if path:
                prefix = path = path.rstrip('/') + '/'
            else:
                prefix = path
            delimiter = '/'
        elif delimiter and not prefix:
            prefix = ''

        container_list = []

        objects = self._update_object_count()
        if objects:
            objects.sort()
        else:
            return container_list

        if end_marker:
            objects = filter_end_marker(objects, end_marker)

        if marker and marker >= prefix:
            objects = filter_marker(objects, marker)
        elif prefix:
            objects = filter_prefix_as_marker(objects, prefix)

        if prefix is None:
            # No prefix, we don't need to apply the other arguments, we just
            # return what we have.
            pass
        else:
            # We have a non-None (for all intents and purposes it is a string)
            # prefix.
            if not delimiter:
                if not prefix:
                    # We have nothing more to do
                    pass
                else:
                    objects = filter_prefix(objects, prefix)
            else:
                objects = filter_delimiter(objects, delimiter, prefix, marker,
                                           path)

        count = 0
        for obj in objects:
            obj_path = os.path.join(self.datadir, obj)
            metadata = read_metadata(obj_path)
            if not metadata or not validate_object(metadata):
                if delimiter == '/' and obj_path[-1] == delimiter:
                    clean_obj_path = obj_path[:-1]
                else:
                    clean_obj_path = obj_path
                try:
                    metadata = create_object_metadata(clean_obj_path)
                except OSError as e:
                    # FIXME - total hack to get upstream swift ported unit
                    # test cases working for now.
                    if e.errno != errno.ENOENT:
                        raise
            if Glusterfs.OBJECT_ONLY and metadata \
                    and metadata[X_CONTENT_TYPE] == DIR_TYPE \
                    and not dir_is_object(metadata):
                continue
            list_item = []
            list_item.append(obj)
            if metadata:
                list_item.append(metadata[X_TIMESTAMP])
                list_item.append(int(metadata[X_CONTENT_LENGTH]))
                list_item.append(metadata[X_CONTENT_TYPE])
                list_item.append(metadata[X_ETAG])
            container_list.append(list_item)
            count += 1
            if count >= limit:
                break

        return container_list
示例#39
0
    def get_container_timestamp(self, container):
        cont_path = os.path.join(self.datadir, container)
        metadata = read_metadata(cont_path)

        return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None
示例#40
0
 def _clear_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, obj), metadata)
示例#41
0
    def __init__(self,
                 path,
                 device,
                 partition,
                 account,
                 container,
                 obj,
                 logger,
                 keep_data_fp=False,
                 disk_chunk_size=65536,
                 uid=DEFAULT_UID,
                 gid=DEFAULT_GID):
        self.disk_chunk_size = disk_chunk_size
        device = account
        #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj

        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
        #Absolute path for obj directory.
        self.datadir = os.path.join(path, device, self.name)
        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        if not os.path.exists(self.datadir + '/' + self.obj):
            return

        self.data_file = os.path.join(self.datadir, self.obj)
        self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not self.metadata:
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)

        if not validate_object(self.metadata):
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)

        self.filter_metadata()

        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)
示例#42
0
 def _clear_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, obj),
                          metadata)
示例#43
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        try:
            if not self._stat:
                self._stat = do_fstat(self._fd)
            obj_size = self._stat.st_size

            if not self._metadata:
                self._metadata = read_metadata(self._fd)
            if not validate_object(self._metadata, self._stat):
                self._metadata = create_object_metadata(self._fd, self._stat,
                                                        self._metadata)
            assert self._metadata is not None
            self._filter_metadata()

            if stat.S_ISDIR(self._stat.st_mode):
                do_close(self._fd)
                obj_size = 0
                self._fd = -1
            else:
                if self._is_object_expired(self._metadata):
                    raise DiskFileExpired(metadata=self._metadata)
            self._obj_size = obj_size
        except (OSError, IOError, DiskFileExpired) as err:
            # Something went wrong. Context manager will not call
            # __exit__. So we close the fd manually here.
            self._close_fd()
            if hasattr(err, 'errno') and \
                    err.errno in (errno.ENOENT, errno.ESTALE):
                # Handle races: ENOENT/ESTALE can be raised by read_metadata()
                # call in GlusterFS if file gets deleted by another
                # client after do_open() succeeds
                logging.warn("open(%s) succeeded but one of the subsequent "
                             "syscalls failed with ENOENT/ESTALE. Raising "
                             "DiskFileNotExist." % (self._data_file))
                raise DiskFileNotExist
            else:
                # Re-raise the original exception after fd has been closed
                raise

        self._disk_file_open = True
        return self
示例#44
0
 def _set_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
                          metadata)
示例#45
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
        except GlusterFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        try:
            if not self._stat:
                self._stat = do_fstat(self._fd)
            obj_size = self._stat.st_size

            if not self._metadata:
                self._metadata = read_metadata(self._fd)
            if not validate_object(self._metadata, self._stat):
                self._metadata = create_object_metadata(self._fd, self._stat,
                                                        self._metadata)
            assert self._metadata is not None
            self._filter_metadata()

            if stat.S_ISDIR(self._stat.st_mode):
                do_close(self._fd)
                obj_size = 0
                self._fd = -1
            else:
                if self._is_object_expired(self._metadata):
                    raise DiskFileExpired(metadata=self._metadata)
            self._obj_size = obj_size
        except (OSError, IOError, DiskFileExpired) as err:
            # Something went wrong. Context manager will not call
            # __exit__. So we close the fd manually here.
            self._close_fd()
            if hasattr(err, 'errno') and \
                    err.errno in (errno.ENOENT, errno.ESTALE):
                # Handle races: ENOENT/ESTALE can be raised by read_metadata()
                # call in GlusterFS if file gets deleted by another
                # client after do_open() succeeds
                logging.warn("open(%s) succeeded but one of the subsequent "
                             "syscalls failed with ENOENT/ESTALE. Raising "
                             "DiskFileNotExist." % (self._data_file))
                raise DiskFileNotExist
            else:
                # Re-raise the original exception after fd has been closed
                raise

        self._disk_file_open = True
        return self