def test_do_stat_eio_twice(self):
        count = [0]
        _os_stat = os.stat

        def mock_os_stat_eio(path):
            count[0] += 1
            if count[0] <= 2:
                raise OSError(errno.EIO, os.strerror(errno.EIO))
            return _os_stat(path)

        with patch('os.stat', mock_os_stat_eio):
            fs.do_stat('/tmp') is not None
    def test_do_stat_eio_ten(self):

        def mock_os_stat_eio(path):
            raise OSError(errno.EIO, os.strerror(errno.EIO))

        try:
            with patch('os.stat', mock_os_stat_eio):
                fs.do_stat('/tmp')
        except SwiftOnFileSystemOSError:
            pass
        else:
            self.fail("SwiftOnFileSystemOSError expected")
    def test_do_stat_err(self):

        def mock_os_stat_eacces(path):
            raise OSError(errno.EACCES, os.strerror(errno.EACCES))

        try:
            with patch('os.stat', mock_os_stat_eacces):
                fs.do_stat('/tmp')
        except SwiftOnFileSystemOSError:
            pass
        else:
            self.fail("SwiftOnFileSystemOSError expected")
Esempio n. 4
0
    def read_metadata(self):
        """
        Return the metadata for an object without opening the object's file on
        disk.

        :returns: metadata dictionary for an object
        :raises DiskFileError: this implementation will raise the same
                            errors as the `open()` method.
        """
        # FIXME: pull a lot of this and the copy of it from open() out to
        # another function

        # Do not actually open the file, in order to duck hpssfs checksum
        # validation and resulting timeouts
        # This means we do a few things DiskFile.open() does.
        try:
            if not self._stat:
                self._stat = do_stat(self._data_file)
            self._is_dir = stat.S_ISDIR(self._stat.st_mode)
            self._metadata = read_metadata(self._data_file)
        except SwiftOnFileSystemOSError:
            raise DiskFileNotExist
        if not self._validate_object_metadata():
            self._create_object_metadata(self._data_file)
        self._filter_metadata()
        return self._metadata
Esempio n. 5
0
def get_object_metadata(obj_path_or_fd, stats=None):
    """
    Return metadata of object.
    """
    if not stats:
        if isinstance(obj_path_or_fd, int):
            # We are given a file descriptor, so this is an invocation from the
            # DiskFile.open() method.
            stats = do_fstat(obj_path_or_fd)
        else:
            # We are given a path to the object when the
            # DiskDir.list_objects_iter method invokes us.
            stats = do_stat(obj_path_or_fd)

    if not stats:
        metadata = {}
    else:
        is_dir = stat.S_ISDIR(stats.st_mode)
        metadata = {
            X_TYPE: OBJECT,
            X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
            X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
            X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
            X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
            X_MTIME: 0 if is_dir else normalize_timestamp(stats.st_mtime),
            X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path_or_fd)}
    return metadata
    def test_do_stat(self):
        tmpdir = mkdtemp()
        try:
            fd, tmpfile = mkstemp(dir=tmpdir)
            buf1 = os.stat(tmpfile)
            buf2 = fs.do_stat(tmpfile)

            assert buf1 == buf2
        finally:
            os.close(fd)
            os.remove(tmpfile)
            os.rmdir(tmpdir)
Esempio n. 7
0
    def _validate_object_metadata(self):
        # Has no Swift specific metadata saved as xattr. Probably because
        # object was added/replaced through filesystem interface.
        if not self._metadata and not self._is_dir:
            self._file_has_changed = True
            return False

        required_keys = \
            (X_TIMESTAMP, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_ETAG,
             # SOF specific keys
             X_TYPE, X_OBJECT_TYPE)

        if not all(k in self._metadata for k in required_keys):
            # At least one of the required keys does not exist
            return False

        if not self._is_dir:
            # X_MTIME is a new key added recently, newer objects will
            # have the key set during PUT.
            if X_MTIME in self._metadata:
                # Check if the file has been modified through filesystem
                # interface by comparing mtime stored in xattr during PUT
                # and current mtime of file.
                obj_stat = do_stat(self._data_file)
                if normalize_timestamp(self._metadata[X_MTIME]) != \
                        normalize_timestamp(obj_stat.hpss_st_mtime):
                    self._file_has_changed = True
                    return False
            else:
                # Without X_MTIME key, comparing md5sum is the only way
                # to determine if file has changed or not. This is inefficient
                # but there's no other way!
                self._etag = get_etag(self._data_file)
                if self._etag != self._metadata[X_ETAG]:
                    self._file_has_changed = True
                    return False
                else:
                    # Checksums are same; File has not changed. For the next
                    # GET request on same file, we don't compute md5sum again!
                    # This is achieved by setting X_MTIME to mtime in
                    # _create_object_metadata()
                    return False

        if self._metadata[X_TYPE] == OBJECT:
            return True

        return False
Esempio n. 8
0
def get_object_metadata(obj_path, stats=None):
    """
    Return metadata of object.
    """
    if not stats:
        stats = do_stat(obj_path)

    if not stats:
        metadata = {}
    else:
        is_dir = stat.S_ISDIR(stats.st_mode)
        metadata = {
            X_TYPE: OBJECT,
            X_TIMESTAMP: normalize_timestamp(stats.hpss_st_ctime),
            X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
            X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
            X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
            X_MTIME: 0 if is_dir else normalize_timestamp(stats.hpss_st_mtime),
            X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path)}
    return metadata
Esempio n. 9
0
def make_directory(full_path, uid, gid, metadata=None):
    """
    Make a directory and change the owner ship as specified, and potentially
    creating the object metadata if requested.
    """
    try:
        do_mkdir(full_path)
    except OSError as err:
        if err.errno == errno.ENOENT:
            # Tell the caller some directory of the parent path does not
            # exist.
            return False, metadata
        elif err.errno == errno.EEXIST:
            # Possible race, in that the caller invoked this method when it
            # had previously determined the file did not exist.
            #
            # FIXME: When we are confident, remove this stat() call as it is
            # not necessary.
            try:
                stats = do_stat(full_path)
            except SwiftOnFileSystemOSError as serr:
                # FIXME: Ideally we'd want to return an appropriate error
                # message and code in the PUT Object REST API response.
                raise DiskFileError("make_directory: mkdir failed"
                                    " because path %s already exists, and"
                                    " a subsequent stat on that same"
                                    " path failed (%s)" % (full_path,
                                                           str(serr)))
            else:
                is_dir = stat.S_ISDIR(stats.st_mode)
                if not is_dir:
                    # FIXME: Ideally we'd want to return an appropriate error
                    # message and code in the PUT Object REST API response.
                    raise AlreadyExistsAsFile("make_directory:"
                                              " mkdir failed on path %s"
                                              " because it already exists"
                                              " but not as a directory"
                                              % (full_path))
            return True, metadata
        elif err.errno == errno.ENOTDIR:
            # FIXME: Ideally we'd want to return an appropriate error
            # message and code in the PUT Object REST API response.
            raise AlreadyExistsAsFile("make_directory:"
                                      " mkdir failed because some "
                                      "part of path %s is not in fact"
                                      " a directory" % (full_path))
        elif err.errno == errno.EIO:
            # Sometimes Fuse will return an EIO error when it does not know
            # how to handle an unexpected, but transient situation. It is
            # possible the directory now exists, stat() it to find out after a
            # short period of time.
            _random_sleep()
            try:
                stats = do_stat(full_path)
            except SwiftOnFileSystemOSError as serr:
                if serr.errno == errno.ENOENT:
                    errmsg = "make_directory: mkdir failed on" \
                             " path %s (EIO), and a subsequent stat on" \
                             " that same path did not find the file." % (
                                 full_path,)
                else:
                    errmsg = "make_directory: mkdir failed on" \
                             " path %s (%s), and a subsequent stat on" \
                             " that same path failed as well (%s)" % (
                                 full_path, str(err), str(serr))
                raise DiskFileError(errmsg)
            else:
                if not stats:
                    errmsg = "make_directory: mkdir failed on" \
                             " path %s (EIO), and a subsequent stat on" \
                             " that same path did not find the file." % (
                                 full_path,)
                    raise DiskFileError(errmsg)
                else:
                    # The directory at least exists now
                    is_dir = stat.S_ISDIR(stats.st_mode)
                    if is_dir:
                        # Dump the stats to the log with the original exception
                        logging.warn("make_directory: mkdir initially"
                                     " failed on path %s (%s) but a stat()"
                                     " following that succeeded: %r" %
                                     (full_path, str(err), stats))
                        # Assume another entity took care of the proper setup.
                        return True, metadata
                    else:
                        raise DiskFileError("make_directory: mkdir"
                                            " initially failed on path %s (%s)"
                                            " but now we see that it exists"
                                            " but is not a directory (%r)" %
                                            (full_path, str(err), stats))
        else:
            # Some other potentially rare exception occurred that does not
            # currently warrant a special log entry to help diagnose.
            raise DiskFileError("make_directory: mkdir failed on"
                                " path %s (%s)" % (full_path, str(err)))
    else:
        if metadata:
            # We were asked to set the initial metadata for this object.
            metadata_orig = get_object_metadata(full_path)
            metadata_orig.update(metadata)
            write_metadata(full_path, metadata_orig)
            metadata = metadata_orig

        # We created it, so we are reponsible for always setting the proper
        # ownership.
        do_chown(full_path, uid, gid)
        return True, metadata
Esempio n. 10
0
    def open(self):
        """
        Open the object.

        This implementation opens the data file representing the object, reads
        the associated metadata in the extended attributes, additionally
        combining metadata from fast-POST `.meta` files.

        .. note::

            An implementation is allowed to raise any of the following
            exceptions, but is only required to raise `DiskFileNotExist` when
            the object representation does not exist.

        :raises DiskFileNotExist: if the object does not exist
        :raises DiskFileExpired: if the object has expired
        :returns: itself for use as a context manager
        """
        # Writes are always performed to a temporary file
        try:
            self._logger.debug("DiskFile: Opening %s" % self._data_file)
            self._stat = do_stat(self._data_file)
            self._is_dir = stat.S_ISDIR(self._stat.st_mode)
            if not self._is_dir:
                self._fd = do_open(self._data_file, hpss.O_RDONLY)
                self._obj_size = self._stat.st_size
            else:
                self._fd = -1
                self._obj_size = 0
        except SwiftOnFileSystemOSError as err:
            if err.errno in (errno.ENOENT, errno.ENOTDIR):
                # If the file does exist, or some part of the path does not
                # exist, raise the expected DiskFileNotExist
                raise DiskFileNotExist
            raise
        try:
            self._logger.debug("DiskFile: Reading metadata")
            self._metadata = read_metadata(self._data_file)
            if not self._validate_object_metadata():
                self._create_object_metadata(self._data_file)

            assert self._metadata is not None
            self._filter_metadata()

            if not self._is_dir:
                if self._is_object_expired(self._metadata):
                    raise DiskFileExpired(metadata=self._metadata)

        except (OSError, IOError, DiskFileExpired) as err:
            # Something went wrong. Context manager will not call
            # __exit__. So we close the fd manually here.
            self._close_fd()
            if hasattr(err, 'errno') and err.errno == errno.ENOENT:
                # Handle races: ENOENT can be raised by read_metadata()
                # call in GlusterFS if file gets deleted by another
                # client after do_open() succeeds
                logging.warn("open(%s) succeeded but one of the subsequent "
                             "syscalls failed with ENOENT. Raising "
                             "DiskFileNotExist." % (self._data_file))
                raise DiskFileNotExist
            else:
                # Re-raise the original exception after fd has been closed
                raise

        return self
Esempio n. 11
0
    def _finalize_put(self, metadata):
        # Write out metadata before fsync() to ensure it is also forced to
        # disk.
        write_metadata(self._tmppath, metadata)

        do_fsync(self._fd)

        self.set_checksum(metadata['ETag'])

        # At this point we know that the object's full directory path
        # exists, so we can just rename it directly without using Swift's
        # swift.common.utils.renamer(), which makes the directory path and
        # adds extra stat() calls.
        df = self._disk_file
        attempts = 1
        while True:
            try:
                do_rename(self._tmppath, df._data_file)
            except OSError as err:
                if err.errno in (errno.ENOENT, errno.EIO) \
                        and attempts < MAX_RENAME_ATTEMPTS:
                    # FIXME: Why either of these two error conditions is
                    # happening is unknown at this point. This might be a
                    # FUSE issue of some sort or a possible race
                    # condition. So let's sleep on it, and double check
                    # the environment after a good nap.
                    _random_sleep()
                    # Tease out why this error occurred. The man page for
                    # rename reads:
                    #   "The link named by tmppath does not exist; or, a
                    #    directory component in data_file does not exist;
                    #    or, tmppath or data_file is an empty string."
                    assert len(self._tmppath) > 0 and len(df._data_file) > 0
                    tpstats = do_stat(self._tmppath)
                    tfstats = do_fstat(self._fd)
                    assert tfstats
                    if not tpstats or tfstats.st_ino != tpstats.st_ino:
                        # Temporary file name conflict
                        raise DiskFileError(
                            'DiskFile.put(): temporary file, %s, was'
                            ' already renamed (targeted for %s)' % (
                                self._tmppath, df._data_file))
                    else:
                        # Data file target name now has a bad path!
                        dfstats = do_stat(df._put_datadir)
                        if not dfstats:
                            raise DiskFileError(
                                'DiskFile.put(): path to object, %s, no'
                                ' longer exists (targeted for %s)' % (
                                    df._put_datadir, df._data_file))
                        else:
                            is_dir = stat.S_ISDIR(dfstats.st_mode)
                            if not is_dir:
                                raise DiskFileError(
                                    'DiskFile.put(): path to object, %s,'
                                    ' no longer a directory (targeted for'
                                    ' %s)' % (self._put_datadir,
                                              df._data_file))
                            else:
                                # Let's retry since everything looks okay
                                logging.warn(
                                    "DiskFile.put(): rename('%s','%s')"
                                    " initially failed (%s) but a"
                                    " stat('%s') following that succeeded:"
                                    " %r" % (
                                        self._tmppath, df._data_file, str(err),
                                        df._put_datadir, dfstats))
                                attempts += 1
                                continue
                else:
                    raise SwiftOnFileSystemOSError(
                        err.errno, "%s, rename('%s', '%s')" % (
                            err.strerror, self._tmppath, df._data_file))
            else:
                # Success!
                break

        # Close here so the calling context does not have to perform this
        # in a thread.
        self.close()
 def test_do_stat_enoent(self):
     res = fs.do_stat(os.path.join('/tmp', str(random.random())))
     assert res is None