Esempio n. 1
0
    def _unlinkold(self):
        if self._is_dir:
            # Marker, or object, directory.
            #
            # Delete from the filesystem only if it contains no objects.
            # If it does contain objects, then just remove the object
            # metadata tag which will make this directory a
            # fake-filesystem-only directory and will be deleted when the
            # container or parent directory is deleted.
            #
            # FIXME: Ideally we should use an atomic metadata update operation
            metadata = read_metadata(self._data_file)
            if dir_is_object(metadata):
                metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
                write_metadata(self._data_file, metadata)
            rmobjdir(self._data_file)
        else:
            # Delete file object
            do_unlink(self._data_file)

        # Garbage collection of non-object directories.  Now that we
        # deleted the file, determine if the current directory and any
        # parent directory may be deleted.
        dirname = os.path.dirname(self._data_file)
        while dirname and dirname != self._container_path:
            # Try to remove any directories that are not objects.
            if not rmobjdir(dirname):
                # If a directory with objects has been found, we can stop
                # garabe collection
                break
            else:
                dirname = os.path.dirname(dirname)
Esempio n. 2
0
    def _create_object_metadata(self, file_path):
        if self._etag is None:
            self._etag = md5().hexdigest() if self._is_dir \
                else get_etag(file_path)

        if self._file_has_changed or (X_TIMESTAMP not in self._metadata):
            timestamp = normalize_timestamp(self._stat.st_mtime)
        else:
            timestamp = self._metadata[X_TIMESTAMP]

        metadata = {
            X_TYPE: OBJECT,
            X_TIMESTAMP: timestamp,
            X_CONTENT_TYPE: DIR_TYPE if self._is_dir else FILE_TYPE,
            X_OBJECT_TYPE: DIR_NON_OBJECT if self._is_dir else FILE,
            X_CONTENT_LENGTH: 0 if self._is_dir else self._stat.st_size,
            X_ETAG: self._etag}

        # Add X_MTIME key if object is a file
        if not self._is_dir:
            metadata[X_MTIME] = normalize_timestamp(self._stat.hpss_st_mtime)

        meta_new = self._metadata.copy()
        meta_new.update(metadata)
        if self._metadata != meta_new:
            write_metadata(file_path, meta_new)
            # Avoid additional read_metadata() later
            self._metadata = meta_new
 def test_write_metadata(self):
     path = "/tmp/foo/w"
     orig_d = {'bar': 'foo'}
     utils.write_metadata(path, orig_d)
     xkey = _xkey(path, utils.METADATA_KEY)
     assert len(_xattrs.keys()) == 1
     assert xkey in _xattrs
     assert orig_d == deserialize_metadata(_xattrs[xkey])
     assert _xattr_op_cnt['set'] == 1
Esempio n. 4
0
    def write_metadata(self, metadata):
        """
        Write a block of metadata to an object without requiring the caller to
        open the object first.

        :param metadata: dictionary of metadata to be associated with the
                         object
        :raises DiskFileError: this implementation will raise the same
                            errors as the `create()` method.
        """
        metadata = self._keep_sys_metadata(metadata)
        data_file = os.path.join(self._put_datadir, self._obj)
        write_metadata(data_file, metadata)
 def test_write_metadata_err(self):
     path = "/tmp/foo/w"
     orig_d = {'bar': 'foo'}
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattr_set_err[xkey] = errno.EOPNOTSUPP
     try:
         utils.write_metadata(path, orig_d)
     except IOError as e:
         assert e.errno == errno.EOPNOTSUPP
         assert len(_xattrs.keys()) == 0
         assert _xattr_op_cnt['set'] == 1
     else:
         self.fail("Expected an IOError exception on write")
 def test_write_metadata_multiple(self):
     # At 64 KB an xattr key/value pair, this should generate three keys.
     path = "/tmp/foo/w"
     orig_d = {'bar': 'x' * 150000}
     utils.write_metadata(path, orig_d)
     assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys())
     payload = ''
     for i in range(0, 3):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         assert xkey in _xattrs
         assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE
         payload += _xattrs[xkey]
     assert orig_d == deserialize_metadata(payload)
     assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt
    def test_write_metadata_space_err(self):

        def _mock_xattr_setattr(item, name, value):
            raise IOError(errno.ENOSPC, os.strerror(errno.ENOSPC))

        with patch('xattr.setxattr', _mock_xattr_setattr):
            path = "/tmp/foo/w"
            orig_d = {'bar': 'foo'}
            try:
                utils.write_metadata(path, orig_d)
            except DiskFileNoSpace:
                pass
            else:
                self.fail("Expected DiskFileNoSpace exception")
            fd = 0
            try:
                utils.write_metadata(fd, orig_d)
            except DiskFileNoSpace:
                pass
            else:
                self.fail("Expected DiskFileNoSpace exception")
Esempio n. 8
0
def make_directory(full_path, uid, gid, metadata=None):
    """
    Make a directory and change the owner ship as specified, and potentially
    creating the object metadata if requested.
    """
    try:
        do_mkdir(full_path)
    except OSError as err:
        if err.errno == errno.ENOENT:
            # Tell the caller some directory of the parent path does not
            # exist.
            return False, metadata
        elif err.errno == errno.EEXIST:
            # Possible race, in that the caller invoked this method when it
            # had previously determined the file did not exist.
            #
            # FIXME: When we are confident, remove this stat() call as it is
            # not necessary.
            try:
                stats = do_stat(full_path)
            except SwiftOnFileSystemOSError as serr:
                # FIXME: Ideally we'd want to return an appropriate error
                # message and code in the PUT Object REST API response.
                raise DiskFileError("make_directory: mkdir failed"
                                    " because path %s already exists, and"
                                    " a subsequent stat on that same"
                                    " path failed (%s)" % (full_path,
                                                           str(serr)))
            else:
                is_dir = stat.S_ISDIR(stats.st_mode)
                if not is_dir:
                    # FIXME: Ideally we'd want to return an appropriate error
                    # message and code in the PUT Object REST API response.
                    raise AlreadyExistsAsFile("make_directory:"
                                              " mkdir failed on path %s"
                                              " because it already exists"
                                              " but not as a directory"
                                              % (full_path))
            return True, metadata
        elif err.errno == errno.ENOTDIR:
            # FIXME: Ideally we'd want to return an appropriate error
            # message and code in the PUT Object REST API response.
            raise AlreadyExistsAsFile("make_directory:"
                                      " mkdir failed because some "
                                      "part of path %s is not in fact"
                                      " a directory" % (full_path))
        elif err.errno == errno.EIO:
            # Sometimes Fuse will return an EIO error when it does not know
            # how to handle an unexpected, but transient situation. It is
            # possible the directory now exists, stat() it to find out after a
            # short period of time.
            _random_sleep()
            try:
                stats = do_stat(full_path)
            except SwiftOnFileSystemOSError as serr:
                if serr.errno == errno.ENOENT:
                    errmsg = "make_directory: mkdir failed on" \
                             " path %s (EIO), and a subsequent stat on" \
                             " that same path did not find the file." % (
                                 full_path,)
                else:
                    errmsg = "make_directory: mkdir failed on" \
                             " path %s (%s), and a subsequent stat on" \
                             " that same path failed as well (%s)" % (
                                 full_path, str(err), str(serr))
                raise DiskFileError(errmsg)
            else:
                if not stats:
                    errmsg = "make_directory: mkdir failed on" \
                             " path %s (EIO), and a subsequent stat on" \
                             " that same path did not find the file." % (
                                 full_path,)
                    raise DiskFileError(errmsg)
                else:
                    # The directory at least exists now
                    is_dir = stat.S_ISDIR(stats.st_mode)
                    if is_dir:
                        # Dump the stats to the log with the original exception
                        logging.warn("make_directory: mkdir initially"
                                     " failed on path %s (%s) but a stat()"
                                     " following that succeeded: %r" %
                                     (full_path, str(err), stats))
                        # Assume another entity took care of the proper setup.
                        return True, metadata
                    else:
                        raise DiskFileError("make_directory: mkdir"
                                            " initially failed on path %s (%s)"
                                            " but now we see that it exists"
                                            " but is not a directory (%r)" %
                                            (full_path, str(err), stats))
        else:
            # Some other potentially rare exception occurred that does not
            # currently warrant a special log entry to help diagnose.
            raise DiskFileError("make_directory: mkdir failed on"
                                " path %s (%s)" % (full_path, str(err)))
    else:
        if metadata:
            # We were asked to set the initial metadata for this object.
            metadata_orig = get_object_metadata(full_path)
            metadata_orig.update(metadata)
            write_metadata(full_path, metadata_orig)
            metadata = metadata_orig

        # We created it, so we are reponsible for always setting the proper
        # ownership.
        do_chown(full_path, uid, gid)
        return True, metadata
Esempio n. 9
0
    def _finalize_put(self, metadata):
        # Write out metadata before fsync() to ensure it is also forced to
        # disk.
        write_metadata(self._tmppath, metadata)

        do_fsync(self._fd)

        self.set_checksum(metadata['ETag'])

        # At this point we know that the object's full directory path
        # exists, so we can just rename it directly without using Swift's
        # swift.common.utils.renamer(), which makes the directory path and
        # adds extra stat() calls.
        df = self._disk_file
        attempts = 1
        while True:
            try:
                do_rename(self._tmppath, df._data_file)
            except OSError as err:
                if err.errno in (errno.ENOENT, errno.EIO) \
                        and attempts < MAX_RENAME_ATTEMPTS:
                    # FIXME: Why either of these two error conditions is
                    # happening is unknown at this point. This might be a
                    # FUSE issue of some sort or a possible race
                    # condition. So let's sleep on it, and double check
                    # the environment after a good nap.
                    _random_sleep()
                    # Tease out why this error occurred. The man page for
                    # rename reads:
                    #   "The link named by tmppath does not exist; or, a
                    #    directory component in data_file does not exist;
                    #    or, tmppath or data_file is an empty string."
                    assert len(self._tmppath) > 0 and len(df._data_file) > 0
                    tpstats = do_stat(self._tmppath)
                    tfstats = do_fstat(self._fd)
                    assert tfstats
                    if not tpstats or tfstats.st_ino != tpstats.st_ino:
                        # Temporary file name conflict
                        raise DiskFileError(
                            'DiskFile.put(): temporary file, %s, was'
                            ' already renamed (targeted for %s)' % (
                                self._tmppath, df._data_file))
                    else:
                        # Data file target name now has a bad path!
                        dfstats = do_stat(df._put_datadir)
                        if not dfstats:
                            raise DiskFileError(
                                'DiskFile.put(): path to object, %s, no'
                                ' longer exists (targeted for %s)' % (
                                    df._put_datadir, df._data_file))
                        else:
                            is_dir = stat.S_ISDIR(dfstats.st_mode)
                            if not is_dir:
                                raise DiskFileError(
                                    'DiskFile.put(): path to object, %s,'
                                    ' no longer a directory (targeted for'
                                    ' %s)' % (self._put_datadir,
                                              df._data_file))
                            else:
                                # Let's retry since everything looks okay
                                logging.warn(
                                    "DiskFile.put(): rename('%s','%s')"
                                    " initially failed (%s) but a"
                                    " stat('%s') following that succeeded:"
                                    " %r" % (
                                        self._tmppath, df._data_file, str(err),
                                        df._put_datadir, dfstats))
                                attempts += 1
                                continue
                else:
                    raise SwiftOnFileSystemOSError(
                        err.errno, "%s, rename('%s', '%s')" % (
                            err.strerror, self._tmppath, df._data_file))
            else:
                # Success!
                break

        # Close here so the calling context does not have to perform this
        # in a thread.
        self.close()
Esempio n. 10
0
 def _clear_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, obj),
                          metadata)
Esempio n. 11
0
 def _set_dir_object(self, obj):
     metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
     metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
     utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
                          metadata)