def test_do_fstat_err(self): try: fs.do_fstat(1000) except GlusterFileSystemOSError: pass else: self.fail("Expected GlusterFileSystemOSError")
def get_object_metadata(obj_path_or_fd): """ Return metadata of object. """ if isinstance(obj_path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. stats = do_fstat(obj_path_or_fd) else: # We are given a path to the object when the DiskDir.list_objects_iter # method invokes us. stats = do_stat(obj_path_or_fd) if not stats: metadata = {} else: is_dir = stat.S_ISDIR(stats.st_mode) metadata = { X_TYPE: OBJECT, X_TIMESTAMP: normalize_timestamp(stats.st_ctime), X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path_or_fd)} return metadata
def get_object_metadata(obj_path_or_fd): """ Return metadata of object. """ if isinstance(obj_path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. stats = do_fstat(obj_path_or_fd) else: # We are given a path to the object when the DiskDir.list_objects_iter # method invokes us. stats = do_stat(obj_path_or_fd) if not stats: metadata = {} else: is_dir = stat.S_ISDIR(stats.st_mode) metadata = { X_TYPE: OBJECT, X_TIMESTAMP: normalize_timestamp(stats.st_ctime), X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path_or_fd) } return metadata
def test_do_fstat(self): tmpdir = mkdtemp() try: fd, tmpfile = mkstemp(dir=tmpdir) buf1 = os.stat(tmpfile) buf2 = fs.do_fstat(fd) assert buf1 == buf2 finally: os.close(fd) os.remove(tmpfile) os.rmdir(tmpdir)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def _finalize_put(self, metadata): # Write out metadata before fsync() to ensure it is also forced to # disk. write_metadata(self._fd, metadata) # We call fsync() before calling drop_cache() to lower the # amount of redundant work the drop cache code will perform on # the pages (now that after fsync the pages will be all # clean). do_fsync(self._fd) # From the Department of the Redundancy Department, make sure # we call drop_cache() after fsync() to avoid redundant work # (pages all clean). do_fadvise64(self._fd, self._last_sync, self._upload_size) # At this point we know that the object's full directory path # exists, so we can just rename it directly without using Swift's # swift.common.utils.renamer(), which makes the directory path and # adds extra stat() calls. df = self._disk_file attempts = 1 while True: try: do_rename(self._tmppath, df._data_file) except OSError as err: if err.errno in (errno.ENOENT, errno.EIO) \ and attempts < MAX_RENAME_ATTEMPTS: # FIXME: Why either of these two error conditions is # happening is unknown at this point. This might be a # FUSE issue of some sort or a possible race # condition. So let's sleep on it, and double check # the environment after a good nap. _random_sleep() # Tease out why this error occurred. The man page for # rename reads: # "The link named by tmppath does not exist; or, a # directory component in data_file does not exist; # or, tmppath or data_file is an empty string." assert len(self._tmppath) > 0 and len(df._data_file) > 0 tpstats = do_stat(self._tmppath) tfstats = do_fstat(self._fd) assert tfstats if not tpstats or tfstats.st_ino != tpstats.st_ino: # Temporary file name conflict raise DiskFileError( 'DiskFile.put(): temporary file, %s, was' ' already renamed (targeted for %s)' % ( self._tmppath, df._data_file)) else: # Data file target name now has a bad path! dfstats = do_stat(df._put_datadir) if not dfstats: raise DiskFileError( 'DiskFile.put(): path to object, %s, no' ' longer exists (targeted for %s)' % ( df._put_datadir, df._data_file)) else: is_dir = stat.S_ISDIR(dfstats.st_mode) if not is_dir: raise DiskFileError( 'DiskFile.put(): path to object, %s,' ' no longer a directory (targeted for' ' %s)' % (self._put_datadir, df._data_file)) else: # Let's retry since everything looks okay logging.warn( "DiskFile.put(): os.rename('%s','%s')" " initially failed (%s) but a" " stat('%s') following that succeeded:" " %r" % ( self._tmppath, df._data_file, str(err), df._put_datadir, dfstats)) attempts += 1 continue else: raise GlusterFileSystemOSError( err.errno, "%s, os.rename('%s', '%s')" % ( err.strerror, self._tmppath, df._data_file)) else: # Success! break # Close here so the calling context does not have to perform this # in a thread. self.close()
def put(self, fd, metadata, extension='.data'): """ Finalize writing the file on disk, and renames it from the temp file to the real location. This should be called after the data has been written to the temp file. :param fd: file descriptor of the temp file :param metadata: dictionary of metadata to be written :param extension: extension to be used when making the file """ # Our caller will use '.data' here; we just ignore it since we map the # URL directly to the file system. metadata = _adjust_metadata(metadata) if dir_is_object(metadata): if not self.data_file: # Does not exist, create it data_file = os.path.join(self._obj_path, self._obj) _, self.metadata = self._create_dir_object(data_file, metadata) self.data_file = os.path.join(self._container_path, data_file) elif not self.is_dir: # Exists, but as a file raise DiskFileError('DiskFile.put(): directory creation failed' ' since the target, %s, already exists as' ' a file' % self.data_file) return if self._is_dir: # A pre-existing directory already exists on the file # system, perhaps gratuitously created when another # object was created, or created externally to Swift # REST API servicing (UFO use case). raise DiskFileError('DiskFile.put(): file creation failed since' ' the target, %s, already exists as a' ' directory' % self.data_file) # Write out metadata before fsync() to ensure it is also forced to # disk. write_metadata(fd, metadata) if not _relaxed_writes: do_fsync(fd) if X_CONTENT_LENGTH in metadata: # Don't bother doing this before fsync in case the OS gets any # ideas to issue partial writes. fsize = int(metadata[X_CONTENT_LENGTH]) self.drop_cache(fd, 0, fsize) # At this point we know that the object's full directory path exists, # so we can just rename it directly without using Swift's # swift.common.utils.renamer(), which makes the directory path and # adds extra stat() calls. data_file = os.path.join(self.put_datadir, self._obj) while True: try: os.rename(self.tmppath, data_file) except OSError as err: if err.errno in (errno.ENOENT, errno.EIO): # FIXME: Why either of these two error conditions is # happening is unknown at this point. This might be a FUSE # issue of some sort or a possible race condition. So # let's sleep on it, and double check the environment # after a good nap. _random_sleep() # Tease out why this error occurred. The man page for # rename reads: # "The link named by tmppath does not exist; or, a # directory component in data_file does not exist; # or, tmppath or data_file is an empty string." assert len(self.tmppath) > 0 and len(data_file) > 0 tpstats = do_stat(self.tmppath) tfstats = do_fstat(fd) assert tfstats if not tpstats or tfstats.st_ino != tpstats.st_ino: # Temporary file name conflict raise DiskFileError('DiskFile.put(): temporary file,' ' %s, was already renamed' ' (targeted for %s)' % ( self.tmppath, data_file)) else: # Data file target name now has a bad path! dfstats = do_stat(self.put_datadir) if not dfstats: raise DiskFileError('DiskFile.put(): path to' ' object, %s, no longer exists' ' (targeted for %s)' % ( self.put_datadir, data_file)) else: is_dir = stat.S_ISDIR(dfstats.st_mode) if not is_dir: raise DiskFileError('DiskFile.put(): path to' ' object, %s, no longer a' ' directory (targeted for' ' %s)' % (self.put_datadir, data_file)) else: # Let's retry since everything looks okay logging.warn("DiskFile.put(): os.rename('%s'," "'%s') initially failed (%s) but" " a stat('%s') following that" " succeeded: %r" % ( self.tmppath, data_file, str(err), self.put_datadir, dfstats)) continue else: raise GlusterFileSystemOSError( err.errno, "%s, os.rename('%s', '%s')" % ( err.strerror, self.tmppath, data_file)) else: # Success! break # Avoid the unlink() system call as part of the mkstemp context cleanup self.tmppath = None self.metadata = metadata self.filter_metadata() # Mark that it actually exists now self.data_file = os.path.join(self.datadir, self._obj)
def _finalize_put(self, metadata): # Write out metadata before fsync() to ensure it is also forced to # disk. write_metadata(self._fd, metadata) # We call fsync() before calling drop_cache() to lower the # amount of redundant work the drop cache code will perform on # the pages (now that after fsync the pages will be all # clean). do_fsync(self._fd) # From the Department of the Redundancy Department, make sure # we call drop_cache() after fsync() to avoid redundant work # (pages all clean). do_fadvise64(self._fd, self._last_sync, self._upload_size) # At this point we know that the object's full directory path # exists, so we can just rename it directly without using Swift's # swift.common.utils.renamer(), which makes the directory path and # adds extra stat() calls. df = self._disk_file attempts = 1 while True: try: do_rename(self._tmppath, df._data_file) except OSError as err: if err.errno in (errno.ENOENT, errno.EIO) \ and attempts < MAX_RENAME_ATTEMPTS: # FIXME: Why either of these two error conditions is # happening is unknown at this point. This might be a # FUSE issue of some sort or a possible race # condition. So let's sleep on it, and double check # the environment after a good nap. _random_sleep() # Tease out why this error occurred. The man page for # rename reads: # "The link named by tmppath does not exist; or, a # directory component in data_file does not exist; # or, tmppath or data_file is an empty string." assert len(self._tmppath) > 0 and len(df._data_file) > 0 tpstats = do_stat(self._tmppath) tfstats = do_fstat(self._fd) assert tfstats if not tpstats or tfstats.st_ino != tpstats.st_ino: # Temporary file name conflict raise DiskFileError( 'DiskFile.put(): temporary file, %s, was' ' already renamed (targeted for %s)' % (self._tmppath, df._data_file)) else: # Data file target name now has a bad path! dfstats = do_stat(df._put_datadir) if not dfstats: raise DiskFileError( 'DiskFile.put(): path to object, %s, no' ' longer exists (targeted for %s)' % (df._put_datadir, df._data_file)) else: is_dir = stat.S_ISDIR(dfstats.st_mode) if not is_dir: raise DiskFileError( 'DiskFile.put(): path to object, %s,' ' no longer a directory (targeted for' ' %s)' % (self._put_datadir, df._data_file)) else: # Let's retry since everything looks okay logging.warn( "DiskFile.put(): rename('%s','%s')" " initially failed (%s) but a" " stat('%s') following that succeeded:" " %r" % (self._tmppath, df._data_file, str(err), df._put_datadir, dfstats)) attempts += 1 continue else: raise GlusterFileSystemOSError( err.errno, "%s, rename('%s', '%s')" % (err.strerror, self._tmppath, df._data_file)) else: # Success! break # Close here so the calling context does not have to perform this # in a thread. self.close()
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: if not self._stat: self._stat = do_fstat(self._fd) obj_size = self._stat.st_size if not self._metadata: self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): self._metadata = create_object_metadata(self._fd, self._stat, self._metadata) assert self._metadata is not None self._filter_metadata() if stat.S_ISDIR(self._stat.st_mode): do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and \ err.errno in (errno.ENOENT, errno.ESTALE): # Handle races: ENOENT/ESTALE can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT/ESTALE. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise self._disk_file_open = True return self