def test_do_fstat_err(self): try: fs.do_fstat(1000) except GlusterFileSystemOSError: pass else: self.fail("Expected GlusterFileSystemOSError")
def get_object_metadata(obj_path_or_fd): """ Return metadata of object. """ if isinstance(obj_path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. stats = do_fstat(obj_path_or_fd) else: # We are given a path to the object when the DiskDir.list_objects_iter # method invokes us. stats = do_stat(obj_path_or_fd) if not stats: metadata = {} else: is_dir = stat.S_ISDIR(stats.st_mode) metadata = { X_TYPE: OBJECT, X_TIMESTAMP: normalize_timestamp(stats.st_ctime), X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path_or_fd)} return metadata
def _adjust_metadata(fd, metadata): # Fix up the metadata to ensure it has a proper value for the # Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE # metadata values. content_type = metadata.get(X_CONTENT_TYPE, '') if not content_type: # FIXME: How can this be that our caller supplied us with metadata # that has a content type that evaluates to False? # # FIXME: If the file exists, we would already know it is a # directory. So why are we assuming it is a file object? metadata[X_CONTENT_TYPE] = FILE_TYPE metadata[X_OBJECT_TYPE] = FILE else: if content_type.lower() == DIR_TYPE: metadata[X_OBJECT_TYPE] = DIR_OBJECT else: metadata[X_OBJECT_TYPE] = FILE # stat.st_mtime does not change after last write(). We set this to later # detect if the object was changed from filesystem interface (non Swift) statinfo = do_fstat(fd) if stat.S_ISREG(statinfo.st_mode): metadata[X_MTIME] = normalize_timestamp(statinfo.st_mtime) metadata[X_TYPE] = OBJECT return metadata
def get_object_metadata(obj_path_or_fd, stats=None): """ Return metadata of object. """ if not stats: if isinstance(obj_path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. stats = do_fstat(obj_path_or_fd) else: # We are given a path to the object when the # DiskDir.list_objects_iter method invokes us. stats = do_stat(obj_path_or_fd) if not stats: metadata = {} else: is_dir = stat.S_ISDIR(stats.st_mode) metadata = { X_TYPE: OBJECT, X_TIMESTAMP: normalize_timestamp(stats.st_ctime), X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, X_MTIME: 0 if is_dir else normalize_timestamp(stats.st_mtime), X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path_or_fd) } return metadata
def test_do_fstat(self): tmpdir = mkdtemp() try: fd, tmpfile = mkstemp(dir=tmpdir) buf1 = os.stat(tmpfile) buf2 = fs.do_fstat(fd) assert buf1 == buf2 finally: os.close(fd) os.remove(tmpfile) os.rmdir(tmpdir)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def _finalize_put(self, metadata): # Write out metadata before fsync() to ensure it is also forced to # disk. write_metadata(self._fd, metadata) # We call fsync() before calling drop_cache() to lower the # amount of redundant work the drop cache code will perform on # the pages (now that after fsync the pages will be all # clean). do_fsync(self._fd) # From the Department of the Redundancy Department, make sure # we call drop_cache() after fsync() to avoid redundant work # (pages all clean). do_fadvise64(self._fd, self._last_sync, self._upload_size) # At this point we know that the object's full directory path # exists, so we can just rename it directly without using Swift's # swift.common.utils.renamer(), which makes the directory path and # adds extra stat() calls. df = self._disk_file attempts = 1 while True: try: do_rename(self._tmppath, df._data_file) except OSError as err: if err.errno in (errno.ENOENT, errno.EIO) \ and attempts < MAX_RENAME_ATTEMPTS: # FIXME: Why either of these two error conditions is # happening is unknown at this point. This might be a # FUSE issue of some sort or a possible race # condition. So let's sleep on it, and double check # the environment after a good nap. _random_sleep() # Tease out why this error occurred. The man page for # rename reads: # "The link named by tmppath does not exist; or, a # directory component in data_file does not exist; # or, tmppath or data_file is an empty string." assert len(self._tmppath) > 0 and len(df._data_file) > 0 tpstats = do_stat(self._tmppath) tfstats = do_fstat(self._fd) assert tfstats if not tpstats or tfstats.st_ino != tpstats.st_ino: # Temporary file name conflict raise DiskFileError( 'DiskFile.put(): temporary file, %s, was' ' already renamed (targeted for %s)' % ( self._tmppath, df._data_file)) else: # Data file target name now has a bad path! dfstats = do_stat(df._put_datadir) if not dfstats: raise DiskFileError( 'DiskFile.put(): path to object, %s, no' ' longer exists (targeted for %s)' % ( df._put_datadir, df._data_file)) else: is_dir = stat.S_ISDIR(dfstats.st_mode) if not is_dir: raise DiskFileError( 'DiskFile.put(): path to object, %s,' ' no longer a directory (targeted for' ' %s)' % (self._put_datadir, df._data_file)) else: # Let's retry since everything looks okay logging.warn( "DiskFile.put(): rename('%s','%s')" " initially failed (%s) but a" " stat('%s') following that succeeded:" " %r" % ( self._tmppath, df._data_file, str(err), df._put_datadir, dfstats)) attempts += 1 continue else: raise SwiftOnFileSystemOSError( err.errno, "%s, rename('%s', '%s')" % ( err.strerror, self._tmppath, df._data_file)) else: # Success! break # Close here so the calling context does not have to perform this # in a thread. self.close()
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._stat = do_fstat(self._fd) self._is_dir = stat.S_ISDIR(self._stat.st_mode) obj_size = self._stat.st_size self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): create_object_metadata(self._fd) self._metadata = read_metadata(self._fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self
def _finalize_put(self, metadata): # Write out metadata before fsync() to ensure it is also forced to # disk. write_metadata(self._fd, metadata) # We call fsync() before calling drop_cache() to lower the # amount of redundant work the drop cache code will perform on # the pages (now that after fsync the pages will be all # clean). do_fsync(self._fd) # From the Department of the Redundancy Department, make sure # we call drop_cache() after fsync() to avoid redundant work # (pages all clean). do_fadvise64(self._fd, self._last_sync, self._upload_size) # At this point we know that the object's full directory path # exists, so we can just rename it directly without using Swift's # swift.common.utils.renamer(), which makes the directory path and # adds extra stat() calls. df = self._disk_file attempts = 1 while True: try: do_rename(self._tmppath, df._data_file) except OSError as err: if err.errno in (errno.ENOENT, errno.EIO) \ and attempts < MAX_RENAME_ATTEMPTS: # FIXME: Why either of these two error conditions is # happening is unknown at this point. This might be a # FUSE issue of some sort or a possible race # condition. So let's sleep on it, and double check # the environment after a good nap. _random_sleep() # Tease out why this error occurred. The man page for # rename reads: # "The link named by tmppath does not exist; or, a # directory component in data_file does not exist; # or, tmppath or data_file is an empty string." assert len(self._tmppath) > 0 and len(df._data_file) > 0 tpstats = do_stat(self._tmppath) tfstats = do_fstat(self._fd) assert tfstats if not tpstats or tfstats.st_ino != tpstats.st_ino: # Temporary file name conflict raise DiskFileError( 'DiskFile.put(): temporary file, %s, was' ' already renamed (targeted for %s)' % (self._tmppath, df._data_file)) else: # Data file target name now has a bad path! dfstats = do_stat(df._put_datadir) if not dfstats: raise DiskFileError( 'DiskFile.put(): path to object, %s, no' ' longer exists (targeted for %s)' % (df._put_datadir, df._data_file)) else: is_dir = stat.S_ISDIR(dfstats.st_mode) if not is_dir: raise DiskFileError( 'DiskFile.put(): path to object, %s,' ' no longer a directory (targeted for' ' %s)' % (self._put_datadir, df._data_file)) else: # Let's retry since everything looks okay logging.warn( "DiskFile.put(): rename('%s','%s')" " initially failed (%s) but a" " stat('%s') following that succeeded:" " %r" % (self._tmppath, df._data_file, str(err), df._put_datadir, dfstats)) attempts += 1 continue else: raise SwiftOnFileSystemOSError( err.errno, "%s, rename('%s', '%s')" % (err.strerror, self._tmppath, df._data_file)) else: # Success! break # Close here so the calling context does not have to perform this # in a thread. self.close()
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._stat = do_fstat(self._fd) self._is_dir = stat.S_ISDIR(self._stat.st_mode) obj_size = self._stat.st_size self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): self._metadata = create_object_metadata(self._fd, self._stat, self._metadata) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self