def test_do_stat_eio_twice(self): count = [0] _os_stat = os.stat def mock_os_stat_eio(path): count[0] += 1 if count[0] <= 2: raise OSError(errno.EIO, os.strerror(errno.EIO)) return _os_stat(path) with patch('os.stat', mock_os_stat_eio): fs.do_stat('/tmp') is not None
def test_do_stat_eio_ten(self): def mock_os_stat_eio(path): raise OSError(errno.EIO, os.strerror(errno.EIO)) try: with patch('os.stat', mock_os_stat_eio): fs.do_stat('/tmp') except SwiftOnFileSystemOSError: pass else: self.fail("SwiftOnFileSystemOSError expected")
def test_do_stat_err(self): def mock_os_stat_eacces(path): raise OSError(errno.EACCES, os.strerror(errno.EACCES)) try: with patch('os.stat', mock_os_stat_eacces): fs.do_stat('/tmp') except SwiftOnFileSystemOSError: pass else: self.fail("SwiftOnFileSystemOSError expected")
def read_metadata(self): """ Return the metadata for an object without opening the object's file on disk. :returns: metadata dictionary for an object :raises DiskFileError: this implementation will raise the same errors as the `open()` method. """ # FIXME: pull a lot of this and the copy of it from open() out to # another function # Do not actually open the file, in order to duck hpssfs checksum # validation and resulting timeouts # This means we do a few things DiskFile.open() does. try: if not self._stat: self._stat = do_stat(self._data_file) self._is_dir = stat.S_ISDIR(self._stat.st_mode) self._metadata = read_metadata(self._data_file) except SwiftOnFileSystemOSError: raise DiskFileNotExist if not self._validate_object_metadata(): self._create_object_metadata(self._data_file) self._filter_metadata() return self._metadata
def get_object_metadata(obj_path_or_fd, stats=None): """ Return metadata of object. """ if not stats: if isinstance(obj_path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. stats = do_fstat(obj_path_or_fd) else: # We are given a path to the object when the # DiskDir.list_objects_iter method invokes us. stats = do_stat(obj_path_or_fd) if not stats: metadata = {} else: is_dir = stat.S_ISDIR(stats.st_mode) metadata = { X_TYPE: OBJECT, X_TIMESTAMP: normalize_timestamp(stats.st_ctime), X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, X_MTIME: 0 if is_dir else normalize_timestamp(stats.st_mtime), X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path_or_fd)} return metadata
def test_do_stat(self): tmpdir = mkdtemp() try: fd, tmpfile = mkstemp(dir=tmpdir) buf1 = os.stat(tmpfile) buf2 = fs.do_stat(tmpfile) assert buf1 == buf2 finally: os.close(fd) os.remove(tmpfile) os.rmdir(tmpdir)
def _validate_object_metadata(self): # Has no Swift specific metadata saved as xattr. Probably because # object was added/replaced through filesystem interface. if not self._metadata and not self._is_dir: self._file_has_changed = True return False required_keys = \ (X_TIMESTAMP, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_ETAG, # SOF specific keys X_TYPE, X_OBJECT_TYPE) if not all(k in self._metadata for k in required_keys): # At least one of the required keys does not exist return False if not self._is_dir: # X_MTIME is a new key added recently, newer objects will # have the key set during PUT. if X_MTIME in self._metadata: # Check if the file has been modified through filesystem # interface by comparing mtime stored in xattr during PUT # and current mtime of file. obj_stat = do_stat(self._data_file) if normalize_timestamp(self._metadata[X_MTIME]) != \ normalize_timestamp(obj_stat.hpss_st_mtime): self._file_has_changed = True return False else: # Without X_MTIME key, comparing md5sum is the only way # to determine if file has changed or not. This is inefficient # but there's no other way! self._etag = get_etag(self._data_file) if self._etag != self._metadata[X_ETAG]: self._file_has_changed = True return False else: # Checksums are same; File has not changed. For the next # GET request on same file, we don't compute md5sum again! # This is achieved by setting X_MTIME to mtime in # _create_object_metadata() return False if self._metadata[X_TYPE] == OBJECT: return True return False
def get_object_metadata(obj_path, stats=None): """ Return metadata of object. """ if not stats: stats = do_stat(obj_path) if not stats: metadata = {} else: is_dir = stat.S_ISDIR(stats.st_mode) metadata = { X_TYPE: OBJECT, X_TIMESTAMP: normalize_timestamp(stats.hpss_st_ctime), X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE, X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE, X_CONTENT_LENGTH: 0 if is_dir else stats.st_size, X_MTIME: 0 if is_dir else normalize_timestamp(stats.hpss_st_mtime), X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path)} return metadata
def make_directory(full_path, uid, gid, metadata=None): """ Make a directory and change the owner ship as specified, and potentially creating the object metadata if requested. """ try: do_mkdir(full_path) except OSError as err: if err.errno == errno.ENOENT: # Tell the caller some directory of the parent path does not # exist. return False, metadata elif err.errno == errno.EEXIST: # Possible race, in that the caller invoked this method when it # had previously determined the file did not exist. # # FIXME: When we are confident, remove this stat() call as it is # not necessary. try: stats = do_stat(full_path) except SwiftOnFileSystemOSError as serr: # FIXME: Ideally we'd want to return an appropriate error # message and code in the PUT Object REST API response. raise DiskFileError("make_directory: mkdir failed" " because path %s already exists, and" " a subsequent stat on that same" " path failed (%s)" % (full_path, str(serr))) else: is_dir = stat.S_ISDIR(stats.st_mode) if not is_dir: # FIXME: Ideally we'd want to return an appropriate error # message and code in the PUT Object REST API response. raise AlreadyExistsAsFile("make_directory:" " mkdir failed on path %s" " because it already exists" " but not as a directory" % (full_path)) return True, metadata elif err.errno == errno.ENOTDIR: # FIXME: Ideally we'd want to return an appropriate error # message and code in the PUT Object REST API response. raise AlreadyExistsAsFile("make_directory:" " mkdir failed because some " "part of path %s is not in fact" " a directory" % (full_path)) elif err.errno == errno.EIO: # Sometimes Fuse will return an EIO error when it does not know # how to handle an unexpected, but transient situation. It is # possible the directory now exists, stat() it to find out after a # short period of time. _random_sleep() try: stats = do_stat(full_path) except SwiftOnFileSystemOSError as serr: if serr.errno == errno.ENOENT: errmsg = "make_directory: mkdir failed on" \ " path %s (EIO), and a subsequent stat on" \ " that same path did not find the file." % ( full_path,) else: errmsg = "make_directory: mkdir failed on" \ " path %s (%s), and a subsequent stat on" \ " that same path failed as well (%s)" % ( full_path, str(err), str(serr)) raise DiskFileError(errmsg) else: if not stats: errmsg = "make_directory: mkdir failed on" \ " path %s (EIO), and a subsequent stat on" \ " that same path did not find the file." % ( full_path,) raise DiskFileError(errmsg) else: # The directory at least exists now is_dir = stat.S_ISDIR(stats.st_mode) if is_dir: # Dump the stats to the log with the original exception logging.warn("make_directory: mkdir initially" " failed on path %s (%s) but a stat()" " following that succeeded: %r" % (full_path, str(err), stats)) # Assume another entity took care of the proper setup. return True, metadata else: raise DiskFileError("make_directory: mkdir" " initially failed on path %s (%s)" " but now we see that it exists" " but is not a directory (%r)" % (full_path, str(err), stats)) else: # Some other potentially rare exception occurred that does not # currently warrant a special log entry to help diagnose. raise DiskFileError("make_directory: mkdir failed on" " path %s (%s)" % (full_path, str(err))) else: if metadata: # We were asked to set the initial metadata for this object. metadata_orig = get_object_metadata(full_path) metadata_orig.update(metadata) write_metadata(full_path, metadata_orig) metadata = metadata_orig # We created it, so we are reponsible for always setting the proper # ownership. do_chown(full_path, uid, gid) return True, metadata
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._logger.debug("DiskFile: Opening %s" % self._data_file) self._stat = do_stat(self._data_file) self._is_dir = stat.S_ISDIR(self._stat.st_mode) if not self._is_dir: self._fd = do_open(self._data_file, hpss.O_RDONLY) self._obj_size = self._stat.st_size else: self._fd = -1 self._obj_size = 0 except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._logger.debug("DiskFile: Reading metadata") self._metadata = read_metadata(self._data_file) if not self._validate_object_metadata(): self._create_object_metadata(self._data_file) assert self._metadata is not None self._filter_metadata() if not self._is_dir: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self
def _finalize_put(self, metadata): # Write out metadata before fsync() to ensure it is also forced to # disk. write_metadata(self._tmppath, metadata) do_fsync(self._fd) self.set_checksum(metadata['ETag']) # At this point we know that the object's full directory path # exists, so we can just rename it directly without using Swift's # swift.common.utils.renamer(), which makes the directory path and # adds extra stat() calls. df = self._disk_file attempts = 1 while True: try: do_rename(self._tmppath, df._data_file) except OSError as err: if err.errno in (errno.ENOENT, errno.EIO) \ and attempts < MAX_RENAME_ATTEMPTS: # FIXME: Why either of these two error conditions is # happening is unknown at this point. This might be a # FUSE issue of some sort or a possible race # condition. So let's sleep on it, and double check # the environment after a good nap. _random_sleep() # Tease out why this error occurred. The man page for # rename reads: # "The link named by tmppath does not exist; or, a # directory component in data_file does not exist; # or, tmppath or data_file is an empty string." assert len(self._tmppath) > 0 and len(df._data_file) > 0 tpstats = do_stat(self._tmppath) tfstats = do_fstat(self._fd) assert tfstats if not tpstats or tfstats.st_ino != tpstats.st_ino: # Temporary file name conflict raise DiskFileError( 'DiskFile.put(): temporary file, %s, was' ' already renamed (targeted for %s)' % ( self._tmppath, df._data_file)) else: # Data file target name now has a bad path! dfstats = do_stat(df._put_datadir) if not dfstats: raise DiskFileError( 'DiskFile.put(): path to object, %s, no' ' longer exists (targeted for %s)' % ( df._put_datadir, df._data_file)) else: is_dir = stat.S_ISDIR(dfstats.st_mode) if not is_dir: raise DiskFileError( 'DiskFile.put(): path to object, %s,' ' no longer a directory (targeted for' ' %s)' % (self._put_datadir, df._data_file)) else: # Let's retry since everything looks okay logging.warn( "DiskFile.put(): rename('%s','%s')" " initially failed (%s) but a" " stat('%s') following that succeeded:" " %r" % ( self._tmppath, df._data_file, str(err), df._put_datadir, dfstats)) attempts += 1 continue else: raise SwiftOnFileSystemOSError( err.errno, "%s, rename('%s', '%s')" % ( err.strerror, self._tmppath, df._data_file)) else: # Success! break # Close here so the calling context does not have to perform this # in a thread. self.close()
def test_do_stat_enoent(self): res = fs.do_stat(os.path.join('/tmp', str(random.random()))) assert res is None