def test_read_metadata_err(self): path = "/tmp/foo/r" expected_d = {'a': 'y'} xkey = _xkey(path, utils.METADATA_KEY) _xattrs[xkey] = serialize_metadata(expected_d) _xattr_get_err[xkey] = errno.EOPNOTSUPP try: utils.read_metadata(path) except IOError as e: assert e.errno == errno.EOPNOTSUPP assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt else: self.fail("Expected an IOError exception on get")
def read_metadata(self): """ Return the metadata for an object without opening the object's file on disk. :returns: metadata dictionary for an object :raises DiskFileError: this implementation will raise the same errors as the `open()` method. """ # FIXME: pull a lot of this and the copy of it from open() out to # another function # Do not actually open the file, in order to duck hpssfs checksum # validation and resulting timeouts # This means we do a few things DiskFile.open() does. try: if not self._stat: self._stat = do_stat(self._data_file) self._is_dir = stat.S_ISDIR(self._stat.st_mode) self._metadata = read_metadata(self._data_file) except SwiftOnFileSystemOSError: raise DiskFileNotExist if not self._validate_object_metadata(): self._create_object_metadata(self._data_file) self._filter_metadata() return self._metadata
def delete(self, timestamp): """ Delete the object. This implementation creates a tombstone file using the given timestamp, and removes any older versions of the object file. Any file that has an older timestamp than timestamp will be deleted. .. note:: An implementation is free to use or ignore the timestamp parameter. :param timestamp: timestamp to compare with each file :raises DiskFileError: this implementation will raise the same errors as the `create()` method. """ try: metadata = read_metadata(self._data_file) except (IOError, OSError) as err: if err.errno != errno.ENOENT: raise else: if metadata[X_TIMESTAMP] >= timestamp: return self._unlinkold() self._metadata = None self._data_file = None
def _unlinkold(self): if self._is_dir: # Marker, or object, directory. # # Delete from the filesystem only if it contains no objects. # If it does contain objects, then just remove the object # metadata tag which will make this directory a # fake-filesystem-only directory and will be deleted when the # container or parent directory is deleted. # # FIXME: Ideally we should use an atomic metadata update operation metadata = read_metadata(self._data_file) if dir_is_object(metadata): metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT write_metadata(self._data_file, metadata) rmobjdir(self._data_file) else: # Delete file object do_unlink(self._data_file) # Garbage collection of non-object directories. Now that we # deleted the file, determine if the current directory and any # parent directory may be deleted. dirname = os.path.dirname(self._data_file) while dirname and dirname != self._container_path: # Try to remove any directories that are not objects. if not rmobjdir(dirname): # If a directory with objects has been found, we can stop # garabe collection break else: dirname = os.path.dirname(dirname)
def test_read_metadata(self): path = "/tmp/foo/r" expected_d = {'a': 'y'} xkey = _xkey(path, utils.METADATA_KEY) _xattrs[xkey] = serialize_metadata(expected_d) res_d = utils.read_metadata(path) assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self): path = "/tmp/foo/r" expected_d = {'a': 'y' * 150000} expected_p = serialize_metadata(expected_d) for i in range(0, 2): xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] expected_p = expected_p[utils.MAX_XATTR_SIZE:] assert len(expected_p) <= utils.MAX_XATTR_SIZE res_d = utils.read_metadata(path) assert res_d == {} assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple(self): path = "/tmp/foo/r" expected_d = {'a': 'y' * 150000} expected_p = serialize_metadata(expected_d) for i in range(0, 3): xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] expected_p = expected_p[utils.MAX_XATTR_SIZE:] assert not expected_p res_d = utils.read_metadata(path) assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) assert _xattr_op_cnt['get'] == 4, "%r" % _xattr_op_cnt
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._logger.debug("DiskFile: Opening %s" % self._data_file) self._stat = do_stat(self._data_file) self._is_dir = stat.S_ISDIR(self._stat.st_mode) if not self._is_dir: self._fd = do_open(self._data_file, hpss.O_RDONLY) self._obj_size = self._stat.st_size else: self._fd = -1 self._obj_size = 0 except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._logger.debug("DiskFile: Reading metadata") self._metadata = read_metadata(self._data_file) if not self._validate_object_metadata(): self._create_object_metadata(self._data_file) assert self._metadata is not None self._filter_metadata() if not self._is_dir: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self
def _clear_dir_object(self, obj): metadata = utils.read_metadata(os.path.join(self.rootdir, obj)) metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT utils.write_metadata(os.path.join(self.rootdir, obj), metadata)
def _set_dir_object(self, obj): metadata = utils.read_metadata(os.path.join(self.rootdir, obj)) metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]), metadata)
def test_read_metadata_notfound(self): path = "/tmp/foo/r" res_d = utils.read_metadata(path) assert res_d == {} assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt