def test_read_metadata_err(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
_xattr_get_err[xkey] = errno.EOPNOTSUPP
try:
utils.read_metadata(path)
except IOError as e:
assert e.errno == errno.EOPNOTSUPP
assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
else:
self.fail("Expected an IOError exception on get")
def read_metadata(self):
"""
Return the metadata for an object without opening the object's file on
disk.
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
# FIXME: pull a lot of this and the copy of it from open() out to
# another function
# Do not actually open the file, in order to duck hpssfs checksum
# validation and resulting timeouts
# This means we do a few things DiskFile.open() does.
try:
if not self._stat:
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
self._metadata = read_metadata(self._data_file)
except SwiftOnFileSystemOSError:
raise DiskFileNotExist
if not self._validate_object_metadata():
self._create_object_metadata(self._data_file)
self._filter_metadata()
return self._metadata
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
try:
metadata = read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno != errno.ENOENT:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._unlinkold()
self._metadata = None
self._data_file = None
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert not expected_p
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 4, "%r" % _xattr_op_cnt
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._logger.debug("DiskFile: Opening %s" % self._data_file)
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
if not self._is_dir:
self._fd = do_open(self._data_file, hpss.O_RDONLY)
self._obj_size = self._stat.st_size
else:
self._fd = -1
self._obj_size = 0
except SwiftOnFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
self._logger.debug("DiskFile: Reading metadata")
self._metadata = read_metadata(self._data_file)
if not self._validate_object_metadata():
self._create_object_metadata(self._data_file)
assert self._metadata is not None
self._filter_metadata()
if not self._is_dir:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and err.errno == errno.ENOENT:
# Handle races: ENOENT can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
return self
def _clear_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
utils.write_metadata(os.path.join(self.rootdir, obj),
metadata)
def _set_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
metadata)
def test_read_metadata_notfound(self):
path = "/tmp/foo/r"
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt