def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except SwiftOnFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
else:
stats = do_fstat(fd)
if not stats:
return
self._is_dir = stat.S_ISDIR(stats.st_mode)
obj_size = stats.st_size
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._fd = fd
self._obj_size = obj_size
return self
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
else:
stats = do_fstat(fd)
if not stats:
return
self._is_dir = stat.S_ISDIR(stats.st_mode)
obj_size = stats.st_size
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._fd = fd
self._obj_size = obj_size
return self
def test_read_metadata_err(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
_xattr_get_err[xkey] = errno.EOPNOTSUPP
try:
utils.read_metadata(path)
except IOError as e:
assert e.errno == errno.EOPNOTSUPP
assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
else:
self.fail("Expected an IOError exception on get")
def _keep_sys_metadata(self, metadata):
"""
Make sure system metadata is not lost when writing new user metadata
This method will read the existing metadata and check for system
metadata. If there are any, it should be appended to the metadata obj
the user is trying to write.
"""
# If metadata has been previously fetched, use that.
# Stale metadata (outdated size/etag) would've been updated when
# metadata is fetched for the first time.
orig_metadata = self._metadata or read_metadata(self._data_file)
sys_keys = [X_CONTENT_TYPE, X_ETAG, 'name', X_CONTENT_LENGTH,
X_OBJECT_TYPE, X_TYPE]
for key in sys_keys:
if key in orig_metadata:
metadata[key] = orig_metadata[key]
if X_OBJECT_TYPE not in orig_metadata:
if metadata[X_CONTENT_TYPE].lower() == DIR_TYPE:
metadata[X_OBJECT_TYPE] = DIR_OBJECT
else:
metadata[X_OBJECT_TYPE] = FILE
if X_TYPE not in orig_metadata:
metadata[X_TYPE] = OBJECT
return metadata
def _keep_sys_metadata(self, metadata):
"""
Make sure system metadata is not lost when writing new user metadata
This method will read the existing metadata and check for system
metadata. If there are any, it should be appended to the metadata obj
the user is trying to write.
"""
# If metadata has been previously fetched, use that.
# Stale metadata (outdated size/etag) would've been updated when
# metadata is fetched for the first time.
orig_metadata = self._metadata or read_metadata(self._data_file)
sys_keys = [
X_CONTENT_TYPE, X_ETAG, 'name', X_CONTENT_LENGTH, X_OBJECT_TYPE,
X_TYPE
]
for key in sys_keys:
if key in orig_metadata:
metadata[key] = orig_metadata[key]
if X_OBJECT_TYPE not in orig_metadata:
if metadata[X_CONTENT_TYPE].lower() == DIR_TYPE:
metadata[X_OBJECT_TYPE] = DIR_OBJECT
else:
metadata[X_OBJECT_TYPE] = FILE
if X_TYPE not in orig_metadata:
metadata[X_TYPE] = OBJECT
return metadata
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
try:
metadata = read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno != errno.ENOENT:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._threadpool.run_in_thread(self._unlinkold)
self._metadata = None
self._data_file = None
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
try:
metadata = read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno != errno.ENOENT:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._threadpool.run_in_thread(self._unlinkold)
self._metadata = None
self._data_file = None
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d,
res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata_multiple(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
for i in range(0, 3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert not expected_p
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
for i in range(0, 2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys())
def read_metadata(self):
"""
Return the metadata for an object without requiring the caller to open
the object first.
This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
metadata = disk_file.read_metadata()
The operations performed here is very similar to those made in open().
This is to avoid opening and closing of file (two syscalls over
network). IOW, this optimization addresses the case where the fd
returned by open() isn't going to be used i.e the file is not read (GET
or metadata recalculation)
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
try:
self._metadata = read_metadata(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
raise DiskFileNotExist
raise err
if self._metadata and self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
try:
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
raise DiskFileNotExist
raise err
if not validate_object(self._metadata, self._stat):
# Metadata is stale/invalid. So open the object for reading
# to update Etag and other metadata.
with self.open():
return self.get_metadata()
else:
# Metadata is valid. Don't have to open the file.
self._filter_metadata()
return self._metadata
def read_metadata(self):
"""
Return the metadata for an object without requiring the caller to open
the object first.
This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
metadata = disk_file.read_metadata()
The operations performed here is very similar to those made in open().
This is to avoid opening and closing of file (two syscalls over
network). IOW, this optimization addresses the case where the fd
returned by open() isn't going to be used i.e the file is not read (GET
or metadata recalculation)
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
try:
self._metadata = read_metadata(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
raise DiskFileNotExist
raise err
if self._metadata and self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
try:
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
raise DiskFileNotExist
raise err
if not validate_object(self._metadata, self._stat):
# Metadata is stale/invalid. So open the object for reading
# to update Etag and other metadata.
with self.open():
return self.get_metadata()
else:
# Metadata is valid. Don't have to open the file.
self._filter_metadata()
return self._metadata
def _clear_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
utils.write_metadata(os.path.join(self.rootdir, obj),
metadata)
def _set_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
metadata)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except SwiftOnFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
self._stat = do_fstat(self._fd)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
obj_size = self._stat.st_size
self._metadata = read_metadata(self._fd)
if not validate_object(self._metadata, self._stat):
self._metadata = create_object_metadata(self._fd, self._stat,
self._metadata)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(self._fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._obj_size = obj_size
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and err.errno == errno.ENOENT:
# Handle races: ENOENT can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
return self
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except SwiftOnFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
self._stat = do_fstat(self._fd)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
obj_size = self._stat.st_size
self._metadata = read_metadata(self._fd)
if not validate_object(self._metadata, self._stat):
create_object_metadata(self._fd)
self._metadata = read_metadata(self._fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(self._fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._obj_size = obj_size
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and err.errno == errno.ENOENT:
# Handle races: ENOENT can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
return self
def test_read_metadata_notfound(self):
path = "/tmp/foo/r"
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt