def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False, disk_chunk_size=65536,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.disk_chunk_size = disk_chunk_size
device = account
#Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
#Absolute path for obj directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
if not os.path.exists(self.datadir + '/' + self.obj):
return
self.data_file = os.path.join(self.datadir, self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not self.metadata:
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not validate_object(self.metadata):
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' +
self.obj)
self.filter_metadata()
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
else:
stats = do_fstat(fd)
if not stats:
return
self._is_dir = stat.S_ISDIR(stats.st_mode)
obj_size = stats.st_size
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._fd = fd
self._obj_size = obj_size
return self
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
else:
stats = do_fstat(fd)
if not stats:
return
self._is_dir = stat.S_ISDIR(stats.st_mode)
obj_size = stats.st_size
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._fd = fd
self._obj_size = obj_size
return self
def test_read_metadata_err(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
_xattr_get_err[xkey] = errno.EOPNOTSUPP
try:
utils.read_metadata(path)
except IOError as e:
assert e.errno == errno.EOPNOTSUPP
assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
else:
self.fail("Expected an IOError exception on get")
def test_read_metadata_err(self):
path = "/tmp/foo/r"
expected_d = {"a": "y"}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
_xattr_get_err[xkey] = errno.EOPNOTSUPP
try:
utils.read_metadata(path)
except IOError as e:
assert e.errno == errno.EOPNOTSUPP
assert _xattr_op_cnt["get"] == 1, "%r" % _xattr_op_cnt
else:
self.fail("Expected an IOError exception on get")
def test_read_metadata_err(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
_xattr_get_err[xkey] = errno.EOPNOTSUPP
try:
utils.read_metadata(path)
except IOError as e:
assert e.errno == errno.EOPNOTSUPP
assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
else:
self.fail("Expected an IOError exception on get")
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def list_objects_iter(self):
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
# list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
# list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
try:
metadata = read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno != errno.ENOENT:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._threadpool.run_in_thread(self._unlinkold)
self._metadata = None
self._data_file = None
def _keep_sys_metadata(self, metadata):
"""
Make sure system metadata is not lost when writing new user metadata
This method will read the existing metadata and check for system
metadata. If there are any, it should be appended to the metadata obj
the user is trying to write.
"""
# If metadata has been previously fetched, use that.
# Stale metadata (outdated size/etag) would've been updated when
# metadata is fetched for the first time.
orig_metadata = self._metadata or read_metadata(self._data_file)
sys_keys = [X_CONTENT_TYPE, X_ETAG, 'name', X_CONTENT_LENGTH,
X_OBJECT_TYPE, X_TYPE]
for key in sys_keys:
if key in orig_metadata:
metadata[key] = orig_metadata[key]
if X_OBJECT_TYPE not in orig_metadata:
if metadata[X_CONTENT_TYPE].lower() == DIR_TYPE:
metadata[X_OBJECT_TYPE] = DIR_OBJECT
else:
metadata[X_OBJECT_TYPE] = FILE
if X_TYPE not in orig_metadata:
metadata[X_TYPE] = OBJECT
return metadata
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
try:
metadata = read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno != errno.ENOENT:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._threadpool.run_in_thread(self._unlinkold)
self._metadata = None
self._data_file = None
def _keep_sys_metadata(self, metadata):
"""
Make sure system metadata is not lost when writing new user metadata
This method will read the existing metadata and check for system
metadata. If there are any, it should be appended to the metadata obj
the user is trying to write.
"""
# If metadata has been previously fetched, use that.
# Stale metadata (outdated size/etag) would've been updated when
# metadata is fetched for the first time.
orig_metadata = self._metadata or read_metadata(self._data_file)
sys_keys = [X_CONTENT_TYPE, X_ETAG, 'name', X_CONTENT_LENGTH,
X_OBJECT_TYPE, X_TYPE]
for key in sys_keys:
if key in orig_metadata and key not in metadata:
metadata[key] = orig_metadata[key]
if X_OBJECT_TYPE not in orig_metadata:
if metadata[X_CONTENT_TYPE].lower() == DIR_TYPE:
metadata[X_OBJECT_TYPE] = DIR_OBJECT
else:
metadata[X_OBJECT_TYPE] = FILE
if X_TYPE not in orig_metadata:
metadata[X_TYPE] = OBJECT
return metadata
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = { 'a': 'y' }
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y'}
xkey = _xkey(path, utils.METADATA_KEY)
_xattrs[xkey] = serialize_metadata(expected_d)
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d,
res_d)
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple(self):
path = "/tmp/foo/r"
expected_d = {"a": "y" * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ""))
_xattrs[xkey] = expected_p[: utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE :]
assert not expected_p
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt["get"] == 4, "%r" % _xattr_op_cnt
def test_read_metadata_multiple(self):
path = "/tmp/foo/r"
expected_d = { 'a': 'y' * 150000 }
expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
for i in range(0,3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert not expected_p
res_d = utils.read_metadata(path)
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = {'a': 'y' * 150000}
expected_p = serialize_metadata(expected_d)
for i in range(0, 2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
# list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
# list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def test_read_metadata_multiple_one_missing(self):
path = "/tmp/foo/r"
expected_d = { 'a': 'y' * 150000 }
expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
for i in range(0,2):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
_xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
expected_p = expected_p[utils.MAX_XATTR_SIZE:]
assert len(expected_p) <= utils.MAX_XATTR_SIZE
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys())
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def unlinkold(self, timestamp):
"""
Remove any older versions of the object file. Any file that has an
older timestamp than timestamp will be deleted.
:param timestamp: timestamp to compare with each file
"""
if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp:
return
assert self.data_file, \
"Have metadata, %r, but no data_file" % self.metadata
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains
# no objects. If it does contain objects, then just
# remove the object metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted
# when the container or parent directory is deleted.
metadata = read_metadata(self.data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self.data_file, metadata)
rmobjdir(self.data_file)
else:
# Delete file object
do_unlink(self.data_file)
# Garbage collection of non-object directories.
# Now that we deleted the file, determine
# if the current directory and any parent
# directory may be deleted.
dirname = os.path.dirname(self.data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not
# objects.
if not rmobjdir(dirname):
# If a directory with objects has been
# found, we can stop garabe collection
break
else:
dirname = os.path.dirname(dirname)
self.metadata = {}
self.data_file = None
def read_metadata(self):
"""
Return the metadata for an object without requiring the caller to open
the object first.
This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
metadata = disk_file.read_metadata()
The operations performed here is very similar to those made in open().
This is to avoid opening and closing of file (two syscalls over
network). IOW, this optimization addresses the case where the fd
returned by open() isn't going to be used i.e the file is not read (GET
or metadata recalculation)
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
try:
self._metadata = read_metadata(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if self._metadata and self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
try:
self._stat = do_stat(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if not validate_object(self._metadata, self._stat):
# Metadata is stale/invalid. So open the object for reading
# to update Etag and other metadata.
with self.open():
return self._metadata
else:
# Metadata is valid. Don't have to open the file.
self._filter_metadata()
return self._metadata
def read_metadata(self):
"""
Return the metadata for an object without requiring the caller to open
the object first.
This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
metadata = disk_file.read_metadata()
The operations performed here is very similar to those made in open().
This is to avoid opening and closing of file (two syscalls over
network). IOW, this optimization addresses the case where the fd
returned by open() isn't going to be used i.e the file is not read (GET
or metadata recalculation)
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
try:
self._metadata = read_metadata(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if self._metadata and self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
try:
self._stat = do_stat(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if not validate_object(self._metadata, self._stat):
# Metadata is stale/invalid. So open the object for reading
# to update Etag and other metadata.
with self.open():
return self._metadata
else:
# Metadata is valid. Don't have to open the file.
self._filter_metadata()
return self._metadata
def _read_metadata(dd):
""" Filter read metadata so that it always returns a tuple that includes
some kind of timestamp. With 1.4.8 of the Swift integration the
timestamps were not stored. Here we fabricate timestamps for volumes
where the existing data has no timestamp (that is, stored data is not
a tuple), allowing us a measure of backward compatibility.
FIXME: At this time it does not appear that the timestamps on each
metadata are used for much, so this should not hurt anything.
"""
metadata_i = read_metadata(dd)
metadata = {}
timestamp = 0
for key, value in metadata_i.iteritems():
if not isinstance(value, tuple):
value = (value, timestamp)
metadata[key] = value
return metadata
def _read_metadata(dd):
""" Filter read metadata so that it always returns a tuple that includes
some kind of timestamp. With 1.4.8 of the Swift integration the
timestamps were not stored. Here we fabricate timestamps for volumes
where the existing data has no timestamp (that is, stored data is not
a tuple), allowing us a measure of backward compatibility.
FIXME: At this time it does not appear that the timestamps on each
metadata are used for much, so this should not hurt anything.
"""
metadata_i = read_metadata(dd)
metadata = {}
timestamp = 0
for key, value in metadata_i.iteritems():
if not isinstance(value, tuple):
value = (value, timestamp)
metadata[key] = value
return metadata
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
if self._disk_file_does_not_exist:
# It was determined during disk_file.read_metadata() call that
# the file does not exist. Don't proceed with deletion on a
# non-existing path.
return
try:
metadata = self._metadata or read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno in (errno.ESTALE, errno.ENOENT):
return
else:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._threadpool.run_in_thread(self._unlinkold)
self._metadata = None
self._data_file = None
def delete(self, timestamp):
"""
Delete the object.
This implementation creates a tombstone file using the given
timestamp, and removes any older versions of the object file. Any
file that has an older timestamp than timestamp will be deleted.
.. note::
An implementation is free to use or ignore the timestamp
parameter.
:param timestamp: timestamp to compare with each file
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
if self._disk_file_does_not_exist:
# It was determined during disk_file.read_metadata() call that
# the file does not exist. Don't proceed with deletion on a
# non-existing path.
return
try:
metadata = self._metadata or read_metadata(self._data_file)
except (IOError, OSError) as err:
if err.errno in (errno.ESTALE, errno.ENOENT):
return
else:
raise
else:
if metadata[X_TIMESTAMP] >= timestamp:
return
self._threadpool.run_in_thread(self._unlinkold)
self._metadata = None
self._data_file = None
def list_objects_iter(self,
limit,
marker,
end_marker,
prefix,
delimiter,
path=None):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
assert limit >= 0
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
if path is not None:
if path:
prefix = path = path.rstrip('/') + '/'
else:
prefix = path
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
container_list = []
objects = self._update_object_count()
if objects:
objects.sort()
else:
return container_list
if end_marker:
objects = filter_end_marker(objects, end_marker)
if marker and marker >= prefix:
objects = filter_marker(objects, marker)
elif prefix:
objects = filter_prefix_as_marker(objects, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
objects = filter_prefix(objects, prefix)
else:
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
count = 0
for obj in objects:
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
if delimiter == '/' and obj_path[-1] == delimiter:
clean_obj_path = obj_path[:-1]
else:
clean_obj_path = obj_path
try:
metadata = create_object_metadata(clean_obj_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno != errno.ENOENT:
raise
if not Glusterfs._implicit_dir_objects and metadata \
and metadata[X_CONTENT_TYPE] == DIR_TYPE \
and not dir_is_object(metadata):
continue
list_item = []
list_item.append(obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
count += 1
if count >= limit:
break
return container_list
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path=None,
storage_policy_index=0,
out_content_type=None, reverse=False):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
assert limit >= 0
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
if path is not None:
if path:
prefix = path = path.rstrip('/') + '/'
else:
prefix = path
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
container_list = []
if self.account == 'gsexpiring':
objects = list_objects_gsexpiring_container(self.datadir)
else:
objects = self._update_object_count()
if objects:
objects.sort()
else:
# No objects in container , return empty list
return container_list
if marker and end_marker and reverse:
marker, end_marker = end_marker, marker
if end_marker:
objects = filter_end_marker(objects, end_marker)
if marker and marker >= prefix:
objects = filter_marker(objects, marker)
elif prefix:
objects = filter_prefix_as_marker(objects, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
objects = filter_prefix(objects, prefix)
else:
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
if out_content_type == 'text/plain' or \
self.account == 'gsexpiring':
# When out_content_type == 'text/plain':
#
# The client is only asking for a plain list of objects and NOT
# asking for any extended information about objects such as
# bytes used or etag.
#
# When self.account == 'gsexpiring':
#
# This is a JSON request sent by the object expirer to list
# tracker objects in a container in gsexpiring volume.
# When out_content_type is 'application/json', the caller
# expects each record entry to have the following ordered
# fields: (name, timestamp, size, content_type, etag)
for obj in objects:
container_list.append((obj, '0', 0, 'text/plain', ''))
if len(container_list) >= limit:
break
if reverse:
container_list.reverse()
return container_list
count = 0
for obj in objects:
obj_path = os.path.join(self.datadir, obj)
try:
metadata = read_metadata(obj_path)
except GlusterFileSystemIOError as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
# obj might have been deleted by another process
# since the objects list was originally built
continue
else:
raise err
if not metadata or not validate_object(metadata):
if delimiter == '/' and obj_path[-1] == delimiter:
clean_obj_path = obj_path[:-1]
else:
clean_obj_path = obj_path
try:
metadata = create_object_metadata(clean_obj_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
if not Glusterfs._implicit_dir_objects and metadata \
and metadata[X_CONTENT_TYPE] == DIR_TYPE \
and not dir_is_object(metadata):
continue
list_item = []
list_item.append(obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
count += 1
if count >= limit:
break
if reverse:
container_list.reverse()
return container_list
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.iter_hook = iter_hook
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
self._obj_path, self._obj = os.path.split(obj)
else:
self._obj_path = ''
self._obj = obj
if self._obj_path:
self.name = os.path.join(container, self._obj_path)
else:
self.name = container
# Absolute path for object directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self._container_path = os.path.join(path, device, container)
if _use_put_mount:
self.put_datadir = os.path.join(self.device_path + '_PUT',
self.name)
else:
self.put_datadir = self.datadir
self._is_dir = False
self.tmppath = None
self.logger = logger
self.metadata = {}
self.meta_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
self.suppress_file_closing = False
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.put_datadir, self._obj)
try:
stats = do_stat(data_file)
except OSError as err:
if err.errno == errno.ENOTDIR:
return
else:
if not stats:
return
self.data_file = data_file
self._is_dir = stat.S_ISDIR(stats.st_mode)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.filter_metadata()
if not self._is_dir and keep_data_fp:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.iter_hook = iter_hook
# Don't support obj_name ending/begining with '/', like /a, a/, /a/b/,
# etc.
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
self._obj_path, self._obj = os.path.split(obj)
else:
self._obj_path = ''
self._obj = obj
if self._obj_path:
self.name = os.path.join(container, self._obj_path)
else:
self.name = container
# Absolute path for object directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self._container_path = os.path.join(path, device, container)
self._is_dir = False
self.tmppath = None
self.logger = logger
self.metadata = {}
self.meta_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.datadir, self._obj)
if not os.path.exists(data_file):
return
self.data_file = os.path.join(data_file)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.filter_metadata()
if os.path.isdir(data_file):
self._is_dir = True
else:
if keep_data_fp:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
def test_read_metadata_notfound(self):
path = "/tmp/foo/r"
res_d = utils.read_metadata(path)
assert res_d == {}
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
def _set_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
metadata)
def get_container_timestamp(self, container):
cont_path = os.path.join(self.datadir, container)
metadata = read_metadata(cont_path)
return int(metadata.get(X_PUT_TIMESTAMP, ('0', 0))[0]) or None
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path=None):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
assert limit >= 0
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
if path is not None:
if path:
prefix = path = path.rstrip('/') + '/'
else:
prefix = path
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
container_list = []
objects = self._update_object_count()
if objects:
objects.sort()
else:
return container_list
if end_marker:
objects = filter_end_marker(objects, end_marker)
if marker and marker >= prefix:
objects = filter_marker(objects, marker)
elif prefix:
objects = filter_prefix_as_marker(objects, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
objects = filter_prefix(objects, prefix)
else:
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
count = 0
for obj in objects:
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
if delimiter == '/' and obj_path[-1] == delimiter:
clean_obj_path = obj_path[:-1]
else:
clean_obj_path = obj_path
try:
metadata = create_object_metadata(clean_obj_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno != errno.ENOENT:
raise
if Glusterfs.OBJECT_ONLY and metadata \
and metadata[X_CONTENT_TYPE] == DIR_TYPE \
and not dir_is_object(metadata):
continue
list_item = []
list_item.append(obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
count += 1
if count >= limit:
break
return container_list
def get_container_timestamp(self, container):
cont_path = os.path.join(self.datadir, container)
metadata = read_metadata(cont_path)
return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None
def _clear_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
utils.write_metadata(os.path.join(self.rootdir, obj), metadata)
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
uid=DEFAULT_UID,
gid=DEFAULT_GID):
self.disk_chunk_size = disk_chunk_size
device = account
#Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
#Absolute path for obj directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
if not os.path.exists(self.datadir + '/' + self.obj):
return
self.data_file = os.path.join(self.datadir, self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not self.metadata:
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not validate_object(self.metadata):
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
self.filter_metadata()
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
def _clear_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
utils.write_metadata(os.path.join(self.rootdir, obj),
metadata)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
if not self._stat:
self._stat = do_fstat(self._fd)
obj_size = self._stat.st_size
if not self._metadata:
self._metadata = read_metadata(self._fd)
if not validate_object(self._metadata, self._stat):
self._metadata = create_object_metadata(self._fd, self._stat,
self._metadata)
assert self._metadata is not None
self._filter_metadata()
if stat.S_ISDIR(self._stat.st_mode):
do_close(self._fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._obj_size = obj_size
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and \
err.errno in (errno.ENOENT, errno.ESTALE):
# Handle races: ENOENT/ESTALE can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT/ESTALE. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
self._disk_file_open = True
return self
def _set_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
metadata)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
if not self._stat:
self._stat = do_fstat(self._fd)
obj_size = self._stat.st_size
if not self._metadata:
self._metadata = read_metadata(self._fd)
if not validate_object(self._metadata, self._stat):
self._metadata = create_object_metadata(self._fd, self._stat,
self._metadata)
assert self._metadata is not None
self._filter_metadata()
if stat.S_ISDIR(self._stat.st_mode):
do_close(self._fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._obj_size = obj_size
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and \
err.errno in (errno.ENOENT, errno.ESTALE):
# Handle races: ENOENT/ESTALE can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT/ESTALE. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
self._disk_file_open = True
return self