def get_data_file_size(self):
"""
Returns the os_path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata, or if
self.data_file does not exist.
:returns: file size as an int
:raises DiskFileError: on file size mismatch.
:raises DiskFileNotExist: on file not existing (including deleted)
"""
#Marker directory.
if self._is_dir:
return 0
try:
file_size = 0
if self.data_file:
file_size = os_path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
self.metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self.metadata)
return file_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def update_metadata(self, metadata):
assert self.metadata, "Valid container/account metadata should have been created by now"
if metadata:
new_metadata = self.metadata.copy()
if metadata.has_key('X-Account-Meta-Bytes-Add'):
content_length = metadata.get('X-Account-Meta-Bytes-Add')[0]
bused, timestamp = new_metadata[X_BYTES_USED]
new_metadata[X_BYTES_USED] = (int(str(bused)) + int(content_length), timestamp)
elif metadata.has_key('X-Account-Meta-Bytes-Del'):
content_length = metadata.get('X-Account-Meta-Bytes-Del')[0]
bused, timestamp = new_metadata[X_BYTES_USED]
new_metadata[X_BYTES_USED] = (int(str(bused)) - int(content_length), timestamp)
else:
new_metadata.update(metadata)
del_keys = ['X-Account-Meta-Bytes-Del','X-Account-Meta-Bytes-Add']
for dkey in del_keys:
if new_metadata.has_key(dkey):
new_metadata.pop(dkey)
if new_metadata != self.metadata:
write_metadata(self.datadir, new_metadata)
self.metadata = new_metadata
def get_data_file_size(self):
"""
Returns the os.path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata. Or if
self.data_file does not exist.
:returns: file size as an int
:raises DiskFileError: on file size mismatch.
:raises DiskFileNotExist: on file not existing (including deleted)
"""
#Marker directory.
if self._is_dir:
return 0
try:
file_size = 0
if self.data_file:
file_size = os.path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
self.metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self.metadata)
return file_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')
def _unlinkold(self):
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains no objects.
# If it does contain objects, then just remove the object
# metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted when the
# container or parent directory is deleted.
#
# FIXME: Ideally we should use an atomic metadata update operation
metadata = read_metadata(self._data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self._data_file, metadata)
rmobjdir(self._data_file)
else:
# Delete file object
do_unlink(self._data_file)
# Garbage collection of non-object directories. Now that we
# deleted the file, determine if the current directory and any
# parent directory may be deleted.
dirname = os.path.dirname(self._data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not objects.
if not rmobjdir(dirname):
# If a directory with objects has been found, we can stop
# garabe collection
break
else:
dirname = os.path.dirname(dirname)
def update_metadata(self, metadata):
assert self.metadata, "Valid container/account metadata should have been created by now"
if metadata:
new_metadata = self.metadata.copy()
new_metadata.update(metadata)
if new_metadata != self.metadata:
write_metadata(self.datadir, new_metadata)
self.metadata = new_metadata
def _update_container_count(self):
containers, container_count = get_account_details(self.datadir)
if X_CONTAINER_COUNT not in self.metadata or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
write_metadata(self.datadir, self.metadata)
return containers
def update_metadata(self, metadata):
assert self.metadata, "Valid container/account metadata should have been created by now"
if metadata:
new_metadata = self.metadata.copy()
new_metadata.update(metadata)
if new_metadata != self.metadata:
write_metadata(self.datadir, new_metadata)
self.metadata = new_metadata
def test_write_metadata(self):
path = "/tmp/foo/w"
orig_d = {"bar": "foo"}
utils.write_metadata(path, orig_d)
xkey = _xkey(path, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert orig_d == deserialize_metadata(_xattrs[xkey])
assert _xattr_op_cnt["set"] == 1
def initialize(self, timestamp):
"""
Create and write metatdata to directory/account.
:param metadata: Metadata to write.
"""
metadata = get_account_metadata(self.datadir)
metadata[X_TIMESTAMP] = timestamp
write_metadata(self.datadir, metadata)
self.metadata = metadata
def _update_container_count(self):
containers, container_count = get_account_details(self.datadir)
if X_CONTAINER_COUNT not in self.metadata \
or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
write_metadata(self.datadir, self.metadata)
return containers
def update_container_count(self):
if not self.container_info:
self.container_info = get_account_details(self.datadir)
containers, container_count = self.container_info
if X_CONTAINER_COUNT not in self.metadata or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
write_metadata(self.datadir, self.metadata)
def test_write_metadata(self):
path = "/tmp/foo/w"
orig_d = { 'bar' : 'foo' }
utils.write_metadata(path, orig_d)
xkey = _xkey(path, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert orig_d == pickle.loads(_xattrs[xkey])
assert _xattr_op_cnt['set'] == 1
def _old_getsize():
file_size = os_path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
# FIXME - bit rot detection?
self.metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self.metadata)
return file_size
def update_put_timestamp(self, timestamp):
# Since accounts always exists at this point, just update the account
# PUT timestamp if this given timestamp is later than what we already
# know.
assert self._dir_exists
if timestamp > self.metadata[X_PUT_TIMESTAMP][0]:
self.metadata[X_PUT_TIMESTAMP] = (timestamp, 0)
write_metadata(self.datadir, self.metadata)
def initialize(self, timestamp):
"""
Create and write metatdata to directory/account.
:param metadata: Metadata to write.
"""
metadata = get_account_metadata(self.datadir)
metadata[X_TIMESTAMP] = (timestamp, 0)
write_metadata(self.datadir, metadata)
self.metadata = metadata
def update_put_timestamp(self, timestamp):
# Since accounts always exists at this point, just update the account
# PUT timestamp if this given timestamp is later than what we already
# know.
assert self._dir_exists
if timestamp > self.metadata[X_PUT_TIMESTAMP][0]:
self.metadata[X_PUT_TIMESTAMP] = (timestamp, 0)
write_metadata(self.datadir, self.metadata)
def update_container_count(self):
if not self.container_info:
self.container_info = get_account_details(self.datadir)
containers, container_count = self.container_info
if X_CONTAINER_COUNT not in self.metadata \
or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
write_metadata(self.datadir, self.metadata)
def _update_object_count(self):
objects, object_count, bytes_used = get_container_details(self.datadir)
if X_OBJECTS_COUNT not in self.metadata \
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \
or X_BYTES_USED not in self.metadata \
or int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
return objects
def put(self, metadata):
"""
Create and write metatdata to directory/container.
:param metadata: Metadata to write.
"""
if not self.dir_exists:
mkdirs(self.datadir)
os.chown(self.datadir, self.uid, self.gid)
write_metadata(self.datadir, metadata)
self.metadata = metadata
self.dir_exists = True
def put(self, metadata):
"""
Create and write metatdata to directory/container.
:param metadata: Metadata to write.
"""
if not self.dir_exists:
mkdirs(self.datadir)
os.chown(self.datadir, self.uid, self.gid)
write_metadata(self.datadir, metadata)
self.metadata = metadata
self.dir_exists = True
def _update_object_count(self):
objects, object_count, bytes_used = get_container_details(self.datadir)
if X_OBJECTS_COUNT not in self.metadata \
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \
or X_BYTES_USED not in self.metadata \
or int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
return objects
def test_write_metadata_err(self):
path = "/tmp/foo/w"
orig_d = { 'bar' : 'foo' }
xkey = _xkey(path, utils.METADATA_KEY)
_xattr_err[xkey] = errno.EOPNOTSUPP
try:
utils.write_metadata(path, orig_d)
except IOError as e:
assert e.errno == errno.EOPNOTSUPP
assert len(_xattrs.keys()) == 0
assert _xattr_op_cnt['set'] == 1
else:
self.fail("Expected an IOError exception on write")
def initialize(self, timestamp):
"""
Create and write metatdata to directory/container.
:param metadata: Metadata to write.
"""
if not self._dir_exists:
mkdirs(self.datadir)
# If we create it, ensure we own it.
do_chown(self.datadir, self.uid, self.gid)
metadata = get_container_metadata(self.datadir)
metadata[X_TIMESTAMP] = timestamp
write_metadata(self.datadir, metadata)
self.metadata = metadata
self._dir_exists = True
def initialize(self, timestamp):
"""
Create and write metatdata to directory/container.
:param metadata: Metadata to write.
"""
if not self._dir_exists:
mkdirs(self.datadir)
# If we create it, ensure we own it.
do_chown(self.datadir, self.uid, self.gid)
metadata = get_container_metadata(self.datadir)
metadata[X_TIMESTAMP] = timestamp
write_metadata(self.datadir, metadata)
self.metadata = metadata
self._dir_exists = True
def put(self, fd, metadata, extension='.data'):
"""
Finalize writing the file on disk, and renames it from the temp file to
the real location. This should be called after the data has been
written to the temp file.
:param fd: file descriptor of the temp file
:param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
extension = ''
metadata = _adjust_metadata(metadata)
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
if not self.data_file:
self.data_file = os.path.join(self.datadir, self._obj)
self._create_dir_object(self.data_file)
self.put_metadata(metadata)
return
# Check if directory already exists.
if self._is_dir:
# FIXME: How can we have a directory and it not be marked as a
# MARKER_DIR (see above)?
msg = 'File object exists as a directory: %s' % self.data_file
raise AlreadyExistsAsDir(msg)
timestamp = normalize_timestamp(metadata[X_TIMESTAMP])
write_metadata(self.tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
tpool.execute(os.fsync, fd)
if self._obj_path:
dir_objs = self._obj_path.split('/')
assert len(dir_objs) >= 1
tmp_path = self._container_path
for dir_name in dir_objs:
tmp_path = os.path.join(tmp_path, dir_name)
self._create_dir_object(tmp_path)
newpath = os.path.join(self.datadir, self._obj)
renamer(self.tmppath, newpath)
do_chown(newpath, self.uid, self.gid)
self.metadata = metadata
self.data_file = newpath
self.filter_metadata()
return
def test_write_metadata_multiple(self):
# At 64 KB an xattr key/value pair, this should generate three keys.
path = "/tmp/foo/w"
orig_d = { 'bar' : 'x' * 150000 }
utils.write_metadata(path, orig_d)
assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys())
payload = ''
for i in range(0,3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
assert xkey in _xattrs
assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE
payload += _xattrs[xkey]
assert orig_d == pickle.loads(payload)
assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt
def put_metadata(self, metadata, tombstone=False):
"""
Short hand for putting metadata to .meta and .ts files.
:param metadata: dictionary of metadata to be written
:param tombstone: whether or not we are writing a tombstone
"""
if tombstone:
# We don't write tombstone files. So do nothing.
return
assert self.data_file is not None, "put_metadata: no file to put metadata into"
metadata = _adjust_metadata(metadata)
write_metadata(self.data_file, metadata)
self.metadata = metadata
self.filter_metadata()
def update_object_count(self):
if not self.object_info:
self.object_info = get_container_details(self.datadir)
objects, object_count, bytes_used = self.object_info
if (
X_OBJECTS_COUNT not in self.metadata
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count
or X_BYTES_USED not in self.metadata
or int(self.metadata[X_BYTES_USED][0]) != bytes_used
):
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
def unlinkold(self, timestamp):
"""
Remove any older versions of the object file. Any file that has an
older timestamp than timestamp will be deleted.
:param timestamp: timestamp to compare with each file
"""
if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp:
return
assert self.data_file, \
"Have metadata, %r, but no data_file" % self.metadata
if self._is_dir:
# Marker, or object, directory.
#
# Delete from the filesystem only if it contains
# no objects. If it does contain objects, then just
# remove the object metadata tag which will make this directory a
# fake-filesystem-only directory and will be deleted
# when the container or parent directory is deleted.
metadata = read_metadata(self.data_file)
if dir_is_object(metadata):
metadata[X_OBJECT_TYPE] = DIR_NON_OBJECT
write_metadata(self.data_file, metadata)
rmobjdir(self.data_file)
else:
# Delete file object
do_unlink(self.data_file)
# Garbage collection of non-object directories.
# Now that we deleted the file, determine
# if the current directory and any parent
# directory may be deleted.
dirname = os.path.dirname(self.data_file)
while dirname and dirname != self._container_path:
# Try to remove any directories that are not
# objects.
if not rmobjdir(dirname):
# If a directory with objects has been
# found, we can stop garabe collection
break
else:
dirname = os.path.dirname(dirname)
self.metadata = {}
self.data_file = None
def update_put_timestamp(self, timestamp):
"""
Update the PUT timestamp for the container.
If the container does not exist, create it using a PUT timestamp of
the given value.
If the container does exist, update the PUT timestamp only if it is
later than the existing value.
"""
if not os_path.exists(self.datadir):
self.initialize(timestamp)
else:
if timestamp > self.metadata[X_PUT_TIMESTAMP]:
self.metadata[X_PUT_TIMESTAMP] = (timestamp, 0)
write_metadata(self.datadir, self.metadata)
def update_put_timestamp(self, timestamp):
"""
Update the PUT timestamp for the container.
If the container does not exist, create it using a PUT timestamp of
the given value.
If the container does exist, update the PUT timestamp only if it is
later than the existing value.
"""
if not do_exists(self.datadir):
self.initialize(timestamp)
else:
if timestamp > self.metadata[X_PUT_TIMESTAMP]:
self.metadata[X_PUT_TIMESTAMP] = (timestamp, 0)
write_metadata(self.datadir, self.metadata)
def put(self, fd, tmppath, metadata,extension=''):
if extension == '.ts':
# TombStone marker (deleted)
return True
metadata[X_TYPE] = OBJECT
if extension == '.meta':
# Metadata recorded separately from the file
self.put_metadata(metadata)
return True
# Check if directory already exists.
if self.is_dir:
self.logger.error('Directory already exists %s/%s' % \
(self.datadir , self.obj))
return False
write_metadata(tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
tpool.execute(os.fsync, fd)
if self.obj_path:
dir_objs = self.obj_path.split('/')
tmp_path = ''
if len(dir_objs):
for dir_name in dir_objs:
if tmp_path:
tmp_path = tmp_path + '/' + dir_name
else:
tmp_path = dir_name
if not self.create_dir_object(os.path.join(self.container_path,
tmp_path)):
self.logger.error("Failed in subdir %s",\
os.path.join(self.container_path,tmp_path))
return False
renamer(tmppath, os.path.join(self.datadir,
self.obj))
do_chown(os.path.join(self.datadir, self.obj), self.uid, self.gid)
self.metadata = metadata
return True
def test_write_metadata_space_err(self):
def _mock_xattr_setattr(item, name, value):
raise IOError(errno.ENOSPC, os.strerror(errno.ENOSPC))
with patch('xattr.setxattr', _mock_xattr_setattr):
path = "/tmp/foo/w"
orig_d = {'bar': 'foo'}
try:
utils.write_metadata(path, orig_d)
except DiskFileNoSpace:
pass
else:
self.fail("Expected DiskFileNoSpace exception")
fd = 0
try:
utils.write_metadata(fd, orig_d)
except DiskFileNoSpace:
pass
else:
self.fail("Expected DiskFileNoSpace exception")
def test_write_metadata_space_err(self):
def _mock_xattr_setattr(item, name, value):
raise IOError(errno.ENOSPC, os.strerror(errno.ENOSPC))
with patch("xattr.setxattr", _mock_xattr_setattr):
path = "/tmp/foo/w"
orig_d = {"bar": "foo"}
try:
utils.write_metadata(path, orig_d)
except DiskFileNoSpace:
pass
else:
self.fail("Expected DiskFileNoSpace exception")
fd = 0
try:
utils.write_metadata(fd, orig_d)
except DiskFileNoSpace:
pass
else:
self.fail("Expected DiskFileNoSpace exception")
def put_metadata(self, metadata):
"""
Write metadata to directory/container.
"""
write_metadata(self.datadir, metadata)
self.metadata = metadata
def put(self, fd, metadata, extension='.data'):
"""
Finalize writing the file on disk, and renames it from the temp file
to the real location. This should be called after the data has been
written to the temp file.
:param fd: file descriptor of the temp file
:param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
metadata = _adjust_metadata(metadata)
if dir_is_object(metadata):
if not self.data_file:
# Does not exist, create it
data_file = os.path.join(self._obj_path, self._obj)
_, self.metadata = self._create_dir_object(data_file, metadata)
self.data_file = os.path.join(self._container_path, data_file)
elif not self.is_dir:
# Exists, but as a file
raise DiskFileError('DiskFile.put(): directory creation failed'
' since the target, %s, already exists as'
' a file' % self.data_file)
return
if self._is_dir:
# A pre-existing directory already exists on the file
# system, perhaps gratuitously created when another
# object was created, or created externally to Swift
# REST API servicing (UFO use case).
raise DiskFileError('DiskFile.put(): file creation failed since'
' the target, %s, already exists as a'
' directory' % self.data_file)
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(fd, metadata)
if not _relaxed_writes:
do_fsync(fd)
if X_CONTENT_LENGTH in metadata:
# Don't bother doing this before fsync in case the OS gets any
# ideas to issue partial writes.
fsize = int(metadata[X_CONTENT_LENGTH])
self.drop_cache(fd, 0, fsize)
# At this point we know that the object's full directory path exists,
# so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
data_file = os.path.join(self.put_datadir, self._obj)
while True:
try:
os.rename(self.tmppath, data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO):
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a FUSE
# issue of some sort or a possible race condition. So
# let's sleep on it, and double check the environment
# after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self.tmppath) > 0 and len(data_file) > 0
tpstats = do_stat(self.tmppath)
tfstats = do_fstat(fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError('DiskFile.put(): temporary file,'
' %s, was already renamed'
' (targeted for %s)' % (
self.tmppath, data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(self.put_datadir)
if not dfstats:
raise DiskFileError('DiskFile.put(): path to'
' object, %s, no longer exists'
' (targeted for %s)' % (
self.put_datadir,
data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError('DiskFile.put(): path to'
' object, %s, no longer a'
' directory (targeted for'
' %s)' % (self.put_datadir,
data_file))
else:
# Let's retry since everything looks okay
logging.warn("DiskFile.put(): os.rename('%s',"
"'%s') initially failed (%s) but"
" a stat('%s') following that"
" succeeded: %r" % (
self.tmppath, data_file,
str(err), self.put_datadir,
dfstats))
continue
else:
raise GlusterFileSystemOSError(
err.errno, "%s, os.rename('%s', '%s')" % (
err.strerror, self.tmppath, data_file))
else:
# Success!
break
# Avoid the unlink() system call as part of the mkstemp context cleanup
self.tmppath = None
self.metadata = metadata
self.filter_metadata()
# Mark that it actually exists now
self.data_file = os.path.join(self.datadir, self._obj)
def _set_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
metadata)
def _clear_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
utils.write_metadata(os.path.join(self.rootdir, obj), metadata)
def update_object(self, metadata):
obj_path = self.datadir + '/' + self.obj
write_metadata(obj_path, metadata)
self.metadata = metadata
def _set_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_OBJECT
utils.write_metadata(os.path.join(self.rootdir, self.dirs[0]),
metadata)
def make_directory(full_path, uid, gid, metadata=None):
"""
Make a directory and change the owner ship as specified, and potentially
creating the object metadata if requested.
"""
try:
os.mkdir(full_path)
except OSError as err:
if err.errno == errno.ENOENT:
# Tell the caller some directory of the parent path does not
# exist.
return False, metadata
elif err.errno == errno.EEXIST:
# Possible race, in that the caller invoked this method when it
# had previously determined the file did not exist.
#
# FIXME: When we are confident, remove this stat() call as it is
# not necessary.
try:
stats = os.stat(full_path)
except OSError as serr:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise DiskFileError("_make_directory_unlocked: os.mkdir failed"
" because path %s already exists, and"
" a subsequent os.stat on that same"
" path failed (%s)" % (full_path,
str(serr)))
else:
is_dir = stat.S_ISDIR(stats.st_mode)
if not is_dir:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("_make_directory_unlocked:"
" os.mkdir failed on path %s"
" because it already exists"
" but not as a directory"
% (full_path))
return True, metadata
elif err.errno == errno.ENOTDIR:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("_make_directory_unlocked:"
" os.mkdir failed because some "
"part of path %s is not in fact"
" a directory" % (full_path))
elif err.errno == errno.EIO:
# Sometimes Fuse will return an EIO error when it does not know
# how to handle an unexpected, but transient situation. It is
# possible the directory now exists, stat() it to find out after a
# short period of time.
_random_sleep()
try:
stats = os.stat(full_path)
except OSError as serr:
if serr.errno == errno.ENOENT:
errmsg = "_make_directory_unlocked: os.mkdir failed on" \
" path %s (EIO), and a subsequent os.stat on" \
" that same path did not find the file." % (
full_path,)
else:
errmsg = "_make_directory_unlocked: os.mkdir failed on" \
" path %s (%s), and a subsequent os.stat on" \
" that same path failed as well (%s)" % (
full_path, str(err), str(serr))
raise DiskFileError(errmsg)
else:
# The directory at least exists now
is_dir = stat.S_ISDIR(stats.st_mode)
if is_dir:
# Dump the stats to the log with the original exception.
logging.warn("_make_directory_unlocked: os.mkdir initially"
" failed on path %s (%s) but a stat()"
" following that succeeded: %r" % (full_path,
str(err),
stats))
# Assume another entity took care of the proper setup.
return True, metadata
else:
raise DiskFileError("_make_directory_unlocked: os.mkdir"
" initially failed on path %s (%s) but"
" now we see that it exists but is not"
" a directory (%r)" % (full_path,
str(err),
stats))
else:
# Some other potentially rare exception occurred that does not
# currently warrant a special log entry to help diagnose.
raise DiskFileError("_make_directory_unlocked: os.mkdir failed on"
" path %s (%s)" % (full_path, str(err)))
else:
if metadata:
# We were asked to set the initial metadata for this object.
metadata_orig = get_object_metadata(full_path)
metadata_orig.update(metadata)
write_metadata(full_path, metadata_orig)
metadata = metadata_orig
# We created it, so we are reponsible for always setting the proper
# ownership.
do_chown(full_path, uid, gid)
return True, metadata
def put_metadata(self, metadata):
obj_path = self.datadir + '/' + self.obj
write_metadata(obj_path, metadata)
self.metadata = metadata
def make_directory(full_path, uid, gid, metadata=None):
"""
Make a directory and change the owner ship as specified, and potentially
creating the object metadata if requested.
"""
try:
do_mkdir(full_path)
except OSError as err:
if err.errno == errno.ENOENT:
# Tell the caller some directory of the parent path does not
# exist.
return False, metadata
elif err.errno == errno.EEXIST:
# Possible race, in that the caller invoked this method when it
# had previously determined the file did not exist.
#
# FIXME: When we are confident, remove this stat() call as it is
# not necessary.
try:
stats = do_stat(full_path)
except GlusterFileSystemOSError as serr:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise DiskFileError("make_directory: mkdir failed"
" because path %s already exists, and"
" a subsequent stat on that same"
" path failed (%s)" %
(full_path, str(serr)))
else:
is_dir = stat.S_ISDIR(stats.st_mode)
if not is_dir:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed on path %s"
" because it already exists"
" but not as a directory" %
(full_path))
return True, metadata
elif err.errno == errno.ENOTDIR:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed because some "
"part of path %s is not in fact"
" a directory" % (full_path))
elif err.errno == errno.EIO:
# Sometimes Fuse will return an EIO error when it does not know
# how to handle an unexpected, but transient situation. It is
# possible the directory now exists, stat() it to find out after a
# short period of time.
_random_sleep()
try:
stats = do_stat(full_path)
except GlusterFileSystemOSError as serr:
if serr.errno == errno.ENOENT:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
else:
errmsg = "make_directory: mkdir failed on" \
" path %s (%s), and a subsequent stat on" \
" that same path failed as well (%s)" % (
full_path, str(err), str(serr))
raise DiskFileError(errmsg)
else:
if not stats:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
raise DiskFileError(errmsg)
else:
# The directory at least exists now
is_dir = stat.S_ISDIR(stats.st_mode)
if is_dir:
# Dump the stats to the log with the original exception
logging.warn("make_directory: mkdir initially"
" failed on path %s (%s) but a stat()"
" following that succeeded: %r" %
(full_path, str(err), stats))
# Assume another entity took care of the proper setup.
return True, metadata
else:
raise DiskFileError("make_directory: mkdir"
" initially failed on path %s (%s)"
" but now we see that it exists"
" but is not a directory (%r)" %
(full_path, str(err), stats))
else:
# Some other potentially rare exception occurred that does not
# currently warrant a special log entry to help diagnose.
raise DiskFileError("make_directory: mkdir failed on"
" path %s (%s)" % (full_path, str(err)))
else:
if metadata:
# We were asked to set the initial metadata for this object.
metadata_orig = get_object_metadata(full_path)
metadata_orig.update(metadata)
write_metadata(full_path, metadata_orig)
metadata = metadata_orig
# We created it, so we are reponsible for always setting the proper
# ownership.
do_chown(full_path, uid, gid)
return True, metadata
def put_metadata(self, metadata):
"""
Write metadata to directory/container.
"""
write_metadata(self.datadir, metadata)
self.metadata = metadata
def _clear_dir_object(self, obj):
metadata = utils.read_metadata(os.path.join(self.rootdir, obj))
metadata[utils.X_OBJECT_TYPE] = utils.DIR_NON_OBJECT
utils.write_metadata(os.path.join(self.rootdir, obj),
metadata)
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self._fd, metadata)
# We call fsync() before calling drop_cache() to lower the
# amount of redundant work the drop cache code will perform on
# the pages (now that after fsync the pages will be all
# clean).
do_fsync(self._fd)
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
# (pages all clean).
do_fadvise64(self._fd, self._last_sync, self._upload_size)
# At this point we know that the object's full directory path
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self._disk_file
attempts = 1
while True:
try:
do_rename(self._tmppath, df._data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a
# FUSE issue of some sort or a possible race
# condition. So let's sleep on it, and double check
# the environment after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self._tmppath) > 0 and len(df._data_file) > 0
tpstats = do_stat(self._tmppath)
tfstats = do_fstat(self._fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError(
'DiskFile.put(): temporary file, %s, was'
' already renamed (targeted for %s)' % (
self._tmppath, df._data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(df._put_datadir)
if not dfstats:
raise DiskFileError(
'DiskFile.put(): path to object, %s, no'
' longer exists (targeted for %s)' % (
df._put_datadir, df._data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError(
'DiskFile.put(): path to object, %s,'
' no longer a directory (targeted for'
' %s)' % (self._put_datadir,
df._data_file))
else:
# Let's retry since everything looks okay
logging.warn(
"DiskFile.put(): os.rename('%s','%s')"
" initially failed (%s) but a"
" stat('%s') following that succeeded:"
" %r" % (
self._tmppath, df._data_file, str(err),
df._put_datadir, dfstats))
attempts += 1
continue
else:
raise GlusterFileSystemOSError(
err.errno, "%s, os.rename('%s', '%s')" % (
err.strerror, self._tmppath, df._data_file))
else:
# Success!
break
# Close here so the calling context does not have to perform this
# in a thread.
self.close()
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self._fd, metadata)
# We call fsync() before calling drop_cache() to lower the
# amount of redundant work the drop cache code will perform on
# the pages (now that after fsync the pages will be all
# clean).
do_fsync(self._fd)
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
# (pages all clean).
do_fadvise64(self._fd, self._last_sync, self._upload_size)
# At this point we know that the object's full directory path
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self._disk_file
attempts = 1
while True:
try:
do_rename(self._tmppath, df._data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a
# FUSE issue of some sort or a possible race
# condition. So let's sleep on it, and double check
# the environment after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self._tmppath) > 0 and len(df._data_file) > 0
tpstats = do_stat(self._tmppath)
tfstats = do_fstat(self._fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError(
'DiskFile.put(): temporary file, %s, was'
' already renamed (targeted for %s)' %
(self._tmppath, df._data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(df._put_datadir)
if not dfstats:
raise DiskFileError(
'DiskFile.put(): path to object, %s, no'
' longer exists (targeted for %s)' %
(df._put_datadir, df._data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError(
'DiskFile.put(): path to object, %s,'
' no longer a directory (targeted for'
' %s)' %
(self._put_datadir, df._data_file))
else:
# Let's retry since everything looks okay
logging.warn(
"DiskFile.put(): rename('%s','%s')"
" initially failed (%s) but a"
" stat('%s') following that succeeded:"
" %r" %
(self._tmppath, df._data_file, str(err),
df._put_datadir, dfstats))
attempts += 1
continue
else:
raise GlusterFileSystemOSError(
err.errno, "%s, rename('%s', '%s')" %
(err.strerror, self._tmppath, df._data_file))
else:
# Success!
break
# Close here so the calling context does not have to perform this
# in a thread.
self.close()
def put(self, fd, tmppath, metadata, extension=''):
"""
Finalize writing the file on disk, and renames it from the temp file to
the real location. This should be called after the data has been
written to the temp file.
:params fd: file descriptor of the temp file
:param tmppath: path to the temporary file being used
:param metadata: dictionary of metadata to be written
:param extention: extension to be used when making the file
"""
if extension == '.ts':
# TombStone marker (deleted)
return True
# Fix up the metadata to ensure it has a proper value for the
# Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE
# metadata values.
content_type = metadata['Content-Type']
if not content_type:
metadata['Content-Type'] = FILE_TYPE
x_object_type = FILE
else:
x_object_type = MARKER_DIR if content_type.lower(
) == DIR_TYPE else FILE
metadata[X_TYPE] = OBJECT
metadata[X_OBJECT_TYPE] = x_object_type
if extension == '.meta':
# Metadata recorded separately from the file
self.put_metadata(metadata)
return True
extension = ''
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
self.create_dir_object(os.path.join(self.datadir, self.obj))
self.put_metadata(metadata)
self.data_file = self.datadir + '/' + self.obj
return True
# Check if directory already exists.
if self.is_dir:
self.logger.error('Directory already exists %s/%s' % \
(self.datadir , self.obj))
return False
timestamp = normalize_timestamp(metadata[X_TIMESTAMP])
write_metadata(tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
tpool.execute(os.fsync, fd)
if self.obj_path:
dir_objs = self.obj_path.split('/')
tmp_path = ''
if len(dir_objs):
for dir_name in dir_objs:
if tmp_path:
tmp_path = tmp_path + '/' + dir_name
else:
tmp_path = dir_name
if not self.create_dir_object(
os.path.join(self.container_path, tmp_path)):
self.logger.error("Failed in subdir %s",\
os.path.join(self.container_path,tmp_path))
return False
renamer(tmppath, os.path.join(self.datadir, self.obj + extension))
do_chown(os.path.join(self.datadir, self.obj + extension), \
self.uid, self.gid)
self.metadata = metadata
self.data_file = self.datadir + '/' + self.obj + extension
return True
def update_object(self, metadata):
obj_path = self.datadir + '/' + self.obj
write_metadata(obj_path, metadata)
self.metadata = metadata
