def do_stat(path):
serr = None
for i in range(0, _STAT_ATTEMPTS):
try:
stats = os.stat(path)
except OSError as err:
if err.errno == errno.EIO:
# Retry EIO assuming it is a transient error from FUSE after a
# short random sleep
serr = err
sleep(random.uniform(0.001, 0.005))
continue
if err.errno == errno.ENOENT:
stats = None
else:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.stat("%s")[%d attempts]' % (
err.strerror, path, i))
if i > 0:
logging.warn("fs_utils.do_stat():"
" os.stat('%s') retried %d times (%s)",
path, i, 'success' if stats else 'failure')
return stats
else:
raise SwiftOnFileSystemOSError(
serr.errno, '%s, os.stat("%s")[%d attempts]' % (
serr.strerror, path, _STAT_ATTEMPTS))
def do_read(fd, n):
try:
buf = os.read(fd, n)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd))
return buf
def do_rename(old_path, new_path):
try:
os.rename(old_path, new_path)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.rename("%s", "%s")' % (
err.strerror, old_path, new_path))
def do_fstat(fd):
try:
stats = os.fstat(fd)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.fstat(%s)' % (err.strerror, fd))
return stats
def do_fchown(fd, uid, gid):
try:
os.fchown(fd, uid, gid)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.fchown(%s, %s, %s)' % (
err.strerror, fd, uid, gid))
def do_chown(path, uid, gid):
try:
os.chown(path, uid, gid)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.chown("%s", %s, %s)' % (
err.strerror, path, uid, gid))
def do_fdatasync(fd):
try:
os.fdatasync(fd)
except AttributeError:
do_fsync(fd)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.fsync("%s")' % (err.strerror, fd))
def do_open(path, flags, **kwargs):
try:
fd = os.open(path, flags, **kwargs)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.open("%s", %x, %r)' % (
err.strerror, path, flags, kwargs))
return fd
def do_open(path, flags, mode=0o777):
try:
fd = os.open(path, flags, mode)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno,
'%s, os.open("%s", %x, %o)' % (err.strerror, path, flags, mode))
return fd
def do_unlink(path, log=True):
try:
os.unlink(path)
except OSError as err:
if err.errno != errno.ENOENT:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.unlink("%s")' % (err.strerror, path))
else:
logging.warn("fs_utils: os.unlink failed on non-existent path: %s",
path)
def do_close(fd):
try:
os.close(fd)
except OSError as err:
if err.errno in (errno.ENOSPC, errno.EDQUOT):
filename = get_filename_from_fd(fd)
do_log_rl("do_close(%d) failed: %s : %s", fd, err, filename)
raise DiskFileNoSpace()
else:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.close(%s)' % (err.strerror, fd))
def do_ismount(path):
"""
Test whether a path is a mount point.
This is code hijacked from C Python 2.6.8, adapted to remove the extra
lstat() system call.
"""
try:
s1 = os.lstat(path)
except os.error as err:
if err.errno == errno.ENOENT:
# It doesn't exist -- so not a mount point :-)
return False
else:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.lstat("%s")' % (err.strerror, path))
if stat.S_ISLNK(s1.st_mode):
# A symlink can never be a mount point
return False
try:
s2 = os.lstat(os.path.join(path, '..'))
except os.error as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.lstat("%s")' % (err.strerror,
os.path.join(path, '..')))
dev1 = s1.st_dev
dev2 = s2.st_dev
if dev1 != dev2:
# path/.. on a different device as path
return True
ino1 = s1.st_ino
ino2 = s2.st_ino
if ino1 == ino2:
# path/.. is the same i-node as path
return True
return False
def do_write(fd, buf):
try:
cnt = os.write(fd, buf)
except OSError as err:
filename = get_filename_from_fd(fd)
if err.errno in (errno.ENOSPC, errno.EDQUOT):
do_log_rl("do_write(%d, msg[%d]) failed: %s : %s",
fd, len(buf), err, filename)
raise DiskFileNoSpace()
else:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.write("%s", ...)' % (err.strerror, fd))
return cnt
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self._fd, metadata)
# We call fsync() before calling drop_cache() to lower the
# amount of redundant work the drop cache code will perform on
# the pages (now that after fsync the pages will be all
# clean).
do_fsync(self._fd)
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
# (pages all clean).
do_fadvise64(self._fd, self._last_sync, self._upload_size)
# At this point we know that the object's full directory path
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self._disk_file
attempts = 1
while True:
try:
do_rename(self._tmppath, df._data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a
# FUSE issue of some sort or a possible race
# condition. So let's sleep on it, and double check
# the environment after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self._tmppath) > 0 and len(df._data_file) > 0
tpstats = do_stat(self._tmppath)
tfstats = do_fstat(self._fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError(
'DiskFile.put(): temporary file, %s, was'
' already renamed (targeted for %s)' %
(self._tmppath, df._data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(df._put_datadir)
if not dfstats:
raise DiskFileError(
'DiskFile.put(): path to object, %s, no'
' longer exists (targeted for %s)' %
(df._put_datadir, df._data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError(
'DiskFile.put(): path to object, %s,'
' no longer a directory (targeted for'
' %s)' %
(self._put_datadir, df._data_file))
else:
# Let's retry since everything looks okay
logging.warn(
"DiskFile.put(): rename('%s','%s')"
" initially failed (%s) but a"
" stat('%s') following that succeeded:"
" %r" %
(self._tmppath, df._data_file, str(err),
df._put_datadir, dfstats))
attempts += 1
continue
else:
raise SwiftOnFileSystemOSError(
err.errno, "%s, rename('%s', '%s')" %
(err.strerror, self._tmppath, df._data_file))
else:
# Success!
break
# Close here so the calling context does not have to perform this
# in a thread.
self.close()
def do_lseek(fd, pos, how):
try:
os.lseek(fd, pos, how)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.fsync("%s")' % (err.strerror, fd))
def do_rmdir(path):
try:
os.rmdir(path)
except OSError as err:
raise SwiftOnFileSystemOSError(
err.errno, '%s, os.rmdir("%s")' % (err.strerror, path))