def test_do_stat_eio_twice(self):
count = [0]
_os_stat = os.stat
def mock_os_stat_eio(path):
count[0] += 1
if count[0] <= 2:
raise OSError(errno.EIO, os.strerror(errno.EIO))
return _os_stat(path)
with patch('os.stat', mock_os_stat_eio):
fs.do_stat('/tmp') is not None
def test_do_stat_eio_ten(self):
def mock_os_stat_eio(path):
raise OSError(errno.EIO, os.strerror(errno.EIO))
try:
with patch('os.stat', mock_os_stat_eio):
fs.do_stat('/tmp')
except SwiftOnFileSystemOSError:
pass
else:
self.fail("SwiftOnFileSystemOSError expected")
def test_do_stat_err(self):
def mock_os_stat_eacces(path):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
try:
with patch('os.stat', mock_os_stat_eacces):
fs.do_stat('/tmp')
except SwiftOnFileSystemOSError:
pass
else:
self.fail("SwiftOnFileSystemOSError expected")
def read_metadata(self):
"""
Return the metadata for an object without opening the object's file on
disk.
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
# FIXME: pull a lot of this and the copy of it from open() out to
# another function
# Do not actually open the file, in order to duck hpssfs checksum
# validation and resulting timeouts
# This means we do a few things DiskFile.open() does.
try:
if not self._stat:
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
self._metadata = read_metadata(self._data_file)
except SwiftOnFileSystemOSError:
raise DiskFileNotExist
if not self._validate_object_metadata():
self._create_object_metadata(self._data_file)
self._filter_metadata()
return self._metadata
def get_object_metadata(obj_path_or_fd, stats=None):
"""
Return metadata of object.
"""
if not stats:
if isinstance(obj_path_or_fd, int):
# We are given a file descriptor, so this is an invocation from the
# DiskFile.open() method.
stats = do_fstat(obj_path_or_fd)
else:
# We are given a path to the object when the
# DiskDir.list_objects_iter method invokes us.
stats = do_stat(obj_path_or_fd)
if not stats:
metadata = {}
else:
is_dir = stat.S_ISDIR(stats.st_mode)
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_MTIME: 0 if is_dir else normalize_timestamp(stats.st_mtime),
X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path_or_fd)}
return metadata
def test_do_stat(self):
tmpdir = mkdtemp()
try:
fd, tmpfile = mkstemp(dir=tmpdir)
buf1 = os.stat(tmpfile)
buf2 = fs.do_stat(tmpfile)
assert buf1 == buf2
finally:
os.close(fd)
os.remove(tmpfile)
os.rmdir(tmpdir)
def _validate_object_metadata(self):
# Has no Swift specific metadata saved as xattr. Probably because
# object was added/replaced through filesystem interface.
if not self._metadata and not self._is_dir:
self._file_has_changed = True
return False
required_keys = \
(X_TIMESTAMP, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_ETAG,
# SOF specific keys
X_TYPE, X_OBJECT_TYPE)
if not all(k in self._metadata for k in required_keys):
# At least one of the required keys does not exist
return False
if not self._is_dir:
# X_MTIME is a new key added recently, newer objects will
# have the key set during PUT.
if X_MTIME in self._metadata:
# Check if the file has been modified through filesystem
# interface by comparing mtime stored in xattr during PUT
# and current mtime of file.
obj_stat = do_stat(self._data_file)
if normalize_timestamp(self._metadata[X_MTIME]) != \
normalize_timestamp(obj_stat.hpss_st_mtime):
self._file_has_changed = True
return False
else:
# Without X_MTIME key, comparing md5sum is the only way
# to determine if file has changed or not. This is inefficient
# but there's no other way!
self._etag = get_etag(self._data_file)
if self._etag != self._metadata[X_ETAG]:
self._file_has_changed = True
return False
else:
# Checksums are same; File has not changed. For the next
# GET request on same file, we don't compute md5sum again!
# This is achieved by setting X_MTIME to mtime in
# _create_object_metadata()
return False
if self._metadata[X_TYPE] == OBJECT:
return True
return False
def get_object_metadata(obj_path, stats=None):
"""
Return metadata of object.
"""
if not stats:
stats = do_stat(obj_path)
if not stats:
metadata = {}
else:
is_dir = stat.S_ISDIR(stats.st_mode)
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.hpss_st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_MTIME: 0 if is_dir else normalize_timestamp(stats.hpss_st_mtime),
X_ETAG: md5().hexdigest() if is_dir else get_etag(obj_path)}
return metadata
def make_directory(full_path, uid, gid, metadata=None):
"""
Make a directory and change the owner ship as specified, and potentially
creating the object metadata if requested.
"""
try:
do_mkdir(full_path)
except OSError as err:
if err.errno == errno.ENOENT:
# Tell the caller some directory of the parent path does not
# exist.
return False, metadata
elif err.errno == errno.EEXIST:
# Possible race, in that the caller invoked this method when it
# had previously determined the file did not exist.
#
# FIXME: When we are confident, remove this stat() call as it is
# not necessary.
try:
stats = do_stat(full_path)
except SwiftOnFileSystemOSError as serr:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise DiskFileError("make_directory: mkdir failed"
" because path %s already exists, and"
" a subsequent stat on that same"
" path failed (%s)" % (full_path,
str(serr)))
else:
is_dir = stat.S_ISDIR(stats.st_mode)
if not is_dir:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed on path %s"
" because it already exists"
" but not as a directory"
% (full_path))
return True, metadata
elif err.errno == errno.ENOTDIR:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed because some "
"part of path %s is not in fact"
" a directory" % (full_path))
elif err.errno == errno.EIO:
# Sometimes Fuse will return an EIO error when it does not know
# how to handle an unexpected, but transient situation. It is
# possible the directory now exists, stat() it to find out after a
# short period of time.
_random_sleep()
try:
stats = do_stat(full_path)
except SwiftOnFileSystemOSError as serr:
if serr.errno == errno.ENOENT:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
else:
errmsg = "make_directory: mkdir failed on" \
" path %s (%s), and a subsequent stat on" \
" that same path failed as well (%s)" % (
full_path, str(err), str(serr))
raise DiskFileError(errmsg)
else:
if not stats:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
raise DiskFileError(errmsg)
else:
# The directory at least exists now
is_dir = stat.S_ISDIR(stats.st_mode)
if is_dir:
# Dump the stats to the log with the original exception
logging.warn("make_directory: mkdir initially"
" failed on path %s (%s) but a stat()"
" following that succeeded: %r" %
(full_path, str(err), stats))
# Assume another entity took care of the proper setup.
return True, metadata
else:
raise DiskFileError("make_directory: mkdir"
" initially failed on path %s (%s)"
" but now we see that it exists"
" but is not a directory (%r)" %
(full_path, str(err), stats))
else:
# Some other potentially rare exception occurred that does not
# currently warrant a special log entry to help diagnose.
raise DiskFileError("make_directory: mkdir failed on"
" path %s (%s)" % (full_path, str(err)))
else:
if metadata:
# We were asked to set the initial metadata for this object.
metadata_orig = get_object_metadata(full_path)
metadata_orig.update(metadata)
write_metadata(full_path, metadata_orig)
metadata = metadata_orig
# We created it, so we are reponsible for always setting the proper
# ownership.
do_chown(full_path, uid, gid)
return True, metadata
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._logger.debug("DiskFile: Opening %s" % self._data_file)
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
if not self._is_dir:
self._fd = do_open(self._data_file, hpss.O_RDONLY)
self._obj_size = self._stat.st_size
else:
self._fd = -1
self._obj_size = 0
except SwiftOnFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
self._logger.debug("DiskFile: Reading metadata")
self._metadata = read_metadata(self._data_file)
if not self._validate_object_metadata():
self._create_object_metadata(self._data_file)
assert self._metadata is not None
self._filter_metadata()
if not self._is_dir:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and err.errno == errno.ENOENT:
# Handle races: ENOENT can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
return self
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self._tmppath, metadata)
do_fsync(self._fd)
self.set_checksum(metadata['ETag'])
# At this point we know that the object's full directory path
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self._disk_file
attempts = 1
while True:
try:
do_rename(self._tmppath, df._data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a
# FUSE issue of some sort or a possible race
# condition. So let's sleep on it, and double check
# the environment after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self._tmppath) > 0 and len(df._data_file) > 0
tpstats = do_stat(self._tmppath)
tfstats = do_fstat(self._fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError(
'DiskFile.put(): temporary file, %s, was'
' already renamed (targeted for %s)' % (
self._tmppath, df._data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(df._put_datadir)
if not dfstats:
raise DiskFileError(
'DiskFile.put(): path to object, %s, no'
' longer exists (targeted for %s)' % (
df._put_datadir, df._data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError(
'DiskFile.put(): path to object, %s,'
' no longer a directory (targeted for'
' %s)' % (self._put_datadir,
df._data_file))
else:
# Let's retry since everything looks okay
logging.warn(
"DiskFile.put(): rename('%s','%s')"
" initially failed (%s) but a"
" stat('%s') following that succeeded:"
" %r" % (
self._tmppath, df._data_file, str(err),
df._put_datadir, dfstats))
attempts += 1
continue
else:
raise SwiftOnFileSystemOSError(
err.errno, "%s, rename('%s', '%s')" % (
err.strerror, self._tmppath, df._data_file))
else:
# Success!
break
# Close here so the calling context does not have to perform this
# in a thread.
self.close()
def test_do_stat_enoent(self):
res = fs.do_stat(os.path.join('/tmp', str(random.random())))
assert res is None