def test_do_stat_err(self):
try:
fs.do_stat(os.path.join('/tmp', str(random.random())))
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_stat_err(self):
def mock_os_stat_eacces(path):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
try:
with patch('os.stat', mock_os_stat_eacces):
fs.do_stat('/tmp')
except GlusterFileSystemOSError:
pass
else:
self.fail("GlusterFileSystemOSError expected")
def test_do_stat_eio_ten(self):
def mock_os_stat_eio(path):
raise OSError(errno.EIO, os.strerror(errno.EIO))
try:
with patch('os.stat', mock_os_stat_eio):
fs.do_stat('/tmp')
except GlusterFileSystemOSError:
pass
else:
self.fail("GlusterFileSystemOSError expected")
def test_do_stat_eio_twice(self):
count = [0]
_os_stat = os.stat
def mock_os_stat_eio(path):
count[0] += 1
if count[0] <= 2:
raise OSError(errno.EIO, os.strerror(errno.EIO))
return _os_stat(path)
with patch('os.stat', mock_os_stat_eio):
fs.do_stat('/tmp') is not None
def test_do_stat(self):
try:
tmpdir = mkdtemp()
fd, tmpfile = mkstemp(dir=tmpdir)
buf1 = os.stat(tmpfile)
buf2 = fs.do_stat(fd)
buf3 = fs.do_stat(tmpfile)
assert buf1 == buf2
assert buf1 == buf3
finally:
os.close(fd)
os.remove(tmpfile)
os.rmdir(tmpdir)
def __init__(self, root, drive, account, logger, **kwargs):
super(DiskAccount, self).__init__(root, drive, account, logger,
**kwargs)
if self.account == 'gsexpiring':
# Do not bother updating object count, container count and bytes
# used. Return immediately before metadata validation and
# creation happens.
info = do_stat(self.datadir)
if info and stat.S_ISDIR(info.st_mode):
self._dir_exists = True
semi_fake_md = {
'X-Object-Count': (0, 0),
'X-Container-Count': (0, 0),
'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0),
'X-Type': ('Account', 0),
'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0),
'X-Bytes-Used': (0, 0)
}
self.metadata = semi_fake_md
return
# Since accounts should always exist (given an account maps to a
# gluster volume directly, and the mount has already been checked at
# the beginning of the REST API handling), just assert that that
# assumption still holds.
assert self._dir_exists_read_metadata()
assert self._dir_exists
if not self.metadata or not validate_account(self.metadata):
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def _make_directory_locked(full_path, uid, gid, metadata=None):
fd = _lock_parent(full_path)
if fd is False:
# Parent does not exist either, pass this situation on to the caller
# to handle.
return False, metadata
try:
# Check for directory existence
stats = do_stat(full_path)
if stats:
# It now exists, having acquired the lock of its parent directory,
# but verify it is actually a directory
is_dir = stat.S_ISDIR(stats.st_mode)
if not is_dir:
# It is not a directory!
raise DiskFileError("_make_directory_locked: non-directory"
" found at path %s when expecting a"
" directory", full_path)
return True, metadata
# We know the parent directory exists, and we have it locked, attempt
# the creation of the target directory.
return _make_directory_unlocked(full_path, uid, gid, metadata=metadata)
finally:
# We're done here, be sure to remove our lock and close our open FD.
try:
fcntl.flock(fd, fcntl.LOCK_UN)
except:
pass
os.close(fd)
def get_object_metadata(obj_path_or_fd):
"""
Return metadata of object.
"""
if isinstance(obj_path_or_fd, int):
# We are given a file descriptor, so this is an invocation from the
# DiskFile.open() method.
stats = do_fstat(obj_path_or_fd)
else:
# We are given a path to the object when the DiskDir.list_objects_iter
# method invokes us.
stats = do_stat(obj_path_or_fd)
if not stats:
metadata = {}
else:
is_dir = stat.S_ISDIR(stats.st_mode)
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path_or_fd)
}
return metadata
def get_object_metadata(obj_path_or_fd):
"""
Return metadata of object.
"""
if isinstance(obj_path_or_fd, int):
# We are given a file descriptor, so this is an invocation from the
# DiskFile.open() method.
stats = do_fstat(obj_path_or_fd)
else:
# We are given a path to the object when the DiskDir.list_objects_iter
# method invokes us.
stats = do_stat(obj_path_or_fd)
if not stats:
metadata = {}
else:
is_dir = stat.S_ISDIR(stats.st_mode)
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path_or_fd)}
return metadata
def _get_account_details_from_fs(acc_path, acc_stats):
container_list = []
container_count = 0
if not acc_stats:
acc_stats = do_stat(acc_path)
is_dir = (acc_stats.st_mode & 0040000) != 0
if is_dir:
for name in do_listdir(acc_path):
if name.lower() == TEMP_DIR \
or name.lower() == ASYNCDIR \
or not os_path.isdir(os.path.join(acc_path, name)):
continue
container_count += 1
container_list.append(name)
return AccountDetails(acc_stats.st_mtime, container_count, container_list)
def read_metadata(self):
"""
Return the metadata for an object without requiring the caller to open
the object first.
This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
metadata = disk_file.read_metadata()
The operations performed here is very similar to those made in open().
This is to avoid opening and closing of file (two syscalls over
network). IOW, this optimization addresses the case where the fd
returned by open() isn't going to be used i.e the file is not read (GET
or metadata recalculation)
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
try:
self._metadata = read_metadata(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if self._metadata and self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
try:
self._stat = do_stat(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if not validate_object(self._metadata, self._stat):
# Metadata is stale/invalid. So open the object for reading
# to update Etag and other metadata.
with self.open():
return self._metadata
else:
# Metadata is valid. Don't have to open the file.
self._filter_metadata()
return self._metadata
def read_metadata(self):
"""
Return the metadata for an object without requiring the caller to open
the object first.
This method is invoked by Swift code in POST, PUT, HEAD and DELETE path
metadata = disk_file.read_metadata()
The operations performed here is very similar to those made in open().
This is to avoid opening and closing of file (two syscalls over
network). IOW, this optimization addresses the case where the fd
returned by open() isn't going to be used i.e the file is not read (GET
or metadata recalculation)
:returns: metadata dictionary for an object
:raises DiskFileError: this implementation will raise the same
errors as the `open()` method.
"""
try:
self._metadata = read_metadata(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if self._metadata and self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
try:
self._stat = do_stat(self._data_file)
except (OSError, IOError) as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
self._disk_file_does_not_exist = True
raise DiskFileNotExist
raise err
if not validate_object(self._metadata, self._stat):
# Metadata is stale/invalid. So open the object for reading
# to update Etag and other metadata.
with self.open():
return self._metadata
else:
# Metadata is valid. Don't have to open the file.
self._filter_metadata()
return self._metadata
def get_object_metadata(obj_path):
"""
Return metadata of object.
"""
stats = do_stat(obj_path)
if not stats:
metadata = {}
else:
is_dir = stat.S_ISDIR(stats.st_mode)
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR_NON_OBJECT if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path)}
return metadata
def _get_container_details_from_fs(cont_path):
"""
get container details by traversing the filesystem
"""
bytes_used = 0
object_count = 0
obj_list = []
dir_list = []
if os_path.isdir(cont_path):
for (path, dirs, files) in do_walk(cont_path):
object_count, bytes_used = update_list(path, cont_path, dirs,
files, object_count,
bytes_used, obj_list)
dir_list.append((path, do_stat(path).st_mtime))
sleep()
return ContainerDetails(bytes_used, object_count, obj_list, dir_list)
def get_account_details(acc_path):
"""
Return container_list and container_count.
"""
container_list = []
container_count = 0
acc_stats = do_stat(acc_path)
if acc_stats:
is_dir = stat.S_ISDIR(acc_stats.st_mode)
if is_dir:
for name in do_listdir(acc_path):
if name.lower() == TEMP_DIR \
or name.lower() == ASYNCDIR \
or not os_path.isdir(os.path.join(acc_path, name)):
continue
container_count += 1
container_list.append(name)
return container_list, container_count
def get_object_metadata(obj_path):
"""
Return metadata of object.
"""
try:
stats = do_stat(obj_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
metadata = {}
else:
is_dir = (stats.st_mode & 0040000) != 0
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path)}
return metadata
def __init__(self, path, drive, account, container, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs):
super(DiskDir, self).__init__(path, drive, account, logger, **kwargs)
self.uid = int(uid)
self.gid = int(gid)
self.container = container
self.datadir = os.path.join(self.datadir, self.container)
if self.account == 'gsexpiring':
# Do not bother crawling the entire container tree just to update
# object count and bytes used. Return immediately before metadata
# validation and creation happens.
info = do_stat(self.datadir)
if info and stat.S_ISDIR(info.st_mode):
self._dir_exists = True
if not info:
# Container no longer exists.
return
semi_fake_md = {
'X-Object-Count': (0, 0),
'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0),
'X-Type': ('container', 0),
'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0),
'X-Bytes-Used': (0, 0)
}
self.metadata = semi_fake_md
return
if not self._dir_exists_read_metadata():
return
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def get_container_details(cont_path, memcache=None):
"""
Return object_list, object_count and bytes_used.
"""
mkey = ''
if memcache:
mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path
cd = memcache.get(mkey)
if cd:
if not cd.dir_list:
cd = None
else:
for (path, mtime) in cd.dir_list:
if mtime != do_stat(path).st_mtime:
cd = None
else:
cd = None
if not cd:
cd = _get_container_details_from_fs(cont_path)
if memcache:
memcache.set(mkey, cd)
return cd.obj_list, cd.object_count, cd.bytes_used
def get_account_details(acc_path, memcache=None):
"""
Return container_list and container_count.
"""
acc_stats = None
mkey = ''
if memcache:
mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path
ad = memcache.get(mkey)
if ad:
# FIXME: Do we really need to stat the file? If we are object
# only, then we can track the other Swift HTTP APIs that would
# modify the account and invalidate the cached entry there. If we
# are not object only, are we even called on this path?
acc_stats = do_stat(acc_path)
if ad.mtime != acc_stats.st_mtime:
ad = None
else:
ad = None
if not ad:
ad = _get_account_details_from_fs(acc_path, acc_stats)
if memcache:
memcache.set(mkey, ad)
return ad.container_list, ad.container_count
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self._fd, metadata)
# We call fsync() before calling drop_cache() to lower the
# amount of redundant work the drop cache code will perform on
# the pages (now that after fsync the pages will be all
# clean).
do_fsync(self._fd)
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
# (pages all clean).
do_fadvise64(self._fd, self._last_sync, self._upload_size)
# At this point we know that the object's full directory path
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self._disk_file
attempts = 1
while True:
try:
do_rename(self._tmppath, df._data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a
# FUSE issue of some sort or a possible race
# condition. So let's sleep on it, and double check
# the environment after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self._tmppath) > 0 and len(df._data_file) > 0
tpstats = do_stat(self._tmppath)
tfstats = do_fstat(self._fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError(
'DiskFile.put(): temporary file, %s, was'
' already renamed (targeted for %s)' % (
self._tmppath, df._data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(df._put_datadir)
if not dfstats:
raise DiskFileError(
'DiskFile.put(): path to object, %s, no'
' longer exists (targeted for %s)' % (
df._put_datadir, df._data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError(
'DiskFile.put(): path to object, %s,'
' no longer a directory (targeted for'
' %s)' % (self._put_datadir,
df._data_file))
else:
# Let's retry since everything looks okay
logging.warn(
"DiskFile.put(): os.rename('%s','%s')"
" initially failed (%s) but a"
" stat('%s') following that succeeded:"
" %r" % (
self._tmppath, df._data_file, str(err),
df._put_datadir, dfstats))
attempts += 1
continue
else:
raise GlusterFileSystemOSError(
err.errno, "%s, os.rename('%s', '%s')" % (
err.strerror, self._tmppath, df._data_file))
else:
# Success!
break
# Close here so the calling context does not have to perform this
# in a thread.
self.close()
def make_directory(full_path, uid, gid, metadata=None):
"""
Make a directory and change the owner ship as specified, and potentially
creating the object metadata if requested.
"""
try:
do_mkdir(full_path)
except OSError as err:
if err.errno == errno.ENOENT:
# Tell the caller some directory of the parent path does not
# exist.
return False, metadata
elif err.errno == errno.EEXIST:
# Possible race, in that the caller invoked this method when it
# had previously determined the file did not exist.
#
# FIXME: When we are confident, remove this stat() call as it is
# not necessary.
try:
stats = do_stat(full_path)
except GlusterFileSystemOSError as serr:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise DiskFileError("make_directory: mkdir failed"
" because path %s already exists, and"
" a subsequent stat on that same"
" path failed (%s)" % (full_path,
str(serr)))
else:
is_dir = stat.S_ISDIR(stats.st_mode)
if not is_dir:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed on path %s"
" because it already exists"
" but not as a directory"
% (full_path))
return True, metadata
elif err.errno == errno.ENOTDIR:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed because some "
"part of path %s is not in fact"
" a directory" % (full_path))
elif err.errno == errno.EIO:
# Sometimes Fuse will return an EIO error when it does not know
# how to handle an unexpected, but transient situation. It is
# possible the directory now exists, stat() it to find out after a
# short period of time.
_random_sleep()
try:
stats = do_stat(full_path)
except GlusterFileSystemOSError as serr:
if serr.errno == errno.ENOENT:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
else:
errmsg = "make_directory: mkdir failed on" \
" path %s (%s), and a subsequent stat on" \
" that same path failed as well (%s)" % (
full_path, str(err), str(serr))
raise DiskFileError(errmsg)
else:
if not stats:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
raise DiskFileError(errmsg)
else:
# The directory at least exists now
is_dir = stat.S_ISDIR(stats.st_mode)
if is_dir:
# Dump the stats to the log with the original exception
logging.warn("make_directory: mkdir initially"
" failed on path %s (%s) but a stat()"
" following that succeeded: %r" %
(full_path, str(err), stats))
# Assume another entity took care of the proper setup.
return True, metadata
else:
raise DiskFileError("make_directory: mkdir"
" initially failed on path %s (%s)"
" but now we see that it exists"
" but is not a directory (%r)" %
(full_path, str(err), stats))
else:
# Some other potentially rare exception occurred that does not
# currently warrant a special log entry to help diagnose.
raise DiskFileError("make_directory: mkdir failed on"
" path %s (%s)" % (full_path, str(err)))
else:
if metadata:
# We were asked to set the initial metadata for this object.
metadata_orig = get_object_metadata(full_path)
metadata_orig.update(metadata)
write_metadata(full_path, metadata_orig)
metadata = metadata_orig
# We created it, so we are reponsible for always setting the proper
# ownership.
do_chown(full_path, uid, gid)
return True, metadata
def put(self, fd, metadata, extension='.data'):
"""
Finalize writing the file on disk, and renames it from the temp file
to the real location. This should be called after the data has been
written to the temp file.
:param fd: file descriptor of the temp file
:param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
metadata = _adjust_metadata(metadata)
if dir_is_object(metadata):
if not self.data_file:
# Does not exist, create it
data_file = os.path.join(self._obj_path, self._obj)
_, self.metadata = self._create_dir_object(data_file, metadata)
self.data_file = os.path.join(self._container_path, data_file)
elif not self.is_dir:
# Exists, but as a file
raise DiskFileError('DiskFile.put(): directory creation failed'
' since the target, %s, already exists as'
' a file' % self.data_file)
return
if self._is_dir:
# A pre-existing directory already exists on the file
# system, perhaps gratuitously created when another
# object was created, or created externally to Swift
# REST API servicing (UFO use case).
raise DiskFileError('DiskFile.put(): file creation failed since'
' the target, %s, already exists as a'
' directory' % self.data_file)
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(fd, metadata)
if not _relaxed_writes:
do_fsync(fd)
if X_CONTENT_LENGTH in metadata:
# Don't bother doing this before fsync in case the OS gets any
# ideas to issue partial writes.
fsize = int(metadata[X_CONTENT_LENGTH])
self.drop_cache(fd, 0, fsize)
# At this point we know that the object's full directory path exists,
# so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
data_file = os.path.join(self.put_datadir, self._obj)
while True:
try:
os.rename(self.tmppath, data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO):
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a FUSE
# issue of some sort or a possible race condition. So
# let's sleep on it, and double check the environment
# after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self.tmppath) > 0 and len(data_file) > 0
tpstats = do_stat(self.tmppath)
tfstats = do_fstat(fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError('DiskFile.put(): temporary file,'
' %s, was already renamed'
' (targeted for %s)' % (
self.tmppath, data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(self.put_datadir)
if not dfstats:
raise DiskFileError('DiskFile.put(): path to'
' object, %s, no longer exists'
' (targeted for %s)' % (
self.put_datadir,
data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError('DiskFile.put(): path to'
' object, %s, no longer a'
' directory (targeted for'
' %s)' % (self.put_datadir,
data_file))
else:
# Let's retry since everything looks okay
logging.warn("DiskFile.put(): os.rename('%s',"
"'%s') initially failed (%s) but"
" a stat('%s') following that"
" succeeded: %r" % (
self.tmppath, data_file,
str(err), self.put_datadir,
dfstats))
continue
else:
raise GlusterFileSystemOSError(
err.errno, "%s, os.rename('%s', '%s')" % (
err.strerror, self.tmppath, data_file))
else:
# Success!
break
# Avoid the unlink() system call as part of the mkstemp context cleanup
self.tmppath = None
self.metadata = metadata
self.filter_metadata()
# Mark that it actually exists now
self.data_file = os.path.join(self.datadir, self._obj)
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.iter_hook = iter_hook
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
self._obj_path, self._obj = os.path.split(obj)
else:
self._obj_path = ''
self._obj = obj
if self._obj_path:
self.name = os.path.join(container, self._obj_path)
else:
self.name = container
# Absolute path for object directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self._container_path = os.path.join(path, device, container)
if _use_put_mount:
self.put_datadir = os.path.join(self.device_path + '_PUT',
self.name)
else:
self.put_datadir = self.datadir
self._is_dir = False
self.tmppath = None
self.logger = logger
self.metadata = {}
self.meta_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
self.suppress_file_closing = False
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.put_datadir, self._obj)
try:
stats = do_stat(data_file)
except OSError as err:
if err.errno == errno.ENOTDIR:
return
else:
if not stats:
return
self.data_file = data_file
self._is_dir = stat.S_ISDIR(stats.st_mode)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.filter_metadata()
if not self._is_dir and keep_data_fp:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self._fd, metadata)
# We call fsync() before calling drop_cache() to lower the
# amount of redundant work the drop cache code will perform on
# the pages (now that after fsync the pages will be all
# clean).
do_fsync(self._fd)
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
# (pages all clean).
do_fadvise64(self._fd, self._last_sync, self._upload_size)
# At this point we know that the object's full directory path
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self._disk_file
attempts = 1
while True:
try:
do_rename(self._tmppath, df._data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
# FIXME: Why either of these two error conditions is
# happening is unknown at this point. This might be a
# FUSE issue of some sort or a possible race
# condition. So let's sleep on it, and double check
# the environment after a good nap.
_random_sleep()
# Tease out why this error occurred. The man page for
# rename reads:
# "The link named by tmppath does not exist; or, a
# directory component in data_file does not exist;
# or, tmppath or data_file is an empty string."
assert len(self._tmppath) > 0 and len(df._data_file) > 0
tpstats = do_stat(self._tmppath)
tfstats = do_fstat(self._fd)
assert tfstats
if not tpstats or tfstats.st_ino != tpstats.st_ino:
# Temporary file name conflict
raise DiskFileError(
'DiskFile.put(): temporary file, %s, was'
' already renamed (targeted for %s)' %
(self._tmppath, df._data_file))
else:
# Data file target name now has a bad path!
dfstats = do_stat(df._put_datadir)
if not dfstats:
raise DiskFileError(
'DiskFile.put(): path to object, %s, no'
' longer exists (targeted for %s)' %
(df._put_datadir, df._data_file))
else:
is_dir = stat.S_ISDIR(dfstats.st_mode)
if not is_dir:
raise DiskFileError(
'DiskFile.put(): path to object, %s,'
' no longer a directory (targeted for'
' %s)' %
(self._put_datadir, df._data_file))
else:
# Let's retry since everything looks okay
logging.warn(
"DiskFile.put(): rename('%s','%s')"
" initially failed (%s) but a"
" stat('%s') following that succeeded:"
" %r" %
(self._tmppath, df._data_file, str(err),
df._put_datadir, dfstats))
attempts += 1
continue
else:
raise GlusterFileSystemOSError(
err.errno, "%s, rename('%s', '%s')" %
(err.strerror, self._tmppath, df._data_file))
else:
# Success!
break
# Close here so the calling context does not have to perform this
# in a thread.
self.close()
def make_directory(full_path, uid, gid, metadata=None):
"""
Make a directory and change the owner ship as specified, and potentially
creating the object metadata if requested.
"""
try:
do_mkdir(full_path)
except OSError as err:
if err.errno == errno.ENOENT:
# Tell the caller some directory of the parent path does not
# exist.
return False, metadata
elif err.errno == errno.EEXIST:
# Possible race, in that the caller invoked this method when it
# had previously determined the file did not exist.
#
# FIXME: When we are confident, remove this stat() call as it is
# not necessary.
try:
stats = do_stat(full_path)
except GlusterFileSystemOSError as serr:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise DiskFileError("make_directory: mkdir failed"
" because path %s already exists, and"
" a subsequent stat on that same"
" path failed (%s)" %
(full_path, str(serr)))
else:
is_dir = stat.S_ISDIR(stats.st_mode)
if not is_dir:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed on path %s"
" because it already exists"
" but not as a directory" %
(full_path))
return True, metadata
elif err.errno == errno.ENOTDIR:
# FIXME: Ideally we'd want to return an appropriate error
# message and code in the PUT Object REST API response.
raise AlreadyExistsAsFile("make_directory:"
" mkdir failed because some "
"part of path %s is not in fact"
" a directory" % (full_path))
elif err.errno == errno.EIO:
# Sometimes Fuse will return an EIO error when it does not know
# how to handle an unexpected, but transient situation. It is
# possible the directory now exists, stat() it to find out after a
# short period of time.
_random_sleep()
try:
stats = do_stat(full_path)
except GlusterFileSystemOSError as serr:
if serr.errno == errno.ENOENT:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
else:
errmsg = "make_directory: mkdir failed on" \
" path %s (%s), and a subsequent stat on" \
" that same path failed as well (%s)" % (
full_path, str(err), str(serr))
raise DiskFileError(errmsg)
else:
if not stats:
errmsg = "make_directory: mkdir failed on" \
" path %s (EIO), and a subsequent stat on" \
" that same path did not find the file." % (
full_path,)
raise DiskFileError(errmsg)
else:
# The directory at least exists now
is_dir = stat.S_ISDIR(stats.st_mode)
if is_dir:
# Dump the stats to the log with the original exception
logging.warn("make_directory: mkdir initially"
" failed on path %s (%s) but a stat()"
" following that succeeded: %r" %
(full_path, str(err), stats))
# Assume another entity took care of the proper setup.
return True, metadata
else:
raise DiskFileError("make_directory: mkdir"
" initially failed on path %s (%s)"
" but now we see that it exists"
" but is not a directory (%r)" %
(full_path, str(err), stats))
else:
# Some other potentially rare exception occurred that does not
# currently warrant a special log entry to help diagnose.
raise DiskFileError("make_directory: mkdir failed on"
" path %s (%s)" % (full_path, str(err)))
else:
if metadata:
# We were asked to set the initial metadata for this object.
metadata_orig = get_object_metadata(full_path)
metadata_orig.update(metadata)
write_metadata(full_path, metadata_orig)
metadata = metadata_orig
# We created it, so we are reponsible for always setting the proper
# ownership.
do_chown(full_path, uid, gid)
return True, metadata
def test_do_stat_enoent(self):
res = fs.do_stat(os.path.join('/tmp', str(random.random())))
assert res is None
