def test_do_open_err_int_mode(self):
try:
fs.do_open(os.path.join('/tmp', str(random.random())),
os.O_RDONLY)
except SwiftOnFileSystemOSError:
pass
else:
self.fail("SwiftOnFileSystemOSError expected")
def get_etag(path_or_fd):
"""
FIXME: It would be great to have a translator that returns the md5sum() of
the file as an xattr that can be simply fetched.
Since we don't have that we should yield after each chunk read and
computed so that we don't consume the worker thread.
"""
etag = ''
if isinstance(path_or_fd, int):
# We are given a file descriptor, so this is an invocation from the
# DiskFile.open() method.
fd = path_or_fd
dup_fd = do_dup(fd)
try:
etag = _read_for_etag(dup_fd)
do_lseek(fd, 0, os.SEEK_SET)
finally:
do_close(dup_fd)
else:
# We are given a path to the object when the DiskDir.list_objects_iter
# method invokes us.
path = path_or_fd
fd = do_open(path, os.O_RDONLY)
try:
etag = _read_for_etag(fd)
finally:
do_close(fd)
return etag
def test_do_open(self):
_fd, tmpfile = mkstemp()
try:
fd = fs.do_open(tmpfile, os.O_RDONLY)
try:
os.write(fd, 'test')
except OSError:
pass
else:
self.fail("OSError expected")
finally:
os.close(fd)
finally:
os.close(_fd)
os.remove(tmpfile)
def get_etag(path):
"""
FIXME: It would be great to have a translator that returns the md5sum() of
the file as an xattr that can be simply fetched.
Since we don't have that we should yield after each chunk read and
computed so that we don't consume the worker thread.
"""
# FIXME: really do need to grab precomputed checksum instead
fd = do_open(path, os.O_RDONLY)
try:
etag = _read_for_etag(fd)
finally:
do_close(fd)
return etag
def create(self, hpss_hints):
"""
Context manager to create a file. We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
For Gluster, we first optimistically create the temporary file using
the "rsync-friendly" .NAME.random naming. If we find that some path to
the file does not exist, we then create that path and then create the
temporary file again. If we get file name conflict, we'll retry using
different random suffixes 1,000 times before giving up.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
:param hpss_hints: dict containing HPSS class of service hints
:raises DiskFileNoSpace: if a size is specified and allocation fails
:raises AlreadyExistsAsFile: if path or part of a path is not a \
directory
"""
# Create /account/container directory structure on mount point root
try:
self._logger.debug("DiskFile: Creating directories for %s" %
self._container_path)
hpss_utils.makedirs(self._container_path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
# Create directories to object name, if they don't exist.
self._logger.debug("DiskFile: creating directories in obj name %s" %
self._obj)
object_dirs = os.path.split(self._obj)[0]
self._logger.debug("object_dirs: %s" % repr(object_dirs))
self._logger.debug("data_dir: %s" % self._put_datadir)
hpss_utils.makedirs(os.path.join(self._put_datadir, object_dirs))
data_file = os.path.join(self._put_datadir, self._obj)
# Create the temporary file
attempts = 1
while True:
tmpfile = '.%s.%s.temporary' % (self._obj,
random.randint(0, 65536))
tmppath = os.path.join(self._put_datadir, tmpfile)
self._logger.debug("DiskFile: Creating temporary file %s" %
tmppath)
try:
# TODO: figure out how to create hints struct
self._logger.debug("DiskFile: creating file")
fd = do_open(tmppath,
hpss.O_WRONLY | hpss.O_CREAT | hpss.O_EXCL)
self._logger.debug("DiskFile: setting COS")
if hpss_hints['cos']:
hpss_utils.set_cos(tmppath, int(hpss_hints['cos']))
except SwiftOnFileSystemOSError as gerr:
if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
# Raise DiskFileNoSpace to be handled by upper layers when
# there is no space on disk OR when quota is exceeded
self._logger.error("DiskFile: no space")
raise DiskFileNoSpace()
if gerr.errno == errno.EACCES:
self._logger.error("DiskFile: permission denied")
raise DiskFileNoSpace()
if gerr.errno == errno.ENOTDIR:
self._logger.error("DiskFile: not a directory")
raise AlreadyExistsAsFile('do_open(): failed on %s,'
' path or part of a'
' path is not a directory'
% data_file)
if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
# FIXME: Other cases we should handle?
self._logger.error("DiskFile: unknown error %s"
% gerr.errno)
raise
if attempts >= MAX_OPEN_ATTEMPTS:
# We failed after N attempts to create the temporary
# file.
raise DiskFileError('DiskFile.mkstemp(): failed to'
' successfully create a temporary file'
' without running into a name conflict'
' after %d of %d attempts for: %s' % (
attempts, MAX_OPEN_ATTEMPTS,
data_file))
if gerr.errno == errno.EEXIST:
self._logger.debug("DiskFile: file exists already")
# Retry with a different random number.
attempts += 1
else:
break
dw = None
self._logger.debug("DiskFile: created file")
try:
# Ensure it is properly owned before we make it available.
do_chown(tmppath, self._uid, self._gid)
dw = DiskFileWriter(fd, tmppath, self)
yield dw
finally:
if dw:
dw.close()
if dw._tmppath:
do_unlink(dw._tmppath)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._logger.debug("DiskFile: Opening %s" % self._data_file)
self._stat = do_stat(self._data_file)
self._is_dir = stat.S_ISDIR(self._stat.st_mode)
if not self._is_dir:
self._fd = do_open(self._data_file, hpss.O_RDONLY)
self._obj_size = self._stat.st_size
else:
self._fd = -1
self._obj_size = 0
except SwiftOnFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
self._logger.debug("DiskFile: Reading metadata")
self._metadata = read_metadata(self._data_file)
if not self._validate_object_metadata():
self._create_object_metadata(self._data_file)
assert self._metadata is not None
self._filter_metadata()
if not self._is_dir:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and err.errno == errno.ENOENT:
# Handle races: ENOENT can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
return self
