def test_do_open_err_int_mode(self): try: fs.do_open(os.path.join('/tmp', str(random.random())), os.O_RDONLY) except GlusterFileSystemOSError: pass else: self.fail("GlusterFileSystemOSError expected")
def _get_etag(path_or_fd): """ FIXME: It would be great to have a translator that returns the md5sum() of the file as an xattr that can be simply fetched. Since we don't have that we should yield after each chunk read and computed so that we don't consume the worker thread. """ etag = '' if isinstance(path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. fd = path_or_fd dup_fd = do_dup(fd) try: etag = _read_for_etag(dup_fd) do_lseek(fd, 0, os.SEEK_SET) finally: do_close(dup_fd) else: # We are given a path to the object when the DiskDir.list_objects_iter # method invokes us. path = path_or_fd fd = do_open(path, os.O_RDONLY) try: etag = _read_for_etag(fd) finally: do_close(fd) return etag
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def test_do_open(self): _fd, tmpfile = mkstemp() try: fd = fs.do_open(tmpfile, os.O_RDONLY) try: os.write(fd, 'test') except OSError: pass else: self.fail("OSError expected") finally: os.close(fd) finally: os.close(_fd) os.remove(tmpfile)
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: os.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. attempts = 1 cur_thread = str(getcurrent()) while True: postfix = md5(self._obj + _cur_host + _cur_pid + cur_thread + str(random.random())).hexdigest() tmpfile = '.' + self._obj + '.' + postfix tmppath = os.path.join(self._put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % (tmppath)) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError('DiskFile.mkstemp(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % ( attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: # Retry with a different random number. attempts += 1 elif gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif not self._obj_path: # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif attempts > 1: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs" % gerr) attempts += 1 else: # It looks like the path to the object does not already # exist; don't count this as an attempt, though, since # we perform the open() system call optimistically. self._create_dir_object(self._obj_path) else: break dw = None try: # Ensure it is properly owned before we make it available. do_fchown(fd, self._uid, self._gid) # NOTE: we do not perform the fallocate() call at all. We ignore # it completely since at the time of this writing FUSE does not # support it. dw = DiskFileWriter(fd, tmppath, self) yield dw finally: dw.close() if dw._tmppath: do_unlink(dw._tmppath)
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: os.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. attempts = 1 while True: # To know more about why following temp file naming convention is # used, please read this GlusterFS doc: # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa tmpfile = '.' + self._obj + '.' + uuid4().hex tmppath = os.path.join(self._put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % (tmppath)) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError( 'DiskFile.mkstemp(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % (attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: # Retry with a different random number. attempts += 1 elif gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn( "DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif not self._obj_path: # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. _random_sleep() logging.warn( "DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif attempts > 1: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.mkstemp(): %s ... retrying in" " 0.1 secs" % gerr) attempts += 1 else: # It looks like the path to the object does not already # exist; don't count this as an attempt, though, since # we perform the open() system call optimistically. self._create_dir_object(self._obj_path) else: break dw = None try: # Ensure it is properly owned before we make it available. do_fchown(fd, self._uid, self._gid) # NOTE: we do not perform the fallocate() call at all. We ignore # it completely since at the time of this writing FUSE does not # support it. dw = DiskFileWriter(fd, tmppath, self) yield dw finally: dw.close() if dw._tmppath: do_unlink(dw._tmppath)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._stat = do_fstat(self._fd) self._is_dir = stat.S_ISDIR(self._stat.st_mode) obj_size = self._stat.st_size self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): create_object_metadata(self._fd) self._metadata = read_metadata(self._fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._stat = do_fstat(self._fd) self._is_dir = stat.S_ISDIR(self._stat.st_mode) obj_size = self._stat.st_size self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): self._metadata = create_object_metadata(self._fd, self._stat, self._metadata) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: os.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. fd = None attempts = 1 while True: # To know more about why following temp file naming convention is # used, please read this GlusterFS doc: # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa tmpfile = '.' + self._obj + '.' + uuid4().hex tmppath = os.path.join(self._put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % (tmppath)) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError( 'DiskFile.create(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % (attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: # Retry with a different random number. attempts += 1 elif gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn( "DiskFile.create(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif not self._obj_path: # ENOENT # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. # Handle race: # This can be the issue when memcache has cached that the # container exists. If someone removes the container dir # from filesystem, it's not reflected in memcache. So # swift reports that the container exists and this code # tries to create a file in a directory that does not # exist. However, it's wrong to create the container here. _random_sleep() logging.warn( "DiskFile.create(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 if attempts > 2: # Ideally we would want to return 404 indicating that # the container itself does not exist. Can't be done # though as the caller won't catch DiskFileNotExist. # We raise an exception with a meaningful name for # correctness. logging.warn("Container dir %s does not exist", self._container_path) raise DiskFileContainerDoesNotExist elif attempts > 1: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.create(): %s ... retrying in" " 0.1 secs" % gerr) attempts += 1 else: # It looks like the path to the object does not already # exist; don't count this as an attempt, though, since # we perform the open() system call optimistically. self._create_dir_object(self._obj_path) else: break dw = None try: if size is not None and size > 0: try: fallocate(fd, size) except OSError as err: if err.errno in (errno.ENOSPC, errno.EDQUOT): raise DiskFileNoSpace() raise # Ensure it is properly owned before we make it available. if not ((self._uid == DEFAULT_UID) and (self._gid == DEFAULT_GID)): # If both UID and GID is -1 (default values), it has no effect. # So don't do a fchown. # Further, at the time of this writing, UID and GID information # is not passed to DiskFile. do_fchown(fd, self._uid, self._gid) dw = DiskFileWriter(fd, tmppath, self) # It's now the responsibility of DiskFileWriter to close this fd. fd = None yield dw finally: if dw: dw.close() if dw._tmppath: do_unlink(dw._tmppath)
def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: os.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and # construct the proper name for the temporary file. fd = None attempts = 1 while True: # To know more about why following temp file naming convention is # used, please read this GlusterFS doc: # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations # noqa tmpfile = '.' + self._obj + '.' + uuid4().hex tmppath = os.path.join(self._put_datadir, tmpfile) try: fd = do_open(tmppath, os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % (tmppath)) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError('DiskFile.create(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % ( attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: # Retry with a different random number. attempts += 1 elif gerr.errno == errno.EIO: # FIXME: Possible FUSE issue or race condition, let's # sleep on it and retry the operation. _random_sleep() logging.warn("DiskFile.create(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 elif not self._obj_path: # ENOENT # No directory hierarchy and the create failed telling us # the container or volume directory does not exist. This # could be a FUSE issue or some race condition, so let's # sleep a bit and retry. # Handle race: # This can be the issue when memcache has cached that the # container exists. If someone removes the container dir # from filesystem, it's not reflected in memcache. So # swift reports that the container exists and this code # tries to create a file in a directory that does not # exist. However, it's wrong to create the container here. _random_sleep() logging.warn("DiskFile.create(): %s ... retrying in" " 0.1 secs", gerr) attempts += 1 if attempts > 2: # Ideally we would want to return 404 indicating that # the container itself does not exist. Can't be done # though as the caller won't catch DiskFileNotExist. # We raise an exception with a meaningful name for # correctness. logging.warn("Container dir %s does not exist", self._container_path) raise DiskFileContainerDoesNotExist elif attempts > 1: # Got ENOENT after previously making the path. This could # also be a FUSE issue or some race condition, nap and # retry. _random_sleep() logging.warn("DiskFile.create(): %s ... retrying in" " 0.1 secs" % gerr) attempts += 1 else: # It looks like the path to the object does not already # exist; don't count this as an attempt, though, since # we perform the open() system call optimistically. self._create_dir_object(self._obj_path) else: break dw = None try: if size is not None and size > 0: try: fallocate(fd, size) except OSError as err: if err.errno in (errno.ENOSPC, errno.EDQUOT): raise DiskFileNoSpace() raise # Ensure it is properly owned before we make it available. if not ((self._uid == DEFAULT_UID) and (self._gid == DEFAULT_GID)): # If both UID and GID is -1 (default values), it has no effect. # So don't do a fchown. # Further, at the time of this writing, UID and GID information # is not passed to DiskFile. do_fchown(fd, self._uid, self._gid) dw = DiskFileWriter(fd, tmppath, self) # It's now the responsibility of DiskFileWriter to close this fd. fd = None yield dw finally: if dw: dw.close() if dw._tmppath: do_unlink(dw._tmppath)