def test_do_open_err_int_mode(self): try: fs.do_open(os.path.join('/tmp', str(random.random())), os.O_RDONLY) except SwiftOnFileSystemOSError: pass else: self.fail("SwiftOnFileSystemOSError expected")
def get_etag(path_or_fd): """ FIXME: It would be great to have a translator that returns the md5sum() of the file as an xattr that can be simply fetched. Since we don't have that we should yield after each chunk read and computed so that we don't consume the worker thread. """ etag = '' if isinstance(path_or_fd, int): # We are given a file descriptor, so this is an invocation from the # DiskFile.open() method. fd = path_or_fd dup_fd = do_dup(fd) try: etag = _read_for_etag(dup_fd) do_lseek(fd, 0, os.SEEK_SET) finally: do_close(dup_fd) else: # We are given a path to the object when the DiskDir.list_objects_iter # method invokes us. path = path_or_fd fd = do_open(path, os.O_RDONLY) try: etag = _read_for_etag(fd) finally: do_close(fd) return etag
def test_do_open(self): _fd, tmpfile = mkstemp() try: fd = fs.do_open(tmpfile, os.O_RDONLY) try: os.write(fd, 'test') except OSError: pass else: self.fail("OSError expected") finally: os.close(fd) finally: os.close(_fd) os.remove(tmpfile)
def get_etag(path): """ FIXME: It would be great to have a translator that returns the md5sum() of the file as an xattr that can be simply fetched. Since we don't have that we should yield after each chunk read and computed so that we don't consume the worker thread. """ # FIXME: really do need to grab precomputed checksum instead fd = do_open(path, os.O_RDONLY) try: etag = _read_for_etag(fd) finally: do_close(fd) return etag
def create(self, hpss_hints): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. For Gluster, we first optimistically create the temporary file using the "rsync-friendly" .NAME.random naming. If we find that some path to the file does not exist, we then create that path and then create the temporary file again. If we get file name conflict, we'll retry using different random suffixes 1,000 times before giving up. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param hpss_hints: dict containing HPSS class of service hints :raises DiskFileNoSpace: if a size is specified and allocation fails :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ # Create /account/container directory structure on mount point root try: self._logger.debug("DiskFile: Creating directories for %s" % self._container_path) hpss_utils.makedirs(self._container_path) except OSError as err: if err.errno != errno.EEXIST: raise # Create directories to object name, if they don't exist. self._logger.debug("DiskFile: creating directories in obj name %s" % self._obj) object_dirs = os.path.split(self._obj)[0] self._logger.debug("object_dirs: %s" % repr(object_dirs)) self._logger.debug("data_dir: %s" % self._put_datadir) hpss_utils.makedirs(os.path.join(self._put_datadir, object_dirs)) data_file = os.path.join(self._put_datadir, self._obj) # Create the temporary file attempts = 1 while True: tmpfile = '.%s.%s.temporary' % (self._obj, random.randint(0, 65536)) tmppath = os.path.join(self._put_datadir, tmpfile) self._logger.debug("DiskFile: Creating temporary file %s" % tmppath) try: # TODO: figure out how to create hints struct self._logger.debug("DiskFile: creating file") fd = do_open(tmppath, hpss.O_WRONLY | hpss.O_CREAT | hpss.O_EXCL) self._logger.debug("DiskFile: setting COS") if hpss_hints['cos']: hpss_utils.set_cos(tmppath, int(hpss_hints['cos'])) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): # Raise DiskFileNoSpace to be handled by upper layers when # there is no space on disk OR when quota is exceeded self._logger.error("DiskFile: no space") raise DiskFileNoSpace() if gerr.errno == errno.EACCES: self._logger.error("DiskFile: permission denied") raise DiskFileNoSpace() if gerr.errno == errno.ENOTDIR: self._logger.error("DiskFile: not a directory") raise AlreadyExistsAsFile('do_open(): failed on %s,' ' path or part of a' ' path is not a directory' % data_file) if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO): # FIXME: Other cases we should handle? self._logger.error("DiskFile: unknown error %s" % gerr.errno) raise if attempts >= MAX_OPEN_ATTEMPTS: # We failed after N attempts to create the temporary # file. raise DiskFileError('DiskFile.mkstemp(): failed to' ' successfully create a temporary file' ' without running into a name conflict' ' after %d of %d attempts for: %s' % ( attempts, MAX_OPEN_ATTEMPTS, data_file)) if gerr.errno == errno.EEXIST: self._logger.debug("DiskFile: file exists already") # Retry with a different random number. attempts += 1 else: break dw = None self._logger.debug("DiskFile: created file") try: # Ensure it is properly owned before we make it available. do_chown(tmppath, self._uid, self._gid) dw = DiskFileWriter(fd, tmppath, self) yield dw finally: if dw: dw.close() if dw._tmppath: do_unlink(dw._tmppath)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._logger.debug("DiskFile: Opening %s" % self._data_file) self._stat = do_stat(self._data_file) self._is_dir = stat.S_ISDIR(self._stat.st_mode) if not self._is_dir: self._fd = do_open(self._data_file, hpss.O_RDONLY) self._obj_size = self._stat.st_size else: self._fd = -1 self._obj_size = 0 except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._logger.debug("DiskFile: Reading metadata") self._metadata = read_metadata(self._data_file) if not self._validate_object_metadata(): self._create_object_metadata(self._data_file) assert self._metadata is not None self._filter_metadata() if not self._is_dir: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self