def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def test_create_object_metadata_file(self): tf = tempfile.NamedTemporaryFile() tf.file.write('4567') tf.file.flush() r_md = utils.create_object_metadata(tf.name) xkey = _xkey(tf.name, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['set'] == 1 md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.FILE assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name)) assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_file(self): tf = tempfile.NamedTemporaryFile() tf.file.write('4567') tf.file.flush() r_md = utils.create_object_metadata(tf.name) xkey = _xkey(tf.name, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.FILE assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name)) assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self): td = tempfile.mkdtemp() try: r_md = utils.create_object_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['set'] == 1 md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)
def test_create_object_metadata_dir(self): td = tempfile.mkdtemp() try: r_md = utils.create_object_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._stat = do_fstat(self._fd) self._is_dir = stat.S_ISDIR(self._stat.st_mode) obj_size = self._stat.st_size self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): create_object_metadata(self._fd) self._metadata = read_metadata(self._fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except SwiftOnFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: self._stat = do_fstat(self._fd) self._is_dir = stat.S_ISDIR(self._stat.st_mode) obj_size = self._stat.st_size self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): self._metadata = create_object_metadata(self._fd, self._stat, self._metadata) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and err.errno == errno.ENOENT: # Handle races: ENOENT can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise return self