def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID): self.disk_chunk_size = disk_chunk_size device = account #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container #Absolute path for obj directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) if not os.path.exists(self.datadir + '/' + self.obj): return self.data_file = os.path.join(self.datadir, self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not self.metadata: create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not validate_object(self.metadata): create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) self.filter_metadata() if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False)
def _create_dir_object(self, dir_path): #TODO: if object already exists??? if os.path.exists(dir_path) and not os.path.isdir(dir_path): self.logger.error("Deleting file %s", dir_path) do_unlink(dir_path) #If dir aleady exist just override metadata. mkdirs(dir_path) do_chown(dir_path, self.uid, self.gid) create_object_metadata(dir_path)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise else: stats = do_fstat(fd) if not stats: return self._is_dir = stat.S_ISDIR(stats.st_mode) obj_size = stats.st_size self._metadata = read_metadata(fd) if not validate_object(self._metadata): create_object_metadata(fd) self._metadata = read_metadata(fd) assert self._metadata is not None self._filter_metadata() if self._is_dir: do_close(fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._fd = fd self._obj_size = obj_size return self
def list_objects_iter(self): self.update_object_count() objects, object_count, bytes_used = self.object_info if objects: objects.sort() container_list = [] if objects: for obj in objects: list_item = [] list_item.append(obj) obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): metadata = create_object_metadata(obj_path) if metadata: # list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) # list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' self.update_object_count() objects, object_count, bytes_used = self.object_info if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) container_list = [] if objects: for obj in objects: list_item = [] list_item.append(obj) obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): metadata = create_object_metadata(obj_path) if metadata: # list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) # list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' self.update_object_count() objects, object_count, bytes_used = self.object_info if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) container_list = [] if objects: for obj in objects: list_item = [] list_item.append(obj) obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): metadata = create_object_metadata(obj_path) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def test_create_object_metadata_file(self): tf = tempfile.NamedTemporaryFile() tf.file.write('4567'); tf.file.flush() r_md = utils.create_object_metadata(tf.name) xkey = _xkey(tf.name, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.FILE assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name)) assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self): td = tempfile.mkdtemp() try: r_md = utils.create_object_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['set'] == 1 md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)
def test_create_object_metadata_dir(self): td = tempfile.mkdtemp() try: r_md = utils.create_object_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)
def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileNotExist: if the object does not exist :raises DiskFileExpired: if the object has expired :returns: itself for use as a context manager """ # Writes are always performed to a temporary file try: self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC) except GlusterFileSystemOSError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): # If the file does exist, or some part of the path does not # exist, raise the expected DiskFileNotExist raise DiskFileNotExist raise try: if not self._stat: self._stat = do_fstat(self._fd) obj_size = self._stat.st_size if not self._metadata: self._metadata = read_metadata(self._fd) if not validate_object(self._metadata, self._stat): self._metadata = create_object_metadata(self._fd, self._stat, self._metadata) assert self._metadata is not None self._filter_metadata() if stat.S_ISDIR(self._stat.st_mode): do_close(self._fd) obj_size = 0 self._fd = -1 else: if self._is_object_expired(self._metadata): raise DiskFileExpired(metadata=self._metadata) self._obj_size = obj_size except (OSError, IOError, DiskFileExpired) as err: # Something went wrong. Context manager will not call # __exit__. So we close the fd manually here. self._close_fd() if hasattr(err, 'errno') and \ err.errno in (errno.ENOENT, errno.ESTALE): # Handle races: ENOENT/ESTALE can be raised by read_metadata() # call in GlusterFS if file gets deleted by another # client after do_open() succeeds logging.warn("open(%s) succeeded but one of the subsequent " "syscalls failed with ENOENT/ESTALE. Raising " "DiskFileNotExist." % (self._data_file)) raise DiskFileNotExist else: # Re-raise the original exception after fd has been closed raise self._disk_file_open = True return self
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): self.disk_chunk_size = disk_chunk_size self.iter_hook = iter_hook obj = obj.strip(os.path.sep) if os.path.sep in obj: self._obj_path, self._obj = os.path.split(obj) else: self._obj_path = '' self._obj = obj if self._obj_path: self.name = os.path.join(container, self._obj_path) else: self.name = container # Absolute path for object directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self._container_path = os.path.join(path, device, container) if _use_put_mount: self.put_datadir = os.path.join(self.device_path + '_PUT', self.name) else: self.put_datadir = self.datadir self._is_dir = False self.tmppath = None self.logger = logger self.metadata = {} self.meta_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.uid = int(uid) self.gid = int(gid) self.suppress_file_closing = False # Don't store a value for data_file until we know it exists. self.data_file = None data_file = os.path.join(self.put_datadir, self._obj) try: stats = do_stat(data_file) except OSError as err: if err.errno == errno.ENOTDIR: return else: if not stats: return self.data_file = data_file self._is_dir = stat.S_ISDIR(stats.st_mode) self.metadata = read_metadata(data_file) if not self.metadata: create_object_metadata(data_file) self.metadata = read_metadata(data_file) if not validate_object(self.metadata): create_object_metadata(data_file) self.metadata = read_metadata(data_file) self.filter_metadata() if not self._is_dir and keep_data_fp: # The caller has an assumption that the "fp" field of this # object is an file object if keep_data_fp is set. However, # this implementation of the DiskFile object does not need to # open the file for internal operations. So if the caller # requests it, we'll just open the file for them. self.fp = do_open(data_file, 'rb')
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None): """ Returns tuple of name, created_at, size, content_type, etag. """ assert limit >= 0 assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254) if path is not None: if path: prefix = path = path.rstrip('/') + '/' else: prefix = path delimiter = '/' elif delimiter and not prefix: prefix = '' container_list = [] objects = self._update_object_count() if objects: objects.sort() else: return container_list if end_marker: objects = filter_end_marker(objects, end_marker) if marker and marker >= prefix: objects = filter_marker(objects, marker) elif prefix: objects = filter_prefix_as_marker(objects, prefix) if prefix is None: # No prefix, we don't need to apply the other arguments, we just # return what we have. pass else: # We have a non-None (for all intents and purposes it is a string) # prefix. if not delimiter: if not prefix: # We have nothing more to do pass else: objects = filter_prefix(objects, prefix) else: objects = filter_delimiter(objects, delimiter, prefix, marker, path) count = 0 for obj in objects: obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): if delimiter == '/' and obj_path[-1] == delimiter: clean_obj_path = obj_path[:-1] else: clean_obj_path = obj_path try: metadata = create_object_metadata(clean_obj_path) except OSError as e: # FIXME - total hack to get upstream swift ported unit # test cases working for now. if e.errno != errno.ENOENT: raise if Glusterfs.OBJECT_ONLY and metadata \ and metadata[X_CONTENT_TYPE] == DIR_TYPE \ and not dir_is_object(metadata): continue list_item = [] list_item.append(obj) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) count += 1 if count >= limit: break return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None): """ Returns tuple of name, created_at, size, content_type, etag. """ assert limit >= 0 assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254) if path is not None: if path: prefix = path = path.rstrip('/') + '/' else: prefix = path delimiter = '/' elif delimiter and not prefix: prefix = '' container_list = [] objects = self._update_object_count() if objects: objects.sort() else: return container_list if end_marker: objects = filter_end_marker(objects, end_marker) if marker and marker >= prefix: objects = filter_marker(objects, marker) elif prefix: objects = filter_prefix_as_marker(objects, prefix) if prefix is None: # No prefix, we don't need to apply the other arguments, we just # return what we have. pass else: # We have a non-None (for all intents and purposes it is a string) # prefix. if not delimiter: if not prefix: # We have nothing more to do pass else: objects = filter_prefix(objects, prefix) else: objects = filter_delimiter(objects, delimiter, prefix, marker, path) count = 0 for obj in objects: obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): if delimiter == '/' and obj_path[-1] == delimiter: clean_obj_path = obj_path[:-1] else: clean_obj_path = obj_path try: metadata = create_object_metadata(clean_obj_path) except OSError as e: # FIXME - total hack to get upstream swift ported unit # test cases working for now. if e.errno != errno.ENOENT: raise if not Glusterfs._implicit_dir_objects and metadata \ and metadata[X_CONTENT_TYPE] == DIR_TYPE \ and not dir_is_object(metadata): continue list_item = [] list_item.append(obj) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) count += 1 if count >= limit: break return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None, storage_policy_index=0, out_content_type=None, reverse=False): """ Returns tuple of name, created_at, size, content_type, etag. """ assert limit >= 0 assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254) if path is not None: if path: prefix = path = path.rstrip('/') + '/' else: prefix = path delimiter = '/' elif delimiter and not prefix: prefix = '' container_list = [] if self.account == 'gsexpiring': objects = list_objects_gsexpiring_container(self.datadir) else: objects = self._update_object_count() if objects: objects.sort() else: # No objects in container , return empty list return container_list if marker and end_marker and reverse: marker, end_marker = end_marker, marker if end_marker: objects = filter_end_marker(objects, end_marker) if marker and marker >= prefix: objects = filter_marker(objects, marker) elif prefix: objects = filter_prefix_as_marker(objects, prefix) if prefix is None: # No prefix, we don't need to apply the other arguments, we just # return what we have. pass else: # We have a non-None (for all intents and purposes it is a string) # prefix. if not delimiter: if not prefix: # We have nothing more to do pass else: objects = filter_prefix(objects, prefix) else: objects = filter_delimiter(objects, delimiter, prefix, marker, path) if out_content_type == 'text/plain' or \ self.account == 'gsexpiring': # When out_content_type == 'text/plain': # # The client is only asking for a plain list of objects and NOT # asking for any extended information about objects such as # bytes used or etag. # # When self.account == 'gsexpiring': # # This is a JSON request sent by the object expirer to list # tracker objects in a container in gsexpiring volume. # When out_content_type is 'application/json', the caller # expects each record entry to have the following ordered # fields: (name, timestamp, size, content_type, etag) for obj in objects: container_list.append((obj, '0', 0, 'text/plain', '')) if len(container_list) >= limit: break if reverse: container_list.reverse() return container_list count = 0 for obj in objects: obj_path = os.path.join(self.datadir, obj) try: metadata = read_metadata(obj_path) except GlusterFileSystemIOError as err: if err.errno in (errno.ENOENT, errno.ESTALE): # obj might have been deleted by another process # since the objects list was originally built continue else: raise err if not metadata or not validate_object(metadata): if delimiter == '/' and obj_path[-1] == delimiter: clean_obj_path = obj_path[:-1] else: clean_obj_path = obj_path try: metadata = create_object_metadata(clean_obj_path) except OSError as e: # FIXME - total hack to get upstream swift ported unit # test cases working for now. if e.errno not in (errno.ENOENT, errno.ESTALE): raise if not Glusterfs._implicit_dir_objects and metadata \ and metadata[X_CONTENT_TYPE] == DIR_TYPE \ and not dir_is_object(metadata): continue list_item = [] list_item.append(obj) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) count += 1 if count >= limit: break if reverse: container_list.reverse() return container_list
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): self.disk_chunk_size = disk_chunk_size self.iter_hook = iter_hook # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/, # etc. obj = obj.strip(os.path.sep) if os.path.sep in obj: self._obj_path, self._obj = os.path.split(obj) else: self._obj_path = '' self._obj = obj if self._obj_path: self.name = os.path.join(container, self._obj_path) else: self.name = container # Absolute path for object directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self._container_path = os.path.join(path, device, container) self._is_dir = False self.tmppath = None self.logger = logger self.metadata = {} self.meta_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.uid = int(uid) self.gid = int(gid) # Don't store a value for data_file until we know it exists. self.data_file = None data_file = os.path.join(self.datadir, self._obj) if not os.path.exists(data_file): return self.data_file = os.path.join(data_file) self.metadata = read_metadata(data_file) if not self.metadata: create_object_metadata(data_file) self.metadata = read_metadata(data_file) if not validate_object(self.metadata): create_object_metadata(data_file) self.metadata = read_metadata(data_file) self.filter_metadata() if os.path.isdir(data_file): self._is_dir = True else: if keep_data_fp: # The caller has an assumption that the "fp" field of this # object is an file object if keep_data_fp is set. However, # this implementation of the DiskFile object does not need to # open the file for internal operations. So if the caller # requests it, we'll just open the file for them. self.fp = do_open(data_file, 'rb')