def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' objects = [] object_count = 0 bytes_used = 0 container_list = [] objects, object_count, bytes_used = get_container_details(self.datadir) if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ int(self.metadata[X_BYTES_USED][0]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = (object_count, 0) self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata) if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) if objects: for obj in objects: list_item = [] list_item.append(obj) obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): metadata = create_object_metadata(obj_path) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' objects = [] object_count = 0 bytes_used = 0 container_list = [] objects, object_count, bytes_used = get_container_details(self.datadir) if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ int(self.metadata[X_BYTES_USED]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = object_count self.metadata[X_BYTES_USED] = bytes_used self.update_container(self.metadata) if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) if objects: for obj in objects: list_item = [] list_item.append(obj) metadata = read_metadata(self.datadir + '/' + obj) if not metadata or not validate_object(metadata): metadata = create_object_metadata(self.datadir + '/' + obj) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def __init__( self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, ): self.disk_chunk_size = disk_chunk_size device = account # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc obj = obj.strip("/") if "/" in obj: self.obj_path, self.obj = obj.rsplit("/", 1) else: self.obj_path = "" self.obj = obj if self.obj_path: self.name = "/".join((container, self.obj_path)) else: self.name = container # Absolute path for obj directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) if not check_mount(path, device): check_valid_account(account) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, "tmp") self.logger = logger self.metadata = {} self.meta_file = None self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) if not os.path.exists(self.datadir + "/" + self.obj): return self.data_file = os.path.join(self.datadir, self.obj) self.metadata = read_metadata(self.datadir + "/" + self.obj) if not self.metadata: create_object_metadata(self.datadir + "/" + self.obj) self.metadata = read_metadata(self.datadir + "/" + self.obj) if not validate_object(self.metadata): create_object_metadata(self.datadir + "/" + self.obj) self.metadata = read_metadata(self.datadir + "/" + self.obj) self.filter_metadata() if os.path.isdir(self.datadir + "/" + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, "rb") if not keep_data_fp: self.close(verify_file=False)
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object=None): self.disk_chunk_size = disk_chunk_size device = account #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container #Absolute path for obj directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) if not check_mount(path, device): check_valid_account(account, fs_object) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.meta_file = None self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) if not os.path.exists(self.datadir + '/' + self.obj): return self.data_file = os.path.join(self.datadir, self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not self.metadata: create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not validate_object(self.metadata): create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) self.filter_metadata() if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False)