def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path self.container = container self.datadir = path_std(os.path.join(path, drive, self.container)) self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) self.metauuid = 'ff89f933b2ca8df40' self.fhr_path = parent_path(self.datadir) self.metafile = self.fhr_path+ '/' + self.metauuid+'/' + self.datadir.split('/')[-1] self.meta_fhr_path = parent_path(self.metafile) if self.meta_fhr_dir_is_deleted(): self.create_dir_object(self.meta_fhr_path) if not os.path.exists(self.datadir): return if self.dir_exists: try: self.metadata = meta_read_metadata(self.metafile) except EOFError: meta_create_container_metadata(self.datadir,self.metafile) else: return if self.container: if not self.metadata: meta_create_container_metadata(self.datadir,self.metafile) self.metadata = meta_read_metadata(self.metafile) else: if not validate_container(self.metadata): meta_create_container_metadata(self.datadir,self.metafile) self.metadata = meta_read_metadata(self.metafile)
def get_tree_info(datapath,metauuid): objects = os.listdir(datapath) container_list = [] for obj in objects: if 'ff89f933b2ca8df40' == obj: continue list_item = {} obj_path = path_std(os.path.join(datapath, obj)) obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),metauuid,obj_path.split('/')[-1]) metadata = meta_read_metadata(obj_meta_path) if not metadata or not validate_object(metadata): metadata = meta_create_object_metadata(obj_path,obj_meta_path) if metadata: list_item.update({'name':obj}) list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])}) list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])}) list_item.update({'md5':metadata[X_ETAG]}) list_item.update({'ftype':metadata[X_FILE_TYPE]}) if metadata.get('metadata'): list_item.update({'metadata':metadata['metadata']}) if os.path.isdir(obj_path): cList = get_tree_info(obj_path,metauuid) list_item.update({'list':cList}) container_list.append(list_item) return container_list
def non_recursive_iter(self): objects = os.listdir(self.datadir) if 'ff89f933b2ca8df40' in objects: objects.remove('ff89f933b2ca8df40') if objects: objects.sort() container_list = [] if objects: for obj in objects: list_item = {} obj_path = path_std(os.path.join(self.datadir, obj)) obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),self.metauuid,obj_path.split('/')[-1]) metadata = meta_read_metadata(obj_meta_path) if not metadata or not validate_object(metadata): metadata = meta_create_object_metadata(obj_path,obj_meta_path) if metadata: list_item.update({'name':obj}) list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])}) list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])}) list_item.update({'md5':metadata[X_ETAG]}) list_item.update({'ftype':metadata[X_FILE_TYPE]}) if metadata.has_key('metadata'): list_item.update({'metadata':metadata['metadata']}) container_list.append(list_item) return container_list
def __init__(self, path, drive, account, container,direr, logger, uid=DEFAULT_UID, gid=DEFAULT_GID,recycle_uuid=''): self.root = path self.container = container self.datadir = os.path.join(path, drive,container,direr) self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) self.fhr_path = parent_path(self.datadir) if 'recycle' == container: self.recycle_uuid = recycle_uuid self.metafile = os.path.join(path, drive,container +'/'+'meta',self.recycle_uuid) self.meta_fhr_path = parent_path(self.metafile) if self.meta_fhr_dir_is_deleted(): # self.create_dir_object(self.meta_fhr_path) self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid if not os.path.exists(self.datadir): return self.metadata = meta_read_metadata(self.metafile)
def prefix_list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' self.update_object_count() objects, object_count, bytes_used = self.object_info if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) container_list = [] if objects: for obj in objects: list_item = {} obj_path = path_std(os.path.join(self.datadir, obj)) obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),self.metauuid,obj_path.split('/')[-1]) metadata = meta_read_metadata(obj_meta_path) if not metadata or not validate_object(metadata): metadata = meta_create_object_metadata(obj_path,obj_meta_path) if metadata: list_item.update({'name':obj}) list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])}) list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])}) list_item.update({'md5':metadata[X_ETAG]}) list_item.update({'ftype':metadata[X_FILE_TYPE]}) if metadata.has_key('metadata'): list_item.update({'metadata':metadata['metadata']}) container_list.append(list_item) return container_list
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ Return tuple of name, object_count, bytes_used, 0(is_subdir). Used by account server. """ if delimiter and not prefix: prefix = '' self.update_container_count() containers, container_count = self.container_info if containers: containers.sort() if containers and prefix: containers = self.filter_prefix(containers, prefix) if containers and delimiter: containers = self.filter_delimiter(containers, delimiter, prefix) if containers and marker: containers = self.filter_marker(containers, marker) if containers and end_marker: containers = self.filter_end_marker(containers, end_marker) if containers and limit: if len(containers) > limit: containers = self.filter_limit(containers, limit) account_list = [] if containers: for cont in containers: list_item = [] metadata = None list_item.append(cont) cont_path = path_std(os.path.join(self.datadir, cont)) cont_meta_path = '/'.join(cont_path.split('/')[:-1])+ '/' + self.metauuid+ '/' + cont_path.split('/')[-1] metadata = meta_read_metadata(cont_meta_path) if not metadata or not validate_container(metadata): metadata = meta_create_container_metadata(cont_path,cont_meta_path) if metadata: list_item.append(metadata[X_OBJECTS_COUNT]) list_item.append(metadata[X_BYTES_USED]) list_item.append(0) account_list.append(list_item) return account_list
def list_objects_meta_iter(self,start,limit): self.update_object_count() objects, object_count, bytes_used = self.object_info if objects: objects.sort() container_list = [] if objects: for obj in objects: list_item = [] list_item.append(obj) obj_path = os.path.join(self.datadir, obj) metadata = meta_read_metadata(obj_path) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_ETAG]) list_item.append(metadata[X_FILE_TYPE]) if 'recycle' == self.container: list_item.append(metadata['user_path']) list_item.append(metadata['recycle_uuid']) list_item.append(metadata['ftype']) container_list.append(list_item) if 'recycle' == self.container: container_list.sort(key=lambda x:float(x[1])) container_list.reverse() if start: start = int(start) if limit: limit = int(limit) return container_list[start:start+limit] else: return container_list[start:] return container_list
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None,recycle_uuid=''): self.iter_hook = iter_hook self.disk_chunk_size = disk_chunk_size obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) self.data_file = os.path.join(self.datadir, self.obj) self.fhr_path = parent_path(self.data_file) self.container = container if 'recycle' == container: self.recycle_uuid = recycle_uuid self.metafile = os.path.join(self.datadir[:-5]+'/'+'meta',self.recycle_uuid) self.meta_fhr_path = parent_path(self.metafile) if self.meta_fhr_dir_is_deleted(): #self.create_dir_object(self.meta_fhr_path) self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid if not os.path.exists(self.datadir + '/' + self.obj): return if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False) self.metadata = meta_read_metadata(self.metafile) if not self.metadata: meta_create_object_metadata(self.data_file,self.metafile) self.metadata = meta_read_metadata(self.metafile) if not validate_object(self.metadata): meta_create_object_metadata(self.data_file,self.metafile) self.metadata = meta_read_metadata(self.metafile) self.filter_metadata()
def get_container_timestamp(self, container): cont_path = path_std(os.path.join(self.datadir, container)) cont_meta_path = '/'.join(cont_path.split('/')[:-1])+ '/' + self.metauuid+ '/' + cont_path.split('/')[-1] metadata = meta_read_metadata(cont_meta_path) return int(metadata.get(X_PUT_TIMESTAMP, '0')) or None
def __init__( self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None, ): self.iter_hook = iter_hook self.disk_chunk_size = disk_chunk_size obj = obj.strip("/") if "/" in obj: self.obj_path, self.obj = obj.rsplit("/", 1) else: self.obj_path = "" self.obj = obj if self.obj_path: self.name = "/".join((container, self.obj_path)) else: self.name = container # Absolute path for obj directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.cntpath = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, "tmp") self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) self.data_file = path_std(os.path.join(self.datadir, self.obj)) self.fhr_path = parent_path(self.data_file) self.container = container self.metauuid = "ff89f933b2ca8df40" self.metafile = self.fhr_path + "/" + self.metauuid + "/" + self.data_file.split("/")[-1] self.meta_fhr_path = parent_path(self.metafile) self.cnt_flag = self.cnt_exists() if not self.cnt_flag: return if self.meta_fhr_dir_is_deleted(): # self.create_dir_object(self.meta_fhr_path) self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + "/" + self.metauuid if not os.path.exists(self.datadir + "/" + self.obj): return if os.path.isdir(self.datadir + "/" + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, "rb") if not keep_data_fp: self.close(verify_file=False) self.metadata = meta_read_metadata(self.metafile) if not self.metadata: meta_create_object_metadata(self.data_file, self.metafile) self.metadata = meta_read_metadata(self.metafile) if not validate_object(self.metadata): meta_create_object_metadata(self.data_file, self.metafile) self.metadata = meta_read_metadata(self.data_file) self.filter_metadata()