def __init__(self, path, drive, account, container,direr, logger, uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path self.container = container self.datadir = os.path.join(path, drive,container,direr) self.cntpath = os.path.join(path, drive,container) self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) self.fhr_path = parent_path(self.datadir) self.metauuid = 'ff89f933b2ca8df40' self.metafile = self.fhr_path+ '/' + self.metauuid+'/' + self.datadir.split('/')[-1] self.meta_fhr_path = parent_path(self.metafile) self.cnt_flag = self.cnt_exists() if not self.cnt_flag: return if self.meta_fhr_dir_is_deleted(): self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid
def __init__(self, path, drive, account, container,direr, logger, uid=DEFAULT_UID, gid=DEFAULT_GID,recycle_uuid=''): self.root = path self.container = container self.datadir = os.path.join(path, drive,container,direr) self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) self.fhr_path = parent_path(self.datadir) if 'recycle' == container: self.recycle_uuid = recycle_uuid self.metafile = os.path.join(path, drive,container +'/'+'meta',self.recycle_uuid) self.meta_fhr_path = parent_path(self.metafile) if self.meta_fhr_dir_is_deleted(): # self.create_dir_object(self.meta_fhr_path) self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid if not os.path.exists(self.datadir): return self.metadata = meta_read_metadata(self.metafile)
def meta_write_metadata(metapath, metadata): assert isinstance(metadata, dict) if not os.path.exists(parent_path(metapath)): mkdirs(parent_path(metapath)) with open(metapath,'wb' ) as f: pickle.dump(metadata , f )
def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path self.container = container self.datadir = path_std(os.path.join(path, drive, self.container)) self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) self.metauuid = 'ff89f933b2ca8df40' self.fhr_path = parent_path(self.datadir) self.metafile = self.fhr_path+ '/' + self.metauuid+'/' + self.datadir.split('/')[-1] self.meta_fhr_path = parent_path(self.metafile) if self.meta_fhr_dir_is_deleted(): self.create_dir_object(self.meta_fhr_path) if not os.path.exists(self.datadir): return if self.dir_exists: try: self.metadata = meta_read_metadata(self.metafile) except EOFError: meta_create_container_metadata(self.datadir,self.metafile) else: return if self.container: if not self.metadata: meta_create_container_metadata(self.datadir,self.metafile) self.metadata = meta_read_metadata(self.metafile) else: if not validate_container(self.metadata): meta_create_container_metadata(self.datadir,self.metafile) self.metadata = meta_read_metadata(self.metafile)
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None,recycle_uuid=''): self.iter_hook = iter_hook self.disk_chunk_size = disk_chunk_size obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) self.data_file = os.path.join(self.datadir, self.obj) self.fhr_path = parent_path(self.data_file) self.container = container if 'recycle' == container: self.recycle_uuid = recycle_uuid self.metafile = os.path.join(self.datadir[:-5]+'/'+'meta',self.recycle_uuid) self.meta_fhr_path = parent_path(self.metafile) if self.meta_fhr_dir_is_deleted(): #self.create_dir_object(self.meta_fhr_path) self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid if not os.path.exists(self.datadir + '/' + self.obj): return if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False) self.metadata = meta_read_metadata(self.metafile) if not self.metadata: meta_create_object_metadata(self.data_file,self.metafile) self.metadata = meta_read_metadata(self.metafile) if not validate_object(self.metadata): meta_create_object_metadata(self.data_file,self.metafile) self.metadata = meta_read_metadata(self.metafile) self.filter_metadata()
def __init__( self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None, ): self.iter_hook = iter_hook self.disk_chunk_size = disk_chunk_size obj = obj.strip("/") if "/" in obj: self.obj_path, self.obj = obj.rsplit("/", 1) else: self.obj_path = "" self.obj = obj if self.obj_path: self.name = "/".join((container, self.obj_path)) else: self.name = container # Absolute path for obj directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.cntpath = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, "tmp") self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) self.data_file = path_std(os.path.join(self.datadir, self.obj)) self.fhr_path = parent_path(self.data_file) self.container = container self.metauuid = "ff89f933b2ca8df40" self.metafile = self.fhr_path + "/" + self.metauuid + "/" + self.data_file.split("/")[-1] self.meta_fhr_path = parent_path(self.metafile) self.cnt_flag = self.cnt_exists() if not self.cnt_flag: return if self.meta_fhr_dir_is_deleted(): # self.create_dir_object(self.meta_fhr_path) self.tmp_meta_fhr_path = self.meta_fhr_path while not os.path.exists(self.tmp_meta_fhr_path): self.create_dir_object(self.tmp_meta_fhr_path) self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + "/" + self.metauuid if not os.path.exists(self.datadir + "/" + self.obj): return if os.path.isdir(self.datadir + "/" + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, "rb") if not keep_data_fp: self.close(verify_file=False) self.metadata = meta_read_metadata(self.metafile) if not self.metadata: meta_create_object_metadata(self.data_file, self.metafile) self.metadata = meta_read_metadata(self.metafile) if not validate_object(self.metadata): meta_create_object_metadata(self.data_file, self.metafile) self.metadata = meta_read_metadata(self.data_file) self.filter_metadata()
def __init__(self, path, device, partition, account, container, obj, logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None): self.iter_hook = iter_hook self.disk_chunk_size = disk_chunk_size obj = obj.strip('/') if '/' in obj: self.obj_path, self.obj = obj.rsplit('/', 1) else: self.obj_path = '' self.obj = obj if self.obj_path: self.name = '/'.join((container, self.obj_path)) else: self.name = container #Absolute path for obj directory. self.datadir = os.path.join(path, device, self.name) self.device_path = os.path.join(path, device) self.container_path = os.path.join(path, device, container) self.cntpath = os.path.join(path, device, container) self.tmpdir = os.path.join(path, device, 'tmp') self.logger = logger self.metadata = {} self.data_file = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None self.keep_cache = False self.is_dir = False self.is_valid = True self.uid = int(uid) self.gid = int(gid) self.data_file = os.path.join(self.datadir, self.obj) self.fhr_path = parent_path(self.data_file) self.cnt_flag = self.cnt_exists() if not self.cnt_flag: return if not os.path.exists(self.datadir + '/' + self.obj): return if os.path.isdir(self.datadir + '/' + self.obj): self.is_dir = True else: self.fp = do_open(self.data_file, 'rb') if not keep_data_fp: self.close(verify_file=False) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not self.metadata: create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) if not validate_object(self.metadata): create_object_metadata(self.datadir + '/' + self.obj) self.metadata = read_metadata(self.datadir + '/' + self.obj) self.filter_metadata()