コード例 #1
0
ファイル: DiskDirer.py プロジェクト: sun3shines/ufo
def get_tree_info(datapath,metauuid):
    
    objects = os.listdir(datapath)
    container_list = []
    
    for obj in objects:
        if 'ff89f933b2ca8df40' == obj:
            continue
        
        list_item = {}
            
        obj_path = path_std(os.path.join(datapath, obj))
        obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),metauuid,obj_path.split('/')[-1])
        metadata = meta_read_metadata(obj_meta_path)
        
        if not metadata or not validate_object(metadata):
            metadata = meta_create_object_metadata(obj_path,obj_meta_path)
                                
        if metadata:
            list_item.update({'name':obj})
            list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])})
            list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])})
            list_item.update({'md5':metadata[X_ETAG]})
            list_item.update({'ftype':metadata[X_FILE_TYPE]})
            if metadata.get('metadata'):
                list_item.update({'metadata':metadata['metadata']})
                
        if os.path.isdir(obj_path):
            cList = get_tree_info(obj_path,metauuid)
            list_item.update({'list':cList})
        container_list.append(list_item)
            
    return container_list
コード例 #2
0
ファイル: DiskDirer.py プロジェクト: sun3shines/ufo
    def non_recursive_iter(self):
        
        objects = os.listdir(self.datadir)
        if 'ff89f933b2ca8df40' in objects:
            objects.remove('ff89f933b2ca8df40')
            
        if objects:
            objects.sort()

        container_list = []
        if objects:
            for obj in objects:
                list_item = {}
                
                obj_path = path_std(os.path.join(self.datadir, obj))
                obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),self.metauuid,obj_path.split('/')[-1])
                metadata = meta_read_metadata(obj_meta_path)
                
                if not metadata or not validate_object(metadata):
                    metadata = meta_create_object_metadata(obj_path,obj_meta_path)
                                        
                if metadata:
                    list_item.update({'name':obj})
                    list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])})
                    list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])})
                    list_item.update({'md5':metadata[X_ETAG]})
                    list_item.update({'ftype':metadata[X_FILE_TYPE]})
                    if metadata.has_key('metadata'):
                        list_item.update({'metadata':metadata['metadata']})
                
                container_list.append(list_item)

        return container_list
コード例 #3
0
ファイル: ContainerMeta.py プロジェクト: sun7shines/Cloudfs
    def prefix_list_objects_iter(self, limit, marker, end_marker,
                          prefix, delimiter, path):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
    
        if path:
            prefix = path = path.rstrip('/') + '/'
            delimiter = '/'
        if delimiter and not prefix:
            prefix = ''

        self.update_object_count()

        objects, object_count, bytes_used = self.object_info

        if objects:
            objects.sort()

        if objects and prefix:
            objects = self.filter_prefix(objects, prefix)

        if objects and delimiter:
            objects = self.filter_delimiter(objects, delimiter, prefix)

        if objects and marker:
            objects = self.filter_marker(objects, marker)

        if objects and end_marker:
            objects = self.filter_end_marker(objects, end_marker)

        if objects and limit:
            if len(objects) > limit:
                objects = self.filter_limit(objects, limit)

        container_list = []
        if objects:
            for obj in objects:
                list_item = {}
                
                obj_path = path_std(os.path.join(self.datadir, obj))
                obj_meta_path = os.path.join('/'.join(obj_path.split('/')[:-1]),self.metauuid,obj_path.split('/')[-1])
                metadata = meta_read_metadata(obj_meta_path)
                
                if not metadata or not validate_object(metadata):
                    metadata = meta_create_object_metadata(obj_path,obj_meta_path)
                if metadata:
               
                    list_item.update({'name':obj})
                    list_item.update({'modificationTime':str(metadata[X_TIMESTAMP])})
                    list_item.update({'bytes':int(metadata[X_CONTENT_LENGTH])})
                    list_item.update({'md5':metadata[X_ETAG]})
                    list_item.update({'ftype':metadata[X_FILE_TYPE]})
                    if metadata.has_key('metadata'):
                        list_item.update({'metadata':metadata['metadata']})
                
                container_list.append(list_item)

        return container_list
コード例 #4
0
ファイル: DiskMeta.py プロジェクト: sun3shines/ufo
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False, disk_chunk_size=65536,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None,recycle_uuid=''):
        
        self.iter_hook = iter_hook
        self.disk_chunk_size = disk_chunk_size
        
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj

        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
            
        self.datadir = os.path.join(path, device, self.name)
        
        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        
        self.data_file = os.path.join(self.datadir, self.obj)
        self.fhr_path = parent_path(self.data_file)
        
        self.container = container
        if 'recycle' == container:
            self.recycle_uuid = recycle_uuid
            self.metafile = os.path.join(self.datadir[:-5]+'/'+'meta',self.recycle_uuid)
            self.meta_fhr_path = parent_path(self.metafile) 
            
        if self.meta_fhr_dir_is_deleted():
            #self.create_dir_object(self.meta_fhr_path)
            self.tmp_meta_fhr_path = self.meta_fhr_path
            while not os.path.exists(self.tmp_meta_fhr_path):
                self.create_dir_object(self.tmp_meta_fhr_path)
                self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + '/' +self.metauuid
                
        if not os.path.exists(self.datadir + '/' + self.obj):
            return
        
        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)
                  
        self.metadata = meta_read_metadata(self.metafile)
        if not self.metadata:
            meta_create_object_metadata(self.data_file,self.metafile)
            self.metadata = meta_read_metadata(self.metafile)

        if not validate_object(self.metadata):
            meta_create_object_metadata(self.data_file,self.metafile)
            self.metadata = meta_read_metadata(self.metafile)
            
        self.filter_metadata()
コード例 #5
0
ファイル: FileMeta.py プロジェクト: sun7shines/Cloudfs
    def __init__(
        self,
        path,
        device,
        partition,
        account,
        container,
        obj,
        logger,
        keep_data_fp=False,
        disk_chunk_size=65536,
        uid=DEFAULT_UID,
        gid=DEFAULT_GID,
        iter_hook=None,
    ):

        self.iter_hook = iter_hook
        self.disk_chunk_size = disk_chunk_size

        obj = obj.strip("/")
        if "/" in obj:
            self.obj_path, self.obj = obj.rsplit("/", 1)
        else:
            self.obj_path = ""
            self.obj = obj

        if self.obj_path:
            self.name = "/".join((container, self.obj_path))
        else:
            self.name = container

        # Absolute path for obj directory.

        self.datadir = os.path.join(path, device, self.name)

        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.cntpath = os.path.join(path, device, container)

        self.tmpdir = os.path.join(path, device, "tmp")
        self.logger = logger
        self.metadata = {}
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)

        self.data_file = path_std(os.path.join(self.datadir, self.obj))
        self.fhr_path = parent_path(self.data_file)

        self.container = container
        self.metauuid = "ff89f933b2ca8df40"
        self.metafile = self.fhr_path + "/" + self.metauuid + "/" + self.data_file.split("/")[-1]

        self.meta_fhr_path = parent_path(self.metafile)

        self.cnt_flag = self.cnt_exists()
        if not self.cnt_flag:
            return

        if self.meta_fhr_dir_is_deleted():
            # self.create_dir_object(self.meta_fhr_path)
            self.tmp_meta_fhr_path = self.meta_fhr_path
            while not os.path.exists(self.tmp_meta_fhr_path):
                self.create_dir_object(self.tmp_meta_fhr_path)
                self.tmp_meta_fhr_path = parent_path(parent_path(self.tmp_meta_fhr_path)) + "/" + self.metauuid

        if not os.path.exists(self.datadir + "/" + self.obj):
            return

        if os.path.isdir(self.datadir + "/" + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, "rb")
            if not keep_data_fp:
                self.close(verify_file=False)

        self.metadata = meta_read_metadata(self.metafile)
        if not self.metadata:
            meta_create_object_metadata(self.data_file, self.metafile)
            self.metadata = meta_read_metadata(self.metafile)

        if not validate_object(self.metadata):
            meta_create_object_metadata(self.data_file, self.metafile)
            self.metadata = meta_read_metadata(self.data_file)

        self.filter_metadata()
コード例 #6
0
ファイル: DiskFile.py プロジェクト: sun3shines/ufo
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False, disk_chunk_size=65536,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
        
        self.iter_hook = iter_hook
        self.disk_chunk_size = disk_chunk_size
        
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj

        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
            
        #Absolute path for obj directory.
        
        self.datadir = os.path.join(path, device, self.name)
        
        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.cntpath = os.path.join(path, device, container)
        
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        
        self.data_file = os.path.join(self.datadir, self.obj)
        self.fhr_path = parent_path(self.data_file)
        
        self.cnt_flag = self.cnt_exists()
        if not self.cnt_flag:
            return
        
        if not os.path.exists(self.datadir + '/' + self.obj):
            return
        
        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)
                
        
        self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not self.metadata:
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)


        if not validate_object(self.metadata):
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' +
                                        self.obj)
            
        self.filter_metadata()