Example #1
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False, disk_chunk_size=65536,
                 uid=DEFAULT_UID, gid=DEFAULT_GID):
        self.disk_chunk_size = disk_chunk_size
        #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj

        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
        #Absolute path for obj directory.
        self.datadir = os.path.join(path, device,
                    storage_directory(DATADIR, partition, self.name))

        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        if not os.path.exists(self.datadir + '/' + self.obj):
            return

        self.data_file = os.path.join(self.datadir, self.obj)
        self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not self.metadata:
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)

        if not validate_object(self.metadata):
            self.logger.error('Metadata validation failed %s %s' % \
                            (self.data_file, self.metadata))
            self.metadata = {}
            self.is_valid = False
            self.data_file = None
            return

        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)
Example #2
0
    def list_account_containers(self, limit, marker, end_marker,
                                                   prefix, delimiter,
                                                   out_content_type):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''
        containers = []
        container_count = 0
        account_list = []

        containers, container_count = get_account_details(self.datadir, self.memcache)

        if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
            self.metadata[X_CONTAINER_COUNT] = container_count
            self.update_account(self.metadata)

        if containers:
            containers.sort()

        if containers and prefix:
            containers = self.filter_prefix(containers, prefix)

        if containers and delimiter:
            containers = self.filter_delimiter(containers, delimiter, prefix)

        if containers and marker:
            containers = self.filter_marker(containers, marker)

        if containers and end_marker:
            containers = self.filter_end_marker(containers, end_marker)

        if containers and limit:
            if len(containers) > limit:
                containers = self.filter_limit(containers, limit)

        if containers:
            for cont in containers:
                list_item = []
                metadata = None
                list_item.append(cont)
                if out_content_type != 'text/plain':
                    metadata = read_metadata(self.datadir + '/' + cont)
                    if not metadata:
                        metadata = create_container_metadata(self.datadir + '/' + cont)

                if metadata:
                    list_item.append(metadata[X_OBJECTS_COUNT])
                    list_item.append(metadata[X_BYTES_USED])
                    list_item.append(0)
                account_list.append(list_item)

        #print 'Gaurav list_containet objs', objects
        return account_list
Example #3
0
    def __init__(self, path, device, partition, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID):
        if container:
            #self.name = '/'.join((account, container))
            self.name = container
        else:
            self.name = None
        if self.name:
            self.datadir = os.path.join(path, device,
                            storage_directory(DATADIR, partition, self.name))
        else:
            self.datadir = os.path.join(path, device)
            
        #print 'Gaurav DiskDir datadir', account, container, path,\
                                         #device, self.datadir
        self.device_path = os.path.join(path, device)
        self.logger = logger
        self.metadata = {}
        self.uid = int(uid)
        self.gid = int(gid)
        self.dir_exists = os.path.isdir (self.datadir)
        if self.dir_exists:
            self.metadata = read_metadata(self.datadir)
        else:
            return
        if container:
            if not self.metadata:
                #objects, object_count, bytes_used = get_container_details(self.datadir)
                #self.update_container(object_count, bytes_used)
                create_container_metadata(self.datadir)
                self.metadata = read_metadata(self.datadir)
            ret = validate_container(self.metadata)
        else:
            if not self.metadata:
                #containers, container_count = get_account_details(self.datadir)
                #self.update_acocunt(container_count)
                create_account_metadata(self.datadir)
                self.metadata = read_metadata(self.datadir)
            ret = validate_account(self.metadata)

        if not ret:
            self.dir_exists = False
            self.metadata = {}
Example #4
0
    def __init__(self, path, device, partition, account, container, logger,
                 memcache=None, uid=DEFAULT_UID, gid=DEFAULT_GID):
        if container:
            self.name = container
        else:
            self.name = None
        if self.name:
            self.datadir = os.path.join(path, device,
                            storage_directory(DATADIR, partition, self.name))
        else:
            self.datadir = os.path.join(path, device)
        self.device_path = os.path.join(path, device)
        self.logger = logger
        self.metadata = {}
        self.uid = int(uid)
        self.gid = int(gid)
        self.memcache = memcache
        self.dir_exists = os.path.isdir (self.datadir)
        if self.dir_exists:
            self.metadata = read_metadata(self.datadir)
        else:
            return
        if container:
            if not self.metadata:
                create_container_metadata(self.datadir, memcache=self.memcache)
                self.metadata = read_metadata(self.datadir)
            ret = validate_container(self.metadata)
        else:
            if not self.metadata:
                create_account_metadata(self.datadir, memcache=self.memcache)
                self.metadata = read_metadata(self.datadir)
            ret = validate_account(self.metadata)

        if not ret:
            self.logger.error('Metadata validation failed %s %s' % \
                              (self.datadir, self.metadata))
            self.dir_exists = False
            self.metadata = {}
Example #5
0
    def list_container_objects(self, limit, marker, end_marker,
                               prefix, delimiter, path, out_content_type):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        if path:
            prefix = path = path.rstrip('/') + '/'
            delimiter = '/'
        if delimiter and not prefix:
            prefix = ''

        #print 'Container_list prefix, del, datadir', prefix,\
                                                   #delimiter, self.datadir

        objects = []
        object_count = 0
        bytes_used = 0
        container_list = []

        objects, object_count, bytes_used = get_container_details(self.datadir,
                                                                  self.memcache)

        #print 'cont', object_count, self.metadata

        if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
           int(self.metadata[X_BYTES_USED]) != bytes_used:
            self.metadata[X_OBJECTS_COUNT] = object_count
            self.metadata[X_BYTES_USED] = bytes_used
            self.update_container(self.metadata)

        if objects:
            objects.sort()

        if objects and prefix:
            objects = self.filter_prefix(objects, prefix)

        if objects and delimiter:
            objects = self.filter_delimiter(objects, delimiter, prefix)

        if objects and marker:
            objects = self.filter_marker(objects, marker)

        if objects and end_marker:
            objects = self.filter_end_marker(objects, end_marker)

        if objects and limit:
            if len(objects) > limit:
                objects = self.filter_limit(objects, limit)

        if objects:
            for obj in objects:
                list_item = []
                metadata = None
                list_item.append(obj)
                if out_content_type != 'text/plain':
                    metadata = read_metadata(self.datadir + '/' + obj)
                    if not metadata:
                        metadata = create_object_metadata(self.datadir + '/' + obj)
                #print 'Gaurav list_obj meta', metadata
                if metadata:
                    list_item.append(metadata[X_TIMESTAMP])
                    list_item.append(metadata[X_CONTENT_LENGTH])
                    list_item.append(metadata[X_CONTENT_TYPE])
                    list_item.append(metadata[X_ETAG])
                container_list.append(list_item)

        #print 'Gaurav list_container objs', container_list
        return container_list
Example #6
0
    def __init__(self, path, device, partition, account, container, obj,
                 logger, keep_data_fp=False, disk_chunk_size=65536, uid=DEFAULT_UID,
                 gid=DEFAULT_GID):
        self.disk_chunk_size = disk_chunk_size
        #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj
        #self.name = '/'.join((account, container, self.obj_path))
        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
        #name_hash = hash_path(account, container, obj)
        self.datadir = os.path.join(path, device,
                    storage_directory(DATADIR, partition, self.name))
        #print 'Gaurav Diskfile datadir', account, container, obj, path,\
                                         #device, self.datadir
        self.device_path = os.path.join(path, device)
        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        if not os.path.exists(self.datadir + '/' + self.obj):
            return
        #TODO: marker objects.
        
        self.data_file = os.path.join(self.datadir, self.obj)
        self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not self.metadata:
            #self.metadata = get_object_details(self.datadir + '/' + self.obj)
            #self.update_object(self.metadata)
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not validate_object(self.metadata):
            self.metadata = {}
            self.is_valid = False
            self.data_file = None
            return

        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)
Example #7
0
            #name, ext = fname.rsplit('.', 1)
            if fname == self.obj:
                try:
                    os.unlink(os.path.join(self.datadir, fname))
                except OSError, err:
                    if err.errno != errno.ENOENT:
                        raise

        if self.obj_path:
            obj_dirs = self.obj_path.split('/')
            tmp_path = self.obj_path
            if len(obj_dirs):
                while tmp_path:
                    #TODO: Check dir is empty (Done in rmdirs)
                    dir_path = os.path.join(self.container_path, tmp_path)
                    metadata = read_metadata(dir_path)
                    #if metadata[X_OBJECT_TYPE] == DIR:
                    rmdirs(dir_path)
                    if '/' in tmp_path:
                        tmp_path = tmp_path.rsplit('/', 1)[0]
                    else:
                        break
                    
        self.metadata = {}
        self.data_file = None

        
                    
    def drop_cache(self, fd, offset, length):
        """Method for no-oping buffer cache drop method."""
        if not self.keep_cache: