예제 #1
0
 def __init__(self,
              path,
              device,
              partition,
              account,
              container,
              logger,
              uid=DEFAULT_UID,
              gid=DEFAULT_GID,
              fs_object=None):
     self.root = path
     device = account
     if container:
         self.name = container
     else:
         self.name = None
     if self.name:
         self.datadir = os.path.join(path, account, self.name)
     else:
         self.datadir = os.path.join(path, device)
     self.account = account
     self.device_path = os.path.join(path, device)
     if not check_mount(path, device):
         check_valid_account(account, fs_object)
     self.logger = logger
     self.metadata = {}
     self.uid = int(uid)
     self.gid = int(gid)
     # Create a dummy db_file in /etc/swift
     self.db_file = '/etc/swift/db_file.db'
     if not os.path.exists(self.db_file):
         file(self.db_file, 'w+')
     self.dir_exists = os.path.exists(self.datadir)
     if self.dir_exists:
         try:
             self.metadata = read_metadata(self.datadir)
         except EOFError:
             create_container_metadata(self.datadir)
     else:
         return
     if container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = read_metadata(self.datadir)
예제 #2
0
 def __init__(self, path, device, partition, account, container, logger,
              uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object=None):
     self.root = path
     device = account
     if container:
         self.name = container
     else:
         self.name = None
     if self.name:
         self.datadir = os.path.join(path, account, self.name)
     else:
         self.datadir = os.path.join(path, device)
     self.account = account
     self.device_path = os.path.join(path, device)
     if not check_mount(path, device):
         check_valid_account(account, fs_object)
     self.logger = logger
     self.metadata = {}
     self.uid = int(uid)
     self.gid = int(gid)
     # Create a dummy db_file in /etc/swift
     self.db_file = '/etc/swift/db_file.db'
     if not os.path.exists(self.db_file):
         file(self.db_file, 'w+')
     self.dir_exists = os.path.exists(self.datadir)
     if self.dir_exists:
         try:
             self.metadata = read_metadata(self.datadir)
         except EOFError:
             create_container_metadata(self.datadir)
     else:
         return
     if container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = read_metadata(self.datadir)
예제 #3
0
    def get_info(self, include_metadata=False):
        """
        Get global data for the account.
        :returns: dict with keys: account, created_at, put_timestamp,
                  delete_timestamp, container_count, object_count,
                  bytes_used, hash, id
        """
        metadata = {}
        if (os.path.exists(self.datadir)):
            metadata = read_metadata(self.datadir)
            if not metadata:
                metadata = create_account_metadata(self.datadir)

        data = {
            'account': self.account,
            'created_at': '1',
            'put_timestamp': '1',
            'delete_timestamp': '1',
            'container_count': metadata.get(X_CONTAINER_COUNT, 0),
            'object_count': metadata.get(X_OBJECTS_COUNT, 0),
            'bytes_used': metadata.get(X_BYTES_USED, 0),
            'hash': '',
            'id': ''
        }

        if include_metadata:
            data['metadata'] = metadata
        return data
예제 #4
0
    def get_info(self, include_metadata=False):
        """
        Get global data for the container.
        :returns: dict with keys: account, container, created_at,
                  put_timestamp, delete_timestamp, object_count, bytes_used,
                  reported_put_timestamp, reported_delete_timestamp,
                  reported_object_count, reported_bytes_used, hash, id,
                  x_container_sync_point1, and x_container_sync_point2.
                  If include_metadata is set, metadata is included as a key
                  pointing to a dict of tuples of the metadata
        """
        # TODO: delete_timestamp, reported_put_timestamp
        #       reported_delete_timestamp, reported_object_count,
        #       reported_bytes_used, created_at

        metadata = {}
        if os.path.exists(self.datadir):
            metadata = read_metadata(self.datadir)

        data = {'account' : self.account, 'container' : self.name,
                'object_count' : metadata.get(X_OBJECTS_COUNT, '0'),
                'bytes_used' : metadata.get(X_BYTES_USED, '0'),
                'hash': '', 'id' : '', 'created_at' : '1',
                'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, '0'),
                'delete_timestamp' : '1',
                'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
                'reported_object_count' : '1', 'reported_bytes_used' : '1'}
        if include_metadata:
            data['metadata'] = metadata
        return data
예제 #5
0
    def list_objects_iter(self, limit, marker, end_marker,
                          prefix, delimiter, path):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        if path:
            prefix = path = path.rstrip('/') + '/'
            delimiter = '/'
        if delimiter and not prefix:
            prefix = ''

        objects = []
        object_count = 0
        bytes_used = 0
        container_list = []

        objects, object_count, bytes_used = get_container_details(self.datadir)

        if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
           int(self.metadata[X_BYTES_USED][0]) != bytes_used:
            self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
            self.metadata[X_BYTES_USED] = (bytes_used, 0)
            self.update_container(self.metadata)

        if objects:
            objects.sort()

        if objects and prefix:
            objects = self.filter_prefix(objects, prefix)

        if objects and delimiter:
            objects = self.filter_delimiter(objects, delimiter, prefix)

        if objects and marker:
            objects = self.filter_marker(objects, marker)

        if objects and end_marker:
            objects = self.filter_end_marker(objects, end_marker)

        if objects and limit:
            if len(objects) > limit:
                objects = self.filter_limit(objects, limit)

        if objects:
            for obj in objects:
                list_item = []
                list_item.append(obj)
                obj_path = os.path.join(self.datadir, obj)
                metadata = read_metadata(obj_path)
                if not metadata or not validate_object(metadata):
                    metadata = create_object_metadata(obj_path)
                if metadata:
                    list_item.append(metadata[X_TIMESTAMP])
                    list_item.append(int(metadata[X_CONTENT_LENGTH]))
                    list_item.append(metadata[X_CONTENT_TYPE])
                    list_item.append(metadata[X_ETAG])
                container_list.append(list_item)

        return container_list
예제 #6
0
 def test_read_metadata(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' }
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
예제 #7
0
    def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
                          path):
        """
        Returns tuple of name, created_at, size, content_type, etag.
        """
        if path:
            prefix = path = path.rstrip('/') + '/'
            delimiter = '/'
        if delimiter and not prefix:
            prefix = ''

        objects = []
        object_count = 0
        bytes_used = 0
        container_list = []

        objects, object_count, bytes_used = get_container_details(self.datadir)

        if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
           int(self.metadata[X_BYTES_USED]) != bytes_used:
            self.metadata[X_OBJECTS_COUNT] = object_count
            self.metadata[X_BYTES_USED] = bytes_used
            self.update_container(self.metadata)

        if objects:
            objects.sort()

        if objects and prefix:
            objects = self.filter_prefix(objects, prefix)

        if objects and delimiter:
            objects = self.filter_delimiter(objects, delimiter, prefix)

        if objects and marker:
            objects = self.filter_marker(objects, marker)

        if objects and end_marker:
            objects = self.filter_end_marker(objects, end_marker)

        if objects and limit:
            if len(objects) > limit:
                objects = self.filter_limit(objects, limit)

        if objects:
            for obj in objects:
                list_item = []
                list_item.append(obj)
                metadata = read_metadata(self.datadir + '/' + obj)
                if not metadata or not validate_object(metadata):
                    metadata = create_object_metadata(self.datadir + '/' + obj)
                if metadata:
                    list_item.append(metadata[X_TIMESTAMP])
                    list_item.append(int(metadata[X_CONTENT_LENGTH]))
                    list_item.append(metadata[X_CONTENT_TYPE])
                    list_item.append(metadata[X_ETAG])
                container_list.append(list_item)

        return container_list
예제 #8
0
 def __init__(self, root, account, fs_object=None):
     self.root = root
     self.account = account
     self.datadir = os.path.join(self.root, self.account)
     if not check_mount(root, account):
         check_valid_account(account, fs_object)
     self.metadata = read_metadata(self.datadir)
     if not self.metadata or not validate_account(self.metadata):
         self.metadata = create_account_metadata(self.datadir)
예제 #9
0
 def __init__(self, root, account, fs_object = None):
     self.root = root
     self.account = account
     self.datadir = os.path.join(self.root, self.account)
     if not check_mount(root, account):
         check_valid_account(account, fs_object)
     self.metadata = read_metadata(self.datadir)
     if not self.metadata or not validate_account(self.metadata):
         self.metadata = create_account_metadata(self.datadir)
예제 #10
0
    def list_containers_iter(self, limit, marker, end_marker, prefix,
                             delimiter):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''
        containers = []
        container_count = 0
        account_list = []

        containers, container_count = get_account_details(self.datadir)

        if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
            self.metadata[X_CONTAINER_COUNT] = container_count
            self.update_account(self.metadata)

        if containers:
            containers.sort()

        if containers and prefix:
            containers = self.filter_prefix(containers, prefix)

        if containers and delimiter:
            containers = self.filter_delimiter(containers, delimiter, prefix)

        if containers and marker:
            containers = self.filter_marker(containers, marker)

        if containers and end_marker:
            containers = self.filter_end_marker(containers, end_marker)

        if containers and limit:
            if len(containers) > limit:
                containers = self.filter_limit(containers, limit)

        if containers:
            for cont in containers:
                list_item = []
                metadata = None
                list_item.append(cont)
                metadata = read_metadata(self.datadir + '/' + cont)
                if not metadata or not validate_container(metadata):
                    metadata = create_container_metadata(self.datadir + '/' +
                                                         cont)

                if metadata:
                    list_item.append(metadata[X_OBJECTS_COUNT])
                    list_item.append(metadata[X_BYTES_USED])
                    list_item.append(0)
                account_list.append(list_item)

        return account_list
예제 #11
0
    def list_containers_iter(self, limit, marker, end_marker,
                             prefix, delimiter):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''
        containers = []
        container_count = 0
        account_list = []

        containers, container_count = get_account_details(self.datadir)

        if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
            self.metadata[X_CONTAINER_COUNT] = container_count
            self.update_account(self.metadata)

        if containers:
            containers.sort()

        if containers and prefix:
            containers = self.filter_prefix(containers, prefix)

        if containers and delimiter:
            containers = self.filter_delimiter(containers, delimiter, prefix)

        if containers and marker:
            containers = self.filter_marker(containers, marker)

        if containers and end_marker:
            containers = self.filter_end_marker(containers, end_marker)

        if containers and limit:
            if len(containers) > limit:
                containers = self.filter_limit(containers, limit)

        if containers:
            for cont in containers:
                list_item = []
                metadata = None
                list_item.append(cont)
                metadata = read_metadata(self.datadir + '/' + cont)
                if not metadata or not validate_container(metadata):
                    metadata = create_container_metadata(self.datadir + '/' + cont)

                if metadata:
                    list_item.append(metadata[X_OBJECTS_COUNT])
                    list_item.append(metadata[X_BYTES_USED])
                    list_item.append(0)
                account_list.append(list_item)

        return account_list
예제 #12
0
 def test_read_metadata_multiple(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' * 150000 }
     expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     for i in range(0,3):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert not expected_p
     res_d = utils.read_metadata(path)
     assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
예제 #13
0
 def test_read_metadata_err(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' }
     xkey = _xkey(path, utils.METADATA_KEY)
     _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     _xattr_err[xkey] = errno.EOPNOTSUPP
     try:
         res_d = utils.read_metadata(path)
     except IOError as e:
         assert e.errno == errno.EOPNOTSUPP
         assert (_xattr_op_cnt['get'] == 1), "%r" % _xattr_op_cnt
     else:
         self.fail("Expected an IOError exception on get")
예제 #14
0
 def test_read_metadata_multiple_one_missing(self):
     path = "/tmp/foo/r"
     expected_d = { 'a': 'y' * 150000 }
     expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL)
     for i in range(0,2):
         xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
         _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE]
         expected_p = expected_p[utils.MAX_XATTR_SIZE:]
     assert len(expected_p) <= utils.MAX_XATTR_SIZE
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
     assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys())
예제 #15
0
파일: DiskDir.py 프로젝트: gf1730/glusterfs
def _read_metadata(dd):
    """ Filter read metadata so that it always returns a tuple that includes
        some kind of timestamp. With 1.4.8 of the Swift integration the
        timestamps were not stored. Here we fabricate timestamps for volumes
        where the existing data has no timestamp (that is, stored data is not
        a tuple), allowing us a measure of backward compatibility.

        FIXME: At this time it does not appear that the timestamps on each
        metadata are used for much, so this should not hurt anything.
    """
    metadata_i = read_metadata(dd)
    metadata = {}
    timestamp = 0
    for key, value in metadata_i.iteritems():
        if not isinstance(value, tuple):
            value = (value, timestamp)
        metadata[key] = value
    return metadata
예제 #16
0
    def get_info(self, include_metadata=False):
        """
        Get global data for the account.
        :returns: dict with keys: account, created_at, put_timestamp,
                  delete_timestamp, container_count, object_count,
                  bytes_used, hash, id
        """
        metadata = {}
        if (os.path.exists(self.datadir)):
            metadata = read_metadata(self.datadir)
            if not metadata:
                metadata = create_account_metadata(self.datadir)

        data = {'account' : self.account, 'created_at' : '1',
                'put_timestamp' : '1', 'delete_timestamp' : '1',
                'container_count' : metadata.get(X_CONTAINER_COUNT, 0),
                'object_count' : metadata.get(X_OBJECTS_COUNT, 0),
                'bytes_used' : metadata.get(X_BYTES_USED, 0),
                'hash' : '', 'id' : ''}

        if include_metadata:
            data['metadata'] = metadata
        return data
예제 #17
0
 def test_read_metadata_notfound(self):
     path = "/tmp/foo/r"
     res_d = utils.read_metadata(path)
     assert res_d == {}
     assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
예제 #18
0
    def __init__(
        self,
        path,
        device,
        partition,
        account,
        container,
        obj,
        logger,
        keep_data_fp=False,
        disk_chunk_size=65536,
        uid=DEFAULT_UID,
        gid=DEFAULT_GID,
    ):
        self.disk_chunk_size = disk_chunk_size
        device = account
        # Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
        obj = obj.strip("/")
        if "/" in obj:
            self.obj_path, self.obj = obj.rsplit("/", 1)
        else:
            self.obj_path = ""
            self.obj = obj

        if self.obj_path:
            self.name = "/".join((container, self.obj_path))
        else:
            self.name = container
        # Absolute path for obj directory.
        self.datadir = os.path.join(path, device, self.name)

        self.device_path = os.path.join(path, device)
        if not check_mount(path, device):
            check_valid_account(account)

        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, "tmp")
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        if not os.path.exists(self.datadir + "/" + self.obj):
            return

        self.data_file = os.path.join(self.datadir, self.obj)
        self.metadata = read_metadata(self.datadir + "/" + self.obj)
        if not self.metadata:
            create_object_metadata(self.datadir + "/" + self.obj)
            self.metadata = read_metadata(self.datadir + "/" + self.obj)

        if not validate_object(self.metadata):
            create_object_metadata(self.datadir + "/" + self.obj)
            self.metadata = read_metadata(self.datadir + "/" + self.obj)

        self.filter_metadata()

        if os.path.isdir(self.datadir + "/" + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, "rb")
            if not keep_data_fp:
                self.close(verify_file=False)
예제 #19
0
    def __init__(self,
                 path,
                 device,
                 partition,
                 account,
                 container,
                 obj,
                 logger,
                 keep_data_fp=False,
                 disk_chunk_size=65536,
                 uid=DEFAULT_UID,
                 gid=DEFAULT_GID,
                 fs_object=None):
        self.disk_chunk_size = disk_chunk_size
        device = account
        #Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
        obj = obj.strip('/')
        if '/' in obj:
            self.obj_path, self.obj = obj.rsplit('/', 1)
        else:
            self.obj_path = ''
            self.obj = obj

        if self.obj_path:
            self.name = '/'.join((container, self.obj_path))
        else:
            self.name = container
        #Absolute path for obj directory.
        self.datadir = os.path.join(path, device, self.name)

        self.device_path = os.path.join(path, device)
        if not check_mount(path, device):
            check_valid_account(account, fs_object)

        self.container_path = os.path.join(path, device, container)
        self.tmpdir = os.path.join(path, device, 'tmp')
        self.logger = logger
        self.metadata = {}
        self.meta_file = None
        self.data_file = None
        self.fp = None
        self.iter_etag = None
        self.started_at_0 = False
        self.read_to_eof = False
        self.quarantined_dir = None
        self.keep_cache = False
        self.is_dir = False
        self.is_valid = True
        self.uid = int(uid)
        self.gid = int(gid)
        if not os.path.exists(self.datadir + '/' + self.obj):
            return

        self.data_file = os.path.join(self.datadir, self.obj)
        self.metadata = read_metadata(self.datadir + '/' + self.obj)
        if not self.metadata:
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)

        if not validate_object(self.metadata):
            create_object_metadata(self.datadir + '/' + self.obj)
            self.metadata = read_metadata(self.datadir + '/' + self.obj)

        self.filter_metadata()

        if os.path.isdir(self.datadir + '/' + self.obj):
            self.is_dir = True
        else:
            self.fp = do_open(self.data_file, 'rb')
            if not keep_data_fp:
                self.close(verify_file=False)