Example #1
0
    def __init__(self,
                 path,
                 drive,
                 account,
                 container,
                 logger,
                 uid=DEFAULT_UID,
                 gid=DEFAULT_GID,
                 **kwargs):
        super(DiskDir, self).__init__(path, drive, account, logger, **kwargs)

        self.uid = int(uid)
        self.gid = int(gid)

        self.container = container
        self.datadir = os.path.join(self.datadir, self.container)

        if not self._dir_exists_read_metadata():
            return

        if not self.metadata:
            create_container_metadata(self.datadir)
            self.metadata = _read_metadata(self.datadir)
        else:
            if not validate_container(self.metadata):
                create_container_metadata(self.datadir)
                self.metadata = _read_metadata(self.datadir)
Example #2
0
 def _initialize(self):
     if self.container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
Example #3
0
    def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID):
        super(DiskDir, self).__init__(path, drive, account, logger)

        self.uid = int(uid)
        self.gid = int(gid)

        self.container = container
        self.datadir = os.path.join(self.datadir, self.container)

        if not self._dir_exists_read_metadata():
            return

        if not self.metadata:
            create_container_metadata(self.datadir)
            self.metadata = _read_metadata(self.datadir)
        else:
            if not validate_container(self.metadata):
                create_container_metadata(self.datadir)
                self.metadata = _read_metadata(self.datadir)
Example #4
0
    def __init__(self, path, drive, account, container, logger,
                 uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs):
        super(DiskDir, self).__init__(path, drive, account, logger, **kwargs)

        self.uid = int(uid)
        self.gid = int(gid)

        self.container = container
        self.datadir = os.path.join(self.datadir, self.container)

        if self.account == 'gsexpiring':
            # Do not bother crawling the entire container tree just to update
            # object count and bytes used. Return immediately before metadata
            # validation and creation happens.
            info = do_stat(self.datadir)
            if info and stat.S_ISDIR(info.st_mode):
                self._dir_exists = True
            if not info:
                # Container no longer exists.
                return
            semi_fake_md = {
                'X-Object-Count': (0, 0),
                'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0),
                'X-Type': ('container', 0),
                'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0),
                'X-Bytes-Used': (0, 0)
            }
            self.metadata = semi_fake_md
            return

        if not self._dir_exists_read_metadata():
            return

        if not self.metadata:
            create_container_metadata(self.datadir)
            self.metadata = _read_metadata(self.datadir)
        else:
            if not validate_container(self.metadata):
                create_container_metadata(self.datadir)
                self.metadata = _read_metadata(self.datadir)
Example #5
0
    def list_containers_iter(self, limit, marker, end_marker, prefix,
                             delimiter):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''

        self.update_container_count()

        containers, container_count = self.container_info

        if containers:
            containers.sort()

        if containers and prefix:
            containers = self.filter_prefix(containers, prefix)

        if containers and delimiter:
            containers = self.filter_delimiter(containers, delimiter, prefix)

        if containers and marker:
            containers = self.filter_marker(containers, marker)

        if containers and end_marker:
            containers = self.filter_end_marker(containers, end_marker)

        if containers and limit:
            if len(containers) > limit:
                containers = self.filter_limit(containers, limit)

        account_list = []
        if containers:
            for cont in containers:
                list_item = []
                metadata = None
                list_item.append(cont)
                cont_path = os.path.join(self.datadir, cont)
                metadata = _read_metadata(cont_path)
                if not metadata or not validate_container(metadata):
                    metadata = create_container_metadata(cont_path)

                if metadata:
                    list_item.append(metadata[X_OBJECTS_COUNT][0])
                    list_item.append(metadata[X_BYTES_USED][0])
                    list_item.append(0)
                account_list.append(list_item)

        return account_list
Example #6
0
    def list_containers_iter(self, limit, marker, end_marker,
                             prefix, delimiter):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''

        self.update_container_count()

        containers, container_count = self.container_info

        if containers:
            containers.sort()

        if containers and prefix:
            containers = self.filter_prefix(containers, prefix)

        if containers and delimiter:
            containers = self.filter_delimiter(containers, delimiter, prefix)

        if containers and marker:
            containers = self.filter_marker(containers, marker)

        if containers and end_marker:
            containers = self.filter_end_marker(containers, end_marker)

        if containers and limit:
            if len(containers) > limit:
                containers = self.filter_limit(containers, limit)

        account_list = []
        if containers:
            for cont in containers:
                list_item = []
                metadata = None
                list_item.append(cont)
                cont_path = os.path.join(self.datadir, cont)
                metadata = _read_metadata(cont_path)
                if not metadata or not validate_container(metadata):
                    metadata = create_container_metadata(cont_path)

                if metadata:
                    list_item.append(metadata[X_OBJECTS_COUNT][0])
                    list_item.append(metadata[X_BYTES_USED][0])
                    list_item.append(0)
                account_list.append(list_item)

        return account_list
Example #7
0
 def __init__(self,
              path,
              account,
              container,
              logger,
              uid=DEFAULT_UID,
              gid=DEFAULT_GID):
     self.root = path
     if container:
         self.container = container
     else:
         self.container = None
     if self.container:
         self.datadir = os.path.join(path, account, self.container)
     else:
         self.datadir = os.path.join(path, account)
     # Note that the account name has a one-to-one mapping to the gluster
     # mount point, or volume name.
     self.account = account
     assert logger is not None
     self.logger = logger
     self.metadata = {}
     self.container_info = None
     self.object_info = None
     self.uid = int(uid)
     self.gid = int(gid)
     self.db_file = _db_file
     self.dir_exists = os.path.exists(self.datadir)
     if self.dir_exists:
         try:
             self.metadata = _read_metadata(self.datadir)
         except EOFError:
             create_container_metadata(self.datadir)
     else:
         return
     if self.container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
Example #8
0
    def test_create_container_metadata(self):
        td = tempfile.mkdtemp()
        try:
            r_md = utils.create_container_metadata(td)

            xkey = _xkey(td, utils.METADATA_KEY)
            assert len(_xattrs.keys()) == 1
            assert xkey in _xattrs
            assert _xattr_op_cnt['get'] == 1
            assert _xattr_op_cnt['set'] == 1
            md = pickle.loads(_xattrs[xkey])
            assert r_md == md

            for key in self.cont_keys:
                assert key in md, "Expected key %s in %r" % (key, md)
            assert md[utils.X_TYPE] == (utils.CONTAINER, 0)
            assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0)
            assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0)
            assert md[utils.X_OBJECTS_COUNT] == (0, 0)
            assert md[utils.X_BYTES_USED] == (0, 0)
        finally:
            os.rmdir(td)
Example #9
0
 def __init__(self, path, account, container, logger,
              uid=DEFAULT_UID, gid=DEFAULT_GID):
     self.root = path
     if container:
         self.container = container
     else:
         self.container = None
     if self.container:
         self.datadir = os.path.join(path, account, self.container)
     else:
         self.datadir = os.path.join(path, account)
     # Note that the account name has a one-to-one mapping to the gluster
     # mount point, or volume name.
     self.account = account
     assert logger is not None
     self.logger = logger
     self.metadata = {}
     self.container_info = None
     self.object_info = None
     self.uid = int(uid)
     self.gid = int(gid)
     self.db_file = _db_file
     self.dir_exists = os.path.exists(self.datadir)
     if self.dir_exists:
         try:
             self.metadata = _read_metadata(self.datadir)
         except EOFError:
             create_container_metadata(self.datadir)
     else:
         return
     if self.container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
Example #10
0
 def __init__(self, path, drive, account, container, logger,
              uid=DEFAULT_UID, gid=DEFAULT_GID):
     self.root = path
     if container:
         self.container = container
     else:
         self.container = None
     if self.container:
         self.datadir = os.path.join(path, drive, self.container)
     else:
         self.datadir = os.path.join(path, drive)
     self.account = account
     assert logger is not None
     self.logger = logger
     self.metadata = {}
     self.container_info = None
     self.object_info = None
     self.uid = int(uid)
     self.gid = int(gid)
     self.db_file = _db_file
     self.dir_exists = os.path.exists(self.datadir)
     if self.dir_exists:
         try:
             self.metadata = _read_metadata(self.datadir)
         except EOFError:
             create_container_metadata(self.datadir)
     else:
         return
     if self.container:
         if not self.metadata:
             create_container_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_container(self.metadata):
                 create_container_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
     else:
         if not self.metadata:
             create_account_metadata(self.datadir)
             self.metadata = _read_metadata(self.datadir)
         else:
             if not validate_account(self.metadata):
                 create_account_metadata(self.datadir)
                 self.metadata = _read_metadata(self.datadir)
Example #11
0
    def list_containers_iter(self, limit, marker, end_marker,
                             prefix, delimiter):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''

        account_list = []
        containers = self._update_container_count()
        if containers:
            containers.sort()
        else:
            return account_list

        if containers and end_marker:
            containers = filter_end_marker(containers, end_marker)

        if containers:
            if marker and marker >= prefix:
                containers = filter_marker(containers, marker)
            elif prefix:
                containers = filter_prefix_as_marker(containers, prefix)

        if prefix is None:
            # No prefix, we don't need to apply the other arguments, we just
            # return what we have.
            pass
        else:
            # We have a non-None (for all intents and purposes it is a string)
            # prefix.
            if not delimiter:
                if not prefix:
                    # We have nothing more to do
                    pass
                else:
                    containers = filter_prefix(containers, prefix)
            else:
                containers = filter_delimiter(containers, delimiter, prefix,
                                              marker)

        count = 0
        for cont in containers:
            list_item = []
            metadata = None
            list_item.append(cont)
            cont_path = os.path.join(self.datadir, cont)
            metadata = _read_metadata(cont_path)
            if not metadata or not validate_container(metadata):
                try:
                    metadata = create_container_metadata(cont_path)
                except OSError as e:
                    # FIXME - total hack to get upstream swift ported unit
                    # test cases working for now.
                    if e.errno != errno.ENOENT:
                        raise
            if metadata:
                list_item.append(metadata[X_OBJECTS_COUNT][0])
                list_item.append(metadata[X_BYTES_USED][0])
                list_item.append(0)
            account_list.append(list_item)
            count += 1
            if count >= limit:
                break

        return account_list
Example #12
0
    def list_containers_iter(self, limit, marker, end_marker,
                             prefix, delimiter, response_content_type=None,
                             reverse=False):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''

        account_list = []
        containers = self._update_container_count()
        if containers:
            containers.sort()
        else:
            # No containers in account, return empty list
            return account_list

        if marker and end_marker and reverse:
            marker, end_marker = end_marker, marker

        if containers and end_marker:
            containers = filter_end_marker(containers, end_marker)

        if containers:
            if marker and marker >= prefix:
                containers = filter_marker(containers, marker)
            elif prefix:
                containers = filter_prefix_as_marker(containers, prefix)

        if prefix is None:
            # No prefix, we don't need to apply the other arguments, we just
            # return what we have.
            pass
        else:
            # We have a non-None (for all intents and purposes it is a string)
            # prefix.
            if not delimiter:
                if not prefix:
                    # We have nothing more to do
                    pass
                else:
                    containers = filter_prefix(containers, prefix)
            else:
                containers = filter_delimiter(containers, delimiter, prefix,
                                              marker)

        if response_content_type == 'text/plain' or \
                self.account == 'gsexpiring':
            # When response_content_type == 'text/plain':
            #
            # The client is only asking for a plain list of containers and NOT
            # asking for any extended information about container such as
            # bytes used or object count.
            #
            # When self.account == 'gsexpiring':
            # This is a JSON request sent by the object expirer to list
            # containers in gsexpiring volume. When out_content_type is
            # 'application/json', the caller expects each record entry to have
            # the following ordered fields:
            # (name, object_count, bytes_used, is_subdir)
            for container in containers:
                # When response_content_type == 'text/plain', Swift will
                # only consume the name of the container (first element of
                # tuple).Refer:
                # swift.account.utils.account_listing_response()
                account_list.append((container, 0, 0, 0, 0))
                if len(account_list) >= limit:
                    break
            if reverse:
                account_list.reverse()
            return account_list

        count = 0
        for cont in containers:
            list_item = []
            metadata = None
            list_item.append(cont)
            cont_path = os.path.join(self.datadir, cont)
            metadata = _read_metadata(cont_path)
            if not metadata or not validate_container(metadata):
                try:
                    metadata = create_container_metadata(cont_path)
                except OSError as e:
                    # FIXME - total hack to get upstream swift ported unit
                    # test cases working for now.
                    if e.errno not in (errno.ENOENT, errno.ESTALE):
                        raise
            if metadata:
                list_item.append(metadata[X_OBJECTS_COUNT][0])
                list_item.append(metadata[X_BYTES_USED][0])
                list_item.append(metadata[X_PUT_TIMESTAMP][0])
                list_item.append(0)
            account_list.append(list_item)
            count += 1
            if count >= limit:
                break
        if reverse:
            account_list.reverse()
        return account_list
Example #13
0
    def list_containers_iter(self, limit, marker, end_marker, prefix,
                             delimiter):
        """
        Return tuple of name, object_count, bytes_used, 0(is_subdir).
        Used by account server.
        """
        if delimiter and not prefix:
            prefix = ''

        account_list = []
        containers = self._update_container_count()
        if containers:
            containers.sort()
        else:
            return account_list

        if containers and end_marker:
            containers = filter_end_marker(containers, end_marker)

        if containers:
            if marker and marker >= prefix:
                containers = filter_marker(containers, marker)
            elif prefix:
                containers = filter_prefix_as_marker(containers, prefix)

        if prefix is None:
            # No prefix, we don't need to apply the other arguments, we just
            # return what we have.
            pass
        else:
            # We have a non-None (for all intents and purposes it is a string)
            # prefix.
            if not delimiter:
                if not prefix:
                    # We have nothing more to do
                    pass
                else:
                    containers = filter_prefix(containers, prefix)
            else:
                containers = filter_delimiter(containers, delimiter, prefix,
                                              marker)

        count = 0
        for cont in containers:
            list_item = []
            metadata = None
            list_item.append(cont)
            cont_path = os.path.join(self.datadir, cont)
            metadata = _read_metadata(cont_path)
            if not metadata or not validate_container(metadata):
                try:
                    metadata = create_container_metadata(cont_path)
                except OSError as e:
                    # FIXME - total hack to get upstream swift ported unit
                    # test cases working for now.
                    if e.errno != errno.ENOENT:
                        raise
            if metadata:
                list_item.append(metadata[X_OBJECTS_COUNT][0])
                list_item.append(metadata[X_BYTES_USED][0])
                list_item.append(0)
            account_list.append(list_item)
            count += 1
            if count >= limit:
                break

        return account_list