def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs): super(DiskDir, self).__init__(path, drive, account, logger, **kwargs) self.uid = int(uid) self.gid = int(gid) self.container = container self.datadir = os.path.join(self.datadir, self.container) if not self._dir_exists_read_metadata(): return if not self.metadata: create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir)
def _initialize(self): if self.container: if not self.metadata: create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not self.metadata: create_account_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_account(self.metadata): create_account_metadata(self.datadir) self.metadata = _read_metadata(self.datadir)
def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID): super(DiskDir, self).__init__(path, drive, account, logger) self.uid = int(uid) self.gid = int(gid) self.container = container self.datadir = os.path.join(self.datadir, self.container) if not self._dir_exists_read_metadata(): return if not self.metadata: create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir)
def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs): super(DiskDir, self).__init__(path, drive, account, logger, **kwargs) self.uid = int(uid) self.gid = int(gid) self.container = container self.datadir = os.path.join(self.datadir, self.container) if self.account == 'gsexpiring': # Do not bother crawling the entire container tree just to update # object count and bytes used. Return immediately before metadata # validation and creation happens. info = do_stat(self.datadir) if info and stat.S_ISDIR(info.st_mode): self._dir_exists = True if not info: # Container no longer exists. return semi_fake_md = { 'X-Object-Count': (0, 0), 'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0), 'X-Type': ('container', 0), 'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0), 'X-Bytes-Used': (0, 0) } self.metadata = semi_fake_md return if not self._dir_exists_read_metadata(): return if not self.metadata: create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir)
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ Return tuple of name, object_count, bytes_used, 0(is_subdir). Used by account server. """ if delimiter and not prefix: prefix = '' self.update_container_count() containers, container_count = self.container_info if containers: containers.sort() if containers and prefix: containers = self.filter_prefix(containers, prefix) if containers and delimiter: containers = self.filter_delimiter(containers, delimiter, prefix) if containers and marker: containers = self.filter_marker(containers, marker) if containers and end_marker: containers = self.filter_end_marker(containers, end_marker) if containers and limit: if len(containers) > limit: containers = self.filter_limit(containers, limit) account_list = [] if containers: for cont in containers: list_item = [] metadata = None list_item.append(cont) cont_path = os.path.join(self.datadir, cont) metadata = _read_metadata(cont_path) if not metadata or not validate_container(metadata): metadata = create_container_metadata(cont_path) if metadata: list_item.append(metadata[X_OBJECTS_COUNT][0]) list_item.append(metadata[X_BYTES_USED][0]) list_item.append(0) account_list.append(list_item) return account_list
def __init__(self, path, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path if container: self.container = container else: self.container = None if self.container: self.datadir = os.path.join(path, account, self.container) else: self.datadir = os.path.join(path, account) # Note that the account name has a one-to-one mapping to the gluster # mount point, or volume name. self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) if self.dir_exists: try: self.metadata = _read_metadata(self.datadir) except EOFError: create_container_metadata(self.datadir) else: return if self.container: if not self.metadata: create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not self.metadata: create_account_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_account(self.metadata): create_account_metadata(self.datadir) self.metadata = _read_metadata(self.datadir)
def test_create_container_metadata(self): td = tempfile.mkdtemp() try: r_md = utils.create_container_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.cont_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == (utils.CONTAINER, 0) assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0) assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0) assert md[utils.X_OBJECTS_COUNT] == (0, 0) assert md[utils.X_BYTES_USED] == (0, 0) finally: os.rmdir(td)
def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID): self.root = path if container: self.container = container else: self.container = None if self.container: self.datadir = os.path.join(path, drive, self.container) else: self.datadir = os.path.join(path, drive) self.account = account assert logger is not None self.logger = logger self.metadata = {} self.container_info = None self.object_info = None self.uid = int(uid) self.gid = int(gid) self.db_file = _db_file self.dir_exists = os.path.exists(self.datadir) if self.dir_exists: try: self.metadata = _read_metadata(self.datadir) except EOFError: create_container_metadata(self.datadir) else: return if self.container: if not self.metadata: create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_container(self.metadata): create_container_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not self.metadata: create_account_metadata(self.datadir) self.metadata = _read_metadata(self.datadir) else: if not validate_account(self.metadata): create_account_metadata(self.datadir) self.metadata = _read_metadata(self.datadir)
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ Return tuple of name, object_count, bytes_used, 0(is_subdir). Used by account server. """ if delimiter and not prefix: prefix = '' account_list = [] containers = self._update_container_count() if containers: containers.sort() else: return account_list if containers and end_marker: containers = filter_end_marker(containers, end_marker) if containers: if marker and marker >= prefix: containers = filter_marker(containers, marker) elif prefix: containers = filter_prefix_as_marker(containers, prefix) if prefix is None: # No prefix, we don't need to apply the other arguments, we just # return what we have. pass else: # We have a non-None (for all intents and purposes it is a string) # prefix. if not delimiter: if not prefix: # We have nothing more to do pass else: containers = filter_prefix(containers, prefix) else: containers = filter_delimiter(containers, delimiter, prefix, marker) count = 0 for cont in containers: list_item = [] metadata = None list_item.append(cont) cont_path = os.path.join(self.datadir, cont) metadata = _read_metadata(cont_path) if not metadata or not validate_container(metadata): try: metadata = create_container_metadata(cont_path) except OSError as e: # FIXME - total hack to get upstream swift ported unit # test cases working for now. if e.errno != errno.ENOENT: raise if metadata: list_item.append(metadata[X_OBJECTS_COUNT][0]) list_item.append(metadata[X_BYTES_USED][0]) list_item.append(0) account_list.append(list_item) count += 1 if count >= limit: break return account_list
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter, response_content_type=None, reverse=False): """ Return tuple of name, object_count, bytes_used, 0(is_subdir). Used by account server. """ if delimiter and not prefix: prefix = '' account_list = [] containers = self._update_container_count() if containers: containers.sort() else: # No containers in account, return empty list return account_list if marker and end_marker and reverse: marker, end_marker = end_marker, marker if containers and end_marker: containers = filter_end_marker(containers, end_marker) if containers: if marker and marker >= prefix: containers = filter_marker(containers, marker) elif prefix: containers = filter_prefix_as_marker(containers, prefix) if prefix is None: # No prefix, we don't need to apply the other arguments, we just # return what we have. pass else: # We have a non-None (for all intents and purposes it is a string) # prefix. if not delimiter: if not prefix: # We have nothing more to do pass else: containers = filter_prefix(containers, prefix) else: containers = filter_delimiter(containers, delimiter, prefix, marker) if response_content_type == 'text/plain' or \ self.account == 'gsexpiring': # When response_content_type == 'text/plain': # # The client is only asking for a plain list of containers and NOT # asking for any extended information about container such as # bytes used or object count. # # When self.account == 'gsexpiring': # This is a JSON request sent by the object expirer to list # containers in gsexpiring volume. When out_content_type is # 'application/json', the caller expects each record entry to have # the following ordered fields: # (name, object_count, bytes_used, is_subdir) for container in containers: # When response_content_type == 'text/plain', Swift will # only consume the name of the container (first element of # tuple).Refer: # swift.account.utils.account_listing_response() account_list.append((container, 0, 0, 0, 0)) if len(account_list) >= limit: break if reverse: account_list.reverse() return account_list count = 0 for cont in containers: list_item = [] metadata = None list_item.append(cont) cont_path = os.path.join(self.datadir, cont) metadata = _read_metadata(cont_path) if not metadata or not validate_container(metadata): try: metadata = create_container_metadata(cont_path) except OSError as e: # FIXME - total hack to get upstream swift ported unit # test cases working for now. if e.errno not in (errno.ENOENT, errno.ESTALE): raise if metadata: list_item.append(metadata[X_OBJECTS_COUNT][0]) list_item.append(metadata[X_BYTES_USED][0]) list_item.append(metadata[X_PUT_TIMESTAMP][0]) list_item.append(0) account_list.append(list_item) count += 1 if count >= limit: break if reverse: account_list.reverse() return account_list