def __init__(self,
path,
drive,
account,
container,
logger,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
**kwargs):
super(DiskDir, self).__init__(path, drive, account, logger, **kwargs)
self.uid = int(uid)
self.gid = int(gid)
self.container = container
self.datadir = os.path.join(self.datadir, self.container)
if not self._dir_exists_read_metadata():
return
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def test_validate_container_good_type(self):
md = {utils.X_TYPE: (utils.CONTAINER, 0),
utils.X_TIMESTAMP: ('na', 0),
utils.X_PUT_TIMESTAMP: ('na', 0),
utils.X_OBJECTS_COUNT: ('na', 0),
utils.X_BYTES_USED: ('na', 0)}
ret = utils.validate_container(md)
assert ret
def test_validate_container_bad_type(self):
md = { utils.X_TYPE: ('bad', 0),
utils.X_TIMESTAMP: ('na', 0),
utils.X_PUT_TIMESTAMP: ('na', 0),
utils.X_OBJECTS_COUNT: ('na', 0),
utils.X_BYTES_USED: ('na', 0) }
ret = utils.validate_container(md)
assert ret == False
def test_validate_container_bad_type(self):
md = {
utils.X_TYPE: ("bad", 0),
utils.X_TIMESTAMP: ("na", 0),
utils.X_PUT_TIMESTAMP: ("na", 0),
utils.X_OBJECTS_COUNT: ("na", 0),
utils.X_BYTES_USED: ("na", 0),
}
ret = utils.validate_container(md)
assert not ret
def test_validate_container_bad_type(self):
md = {
utils.X_TYPE: ('bad', 0),
utils.X_TIMESTAMP: ('na', 0),
utils.X_PUT_TIMESTAMP: ('na', 0),
utils.X_OBJECTS_COUNT: ('na', 0),
utils.X_BYTES_USED: ('na', 0)
}
ret = utils.validate_container(md)
assert ret == False
def test_validate_container_good_type(self):
md = {
utils.X_TYPE: (utils.CONTAINER, 0),
utils.X_TIMESTAMP: ('na', 0),
utils.X_PUT_TIMESTAMP: ('na', 0),
utils.X_OBJECTS_COUNT: ('na', 0),
utils.X_BYTES_USED: ('na', 0)
}
ret = utils.validate_container(md)
assert ret
def __init__(self,
path,
account,
container,
logger,
uid=DEFAULT_UID,
gid=DEFAULT_GID):
self.root = path
if container:
self.container = container
else:
self.container = None
if self.container:
self.datadir = os.path.join(path, account, self.container)
else:
self.datadir = os.path.join(path, account)
# Note that the account name has a one-to-one mapping to the gluster
# mount point, or volume name.
self.account = account
assert logger is not None
self.logger = logger
self.metadata = {}
self.container_info = None
self.object_info = None
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os.path.exists(self.datadir)
if self.dir_exists:
try:
self.metadata = _read_metadata(self.datadir)
except EOFError:
create_container_metadata(self.datadir)
else:
return
if self.container:
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def list_containers_iter(self, limit, marker, end_marker, prefix,
delimiter):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
self.update_container_count()
containers, container_count = self.container_info
if containers:
containers.sort()
if containers and prefix:
containers = self.filter_prefix(containers, prefix)
if containers and delimiter:
containers = self.filter_delimiter(containers, delimiter, prefix)
if containers and marker:
containers = self.filter_marker(containers, marker)
if containers and end_marker:
containers = self.filter_end_marker(containers, end_marker)
if containers and limit:
if len(containers) > limit:
containers = self.filter_limit(containers, limit)
account_list = []
if containers:
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = os.path.join(self.datadir, cont)
metadata = _read_metadata(cont_path)
if not metadata or not validate_container(metadata):
metadata = create_container_metadata(cont_path)
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT][0])
list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
return account_list
def list_containers_iter(self, limit, marker, end_marker,
prefix, delimiter):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
self.update_container_count()
containers, container_count = self.container_info
if containers:
containers.sort()
if containers and prefix:
containers = self.filter_prefix(containers, prefix)
if containers and delimiter:
containers = self.filter_delimiter(containers, delimiter, prefix)
if containers and marker:
containers = self.filter_marker(containers, marker)
if containers and end_marker:
containers = self.filter_end_marker(containers, end_marker)
if containers and limit:
if len(containers) > limit:
containers = self.filter_limit(containers, limit)
account_list = []
if containers:
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = os.path.join(self.datadir, cont)
metadata = _read_metadata(cont_path)
if not metadata or not validate_container(metadata):
metadata = create_container_metadata(cont_path)
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT][0])
list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
return account_list
def __init__(self, path, account, container, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.root = path
if container:
self.container = container
else:
self.container = None
if self.container:
self.datadir = os.path.join(path, account, self.container)
else:
self.datadir = os.path.join(path, account)
# Note that the account name has a one-to-one mapping to the gluster
# mount point, or volume name.
self.account = account
assert logger is not None
self.logger = logger
self.metadata = {}
self.container_info = None
self.object_info = None
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os.path.exists(self.datadir)
if self.dir_exists:
try:
self.metadata = _read_metadata(self.datadir)
except EOFError:
create_container_metadata(self.datadir)
else:
return
if self.container:
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def _initialize(self):
if self.container:
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def __init__(self, path, drive, account, container, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.root = path
if container:
self.container = container
else:
self.container = None
if self.container:
self.datadir = os.path.join(path, drive, self.container)
else:
self.datadir = os.path.join(path, drive)
self.account = account
assert logger is not None
self.logger = logger
self.metadata = {}
self.container_info = None
self.object_info = None
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os.path.exists(self.datadir)
if self.dir_exists:
try:
self.metadata = _read_metadata(self.datadir)
except EOFError:
create_container_metadata(self.datadir)
else:
return
if self.container:
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def __init__(self, path, drive, account, container, logger, uid=DEFAULT_UID, gid=DEFAULT_GID):
super(DiskDir, self).__init__(path, drive, account, logger)
self.uid = int(uid)
self.gid = int(gid)
self.container = container
self.datadir = os.path.join(self.datadir, self.container)
if not self._dir_exists_read_metadata():
return
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def __init__(self, path, drive, account, container, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs):
super(DiskDir, self).__init__(path, drive, account, logger, **kwargs)
self.uid = int(uid)
self.gid = int(gid)
self.container = container
self.datadir = os.path.join(self.datadir, self.container)
if self.account == 'gsexpiring':
# Do not bother crawling the entire container tree just to update
# object count and bytes used. Return immediately before metadata
# validation and creation happens.
info = do_stat(self.datadir)
if info and stat.S_ISDIR(info.st_mode):
self._dir_exists = True
if not info:
# Container no longer exists.
return
semi_fake_md = {
'X-Object-Count': (0, 0),
'X-Timestamp': ((normalize_timestamp(info.st_ctime)), 0),
'X-Type': ('container', 0),
'X-PUT-Timestamp': ((normalize_timestamp(info.st_mtime)), 0),
'X-Bytes-Used': (0, 0)
}
self.metadata = semi_fake_md
return
if not self._dir_exists_read_metadata():
return
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def list_containers_iter(self, limit, marker, end_marker,
prefix, delimiter):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
account_list = []
containers = self._update_container_count()
if containers:
containers.sort()
else:
return account_list
if containers and end_marker:
containers = filter_end_marker(containers, end_marker)
if containers:
if marker and marker >= prefix:
containers = filter_marker(containers, marker)
elif prefix:
containers = filter_prefix_as_marker(containers, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
containers = filter_prefix(containers, prefix)
else:
containers = filter_delimiter(containers, delimiter, prefix,
marker)
count = 0
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = os.path.join(self.datadir, cont)
metadata = _read_metadata(cont_path)
if not metadata or not validate_container(metadata):
try:
metadata = create_container_metadata(cont_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno != errno.ENOENT:
raise
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT][0])
list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
count += 1
if count >= limit:
break
return account_list
def test_validate_container_missing_keys(self):
ret = utils.validate_container({'foo': 'bar'})
assert not ret
def test_validate_container_empty(self):
ret = utils.validate_container({})
assert not ret
def test_validate_container_missing_keys(self):
ret = utils.validate_container({"foo": "bar"})
assert not ret
def list_containers_iter(self, limit, marker, end_marker, prefix,
delimiter):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
account_list = []
containers = self._update_container_count()
if containers:
containers.sort()
else:
return account_list
if containers and end_marker:
containers = filter_end_marker(containers, end_marker)
if containers:
if marker and marker >= prefix:
containers = filter_marker(containers, marker)
elif prefix:
containers = filter_prefix_as_marker(containers, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
containers = filter_prefix(containers, prefix)
else:
containers = filter_delimiter(containers, delimiter, prefix,
marker)
count = 0
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = os.path.join(self.datadir, cont)
metadata = _read_metadata(cont_path)
if not metadata or not validate_container(metadata):
try:
metadata = create_container_metadata(cont_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno != errno.ENOENT:
raise
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT][0])
list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
count += 1
if count >= limit:
break
return account_list
def list_containers_iter(self, limit, marker, end_marker,
prefix, delimiter, response_content_type=None,
reverse=False):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
account_list = []
containers = self._update_container_count()
if containers:
containers.sort()
else:
# No containers in account, return empty list
return account_list
if marker and end_marker and reverse:
marker, end_marker = end_marker, marker
if containers and end_marker:
containers = filter_end_marker(containers, end_marker)
if containers:
if marker and marker >= prefix:
containers = filter_marker(containers, marker)
elif prefix:
containers = filter_prefix_as_marker(containers, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
containers = filter_prefix(containers, prefix)
else:
containers = filter_delimiter(containers, delimiter, prefix,
marker)
if response_content_type == 'text/plain' or \
self.account == 'gsexpiring':
# When response_content_type == 'text/plain':
#
# The client is only asking for a plain list of containers and NOT
# asking for any extended information about container such as
# bytes used or object count.
#
# When self.account == 'gsexpiring':
# This is a JSON request sent by the object expirer to list
# containers in gsexpiring volume. When out_content_type is
# 'application/json', the caller expects each record entry to have
# the following ordered fields:
# (name, object_count, bytes_used, is_subdir)
for container in containers:
# When response_content_type == 'text/plain', Swift will
# only consume the name of the container (first element of
# tuple).Refer:
# swift.account.utils.account_listing_response()
account_list.append((container, 0, 0, 0, 0))
if len(account_list) >= limit:
break
if reverse:
account_list.reverse()
return account_list
count = 0
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = os.path.join(self.datadir, cont)
metadata = _read_metadata(cont_path)
if not metadata or not validate_container(metadata):
try:
metadata = create_container_metadata(cont_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT][0])
list_item.append(metadata[X_BYTES_USED][0])
list_item.append(metadata[X_PUT_TIMESTAMP][0])
list_item.append(0)
account_list.append(list_item)
count += 1
if count >= limit:
break
if reverse:
account_list.reverse()
return account_list
def test_validate_container_missing_keys(self):
ret = utils.validate_container({'foo': 'bar'})
assert not ret
def test_validate_container_empty(self):
ret = utils.validate_container({})
assert not ret