class ObjectStorageApiTestBase(BaseTestCase):
def setUp(self):
super(ObjectStorageApiTestBase, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(ObjectStorageApiTestBase, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
def _create(self, name, metadata=None):
return self.api.container_create(self.account,
name,
properties=metadata)
def _delete(self, name):
self.api.container_delete(self.account, name)
def _clean(self, name, clear=False):
if clear:
# must clean properties before
self.api.container_del_properties(self.account, name, [])
self._delete(name)
def _get_properties(self, name, properties=None):
return self.api.container_get_properties(self.account,
name,
properties=properties)
def _set_properties(self, name, properties=None):
return self.api.container_set_properties(self.account,
name,
properties=properties)
def _upload_empty(self, container, *objs, **kwargs):
"""Upload empty objects to `container`"""
for obj in objs:
self.api.object_create(self.account,
container,
obj_name=obj,
data="",
**kwargs)
self.created.append((container, obj))
class TestContentVersioning(BaseTestCase):
def setUp(self):
super(TestContentVersioning, self).setUp()
self.api = ObjectStorageApi(self.conf['namespace'])
self.container = random_str(8)
system = {'sys.m2.policy.version': '3'}
self.api.container_create(self.account, self.container, system=system)
def test_versioning_enabled(self):
props = self.api.container_get_properties(
self.account, self.container)
self.assertEqual('3', props['system']['sys.m2.policy.version'])
def test_list_versions(self):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(2, len(objects))
self.assertNotEqual(objects[0]['version'], objects[1]['version'])
def test_purge(self):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content2")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content3")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, lambda x: x['version'])
self.api.container.container_purge(self.account, self.container)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
class ContainerBackup(RedisConnection, WerkzeugApp):
"""WSGI Application to dump or restore a container."""
REDIS_TIMEOUT = 3600 * 24 # Redis keys will expire after one day
STREAMING = 52428800 # 50 MB
# Number of blocks to serve to avoid splitting headers (1MiB)
BLOCK_ALIGNMENT = 2048
def __init__(self, conf):
if conf:
self.conf = read_conf(conf['key_file'],
section_name="admin-server")
else:
self.conf = {}
self.logger = get_logger(self.conf, name="ContainerBackup")
self.proxy = ObjectStorageApi(self.conf.get("namespace", NS),
logger=self.logger)
self.url_map = Map([
Rule('/v1.0/container/dump', endpoint='dump'),
Rule('/v1.0/container/restore', endpoint='restore'),
])
self.REDIS_TIMEOUT = self.conf.get("redis_cache_timeout",
self.REDIS_TIMEOUT)
redis_conf = {
k[6:]: v
for k, v in self.conf.items() if k.startswith("redis_")
}
redis_host = redis_conf.pop('host', None)
if redis_host:
parsed = urlparse('http://' + redis_host)
if parsed.port is None:
redis_host = '%s:%s' % (redis_host,
redis_conf.pop('port', '6379'))
redis_sentinel_hosts = redis_conf.pop(
'sentinel_hosts',
# TODO(adu): Delete when it will no longer be used
self.conf.get('sentinel_hosts'))
redis_sentinel_name = redis_conf.pop(
'sentinel_name',
# TODO(adu): Delete when it will no longer be used
self.conf.get('sentinel_master_name'))
RedisConnection.__init__(self,
host=redis_host,
sentinel_hosts=redis_sentinel_hosts,
sentinel_name=redis_sentinel_name,
**redis_conf)
WerkzeugApp.__init__(self, self.url_map, self.logger)
@property
def redis(self):
"""Redis connection object"""
return self.conn
@redis_cnx
def generate_manifest(self, account, container):
"""
Generate a static manifest of a container.
It will help to find quickly which part of object app have to serve
Manifest is cached into Redis with REDIS_TIMEOUT delay
"""
if not container:
raise exc.NoSuchContainer()
# TODO hash_map should contains if deleted or version flags are set
hash_map = "container_streaming:{0}/{1}".format(account, container)
cache = self.redis.get(hash_map)
if cache:
self.logger.debug("using cache")
return json.loads(cache, object_pairs_hook=OrderedDict)
map_objs = []
start_block = 0
meta = self.proxy.container_get_properties(account, container)
if meta['properties']:
# create special file to save properties of container
tar = OioTarEntry(self.proxy,
account,
container,
CONTAINER_PROPERTIES,
data=meta)
entry = {
'name': CONTAINER_PROPERTIES,
'size': tar.filesize,
'hdr_blocks': tar.header_blocks,
'blocks': tar.header_blocks + tar.data_blocks,
'start_block': start_block,
}
start_block += entry['blocks']
entry['end_block'] = start_block - 1
map_objs.append(entry)
objs = self.proxy.object_list(account, container)
for obj in sorted(objs['objects'], key=lambda x: x['name']):
# FIXME: should we backup deleted objects?
if obj['deleted']:
continue
tar = OioTarEntry(self.proxy, account, container, obj['name'])
if (start_block / self.BLOCK_ALIGNMENT) != \
((start_block + tar.header_blocks) / self.BLOCK_ALIGNMENT):
# header is over boundary, we have to add padding blocks
padding = (self.BLOCK_ALIGNMENT -
divmod(start_block, self.BLOCK_ALIGNMENT)[1])
map_objs.append({
'blocks': padding,
'size': padding * BLOCKSIZE,
'start_block': start_block,
'slo': None,
'hdr_blocks': padding,
'end_block': start_block + padding - 1
})
start_block += padding
entry = {
'name': obj['name'],
'size': tar.filesize,
'hdr_blocks': tar.header_blocks,
'blocks': tar.header_blocks + tar.data_blocks,
'start_block': start_block,
'slo': tar.slo,
'checksums': tar.checksums,
}
start_block += entry['blocks']
entry['end_block'] = start_block - 1
map_objs.append(entry)
if not map_objs:
return map_objs
entry = {
'name': CONTAINER_MANIFEST,
'size': 0,
'hdr_blocks': 1, # a simple PAX header consume only 1 block
'blocks': 0,
'start_block': 0,
'slo': None,
}
map_objs.insert(0, entry)
entry['size'] = len(json.dumps(map_objs, sort_keys=True))
# ensure that we reserved enough blocks after recomputing offset
entry['blocks'] = \
1 + int(math.ceil(entry['size'] / float(BLOCKSIZE))) * 2
tar = OioTarEntry(self.proxy,
account,
container,
CONTAINER_MANIFEST,
data=map_objs)
assert tar.header_blocks == 1, "Incorrect size for hdr_blocks"
assert tar.data_blocks <= entry['blocks']
# fix start_block and end_block
start = 0
for _entry in map_objs:
_entry['start_block'] = start
start += _entry['blocks']
_entry['end_block'] = start - 1
tar2 = OioTarEntry(self.proxy,
account,
container,
CONTAINER_MANIFEST,
data=map_objs)
entry['size'] = tar2.filesize
assert tar2.header_blocks == tar.header_blocks
assert tar2.data_blocks <= entry['blocks'], \
"got %d instead of %d" % (tar2.data_blocks, tar.data_blocks)
self.logger.debug("add entry to cache")
self.redis.set(hash_map,
json.dumps(map_objs, sort_keys=True),
ex=self.REDIS_TIMEOUT)
return map_objs
def _do_head(self, _, account, container):
"""
Manage HEAD method and response number of block
Note: Range header is unmanaged
"""
try:
results = self.generate_manifest(account, container)
except exc.NoSuchContainer:
self.logger.info("%s %s not found", account, container)
return Response(status=404)
if not results:
self.logger.info("no data for %s %s", account, container)
return Response(status=204)
hdrs = {
'X-Blocks': sum([i['blocks'] for i in results]),
'Content-Length': sum([i['blocks'] for i in results]) * BLOCKSIZE,
'Accept-Ranges': 'bytes',
'Content-Type': 'application/tar',
}
return Response(headers=hdrs, status=200)
@classmethod
def _extract_range(cls, req, blocks):
"""Convert byte range into block an performs validity check"""
# accept only single part range
val = req.headers['Range']
match = RANGE_RE.match(val)
if match is None:
raise RequestedRangeNotSatisfiable()
start = int(match.group(1))
end = int(match.group(2))
if start >= end:
raise RequestedRangeNotSatisfiable()
def check_range(value):
block, remainder = divmod(value, BLOCKSIZE)
if remainder or block < 0 or (blocks and block > blocks):
raise RequestedRangeNotSatisfiable()
return block
block_start = check_range(start)
block_end = check_range(end + 1) # Check Range RFC
return start, end, block_start, block_end
def _do_get(self, req, account, container):
"""Manage GET method to dump a container"""
try:
results = self.generate_manifest(account, container)
except exc.NoSuchContainer:
self.logger.info("%s %s not found", account, container)
return Response(status=404)
if not results:
self.logger.info("no data for %s %s", account, container)
return Response(status=204)
blocks = sum([i['blocks'] for i in results])
length = blocks * BLOCKSIZE
if 'Range' not in req.headers:
tar = ContainerTarFile(self.proxy, account, container,
(0, blocks - 1), results, self.logger)
return Response(wrap_file(req.environ,
tar,
buffer_size=self.STREAMING),
headers={
'Accept-Ranges': 'bytes',
'Content-Type': 'application/tar',
'Content-Length': length,
},
status=200)
start, end, block_start, block_end = self._extract_range(req, blocks)
tar = ContainerTarFile(self.proxy, account, container,
(block_start, block_end - 1), results,
self.logger)
return Response(wrap_file(req.environ, tar,
buffer_size=self.STREAMING),
headers={
'Accept-Ranges':
'bytes',
'Content-Type':
'application/tar',
'Content-Range':
'bytes %d-%d/%d' % (start, end, length),
'Content-Length':
end - start + 1,
},
status=206)
def on_dump(self, req):
"""Entry point for dump rule"""
# extract account and container
account = req.args.get('acct')
container = req.args.get('ref')
if not account:
raise BadRequest('Missing Account name')
if not container:
raise BadRequest('Missing Container name')
if req.method == 'HEAD':
return self._do_head(req, account, container)
if req.method == 'GET':
return self._do_get(req, account, container)
return Response("Not supported", 405)
@redis_cnx
def _do_put_head(self, req, account, container):
results = self.redis.get("restore:%s:%s" % (account, container))
if not results:
return UnprocessableEntity("No restoration in progress")
in_progress = self.redis.get('restore:%s:%s:lock' %
(account, container)) or '0'
results = json.loads(results)
blocks = sum(i['blocks'] for i in results['manifest'])
return Response(headers={
'X-Tar-Size': blocks * BLOCKSIZE,
'X-Consumed-Size': results['end'] * BLOCKSIZE,
'X-Upload-In-Progress': in_progress
},
status=200)
@redis_cnx
def _do_put(self, req, account, container):
"""Manage PUT method for restoring a container"""
obj = ContainerRestore(self.redis, self.proxy, self.logger)
key = "restore:%s:%s:lock" % (account, container)
if not self.redis.set(key, 1, nx=True):
raise UnprocessableEntity("A restore is already in progress")
try:
return obj.restore(req, account, container)
finally:
self.redis.delete(key)
def on_restore(self, req):
"""Entry point for restore rule"""
account = req.args.get('acct')
container = req.args.get('ref')
if not account:
raise BadRequest('Missing Account name')
if not container:
raise BadRequest('Missing Container name')
if req.method not in ('PUT', 'HEAD'):
return Response("Not supported", 405)
try:
self.proxy.container_get_properties(account, container)
if not req.headers.get('range') and req.method == 'PUT':
raise Conflict('Container already exists')
except exc.NoSuchContainer:
pass
except Conflict:
raise
except Exception:
raise BadRequest('Fail to verify container')
if req.method == 'HEAD':
return self._do_put_head(req, account, container)
return self._do_put(req, account, container)
class TestContentVersioning(BaseTestCase):
def setUp(self):
super(TestContentVersioning, self).setUp()
self.api = ObjectStorageApi(self.conf['namespace'])
self.container = random_str(8)
system = {'sys.m2.policy.version': '3'}
self.api.container_create(self.account, self.container, system=system)
def test_versioning_enabled(self):
props = self.api.container_get_properties(
self.account, self.container)
self.assertEqual('3', props['system']['sys.m2.policy.version'])
def test_list_versions(self):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(2, len(objects))
self.assertNotEqual(objects[0]['version'], objects[1]['version'])
def test_container_purge(self):
# many contents
for i in range(0, 4):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the container configuration
self.api.container_purge(self.account, self.container)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the request
self.api.container_purge(self.account, self.container, maxvers=1)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(1, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
def test_content_purge(self):
# many contents
for i in range(0, 4):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the container configuration
self.api.container.content_purge(self.account, self.container,
"versioned")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the request
self.api.container.content_purge(self.account, self.container,
"versioned", maxvers=1)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(1, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
# other contents
for i in range(0, 4):
self.api.object_create(self.account, self.container,
obj_name="versioned2",
data="content"+str(i))
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(5, len(objects))
# use the maxvers of the container configuration
self.api.container.content_purge(self.account, self.container,
"versioned")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(5, len(objects))
def test_delete_exceeding_version(self):
def check_num_objects_and_get_oldest_version(expected):
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(expected, len(objects))
return min(objects, key=lambda x: x['version'])
system = {'sys.m2.policy.version.delete_exceeding': '1'}
self.api.container_set_properties(self.account, self.container,
system=system)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content2")
oldest_version = check_num_objects_and_get_oldest_version(3)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content3")
new_oldest_version = check_num_objects_and_get_oldest_version(3)
self.assertLess(oldest_version['version'],
new_oldest_version['version'])
def test_change_flag_delete_exceeding_versions(self):
def check_num_objects(expected):
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(expected, len(objects))
for i in range(5):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content"+str(i))
check_num_objects(5)
system = {'sys.m2.policy.version.delete_exceeding': '1'}
self.api.container_set_properties(self.account, self.container,
system=system)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content5")
check_num_objects(3)
for i in range(6, 10):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content"+str(i))
check_num_objects(3)
system['sys.m2.policy.version.delete_exceeding'] = '0'
self.api.container_set_properties(self.account, self.container,
system=system)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content11")
check_num_objects(4)
class TestMeta2Database(BaseTestCase):
def setUp(self):
super(TestMeta2Database, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_meta2_database"
self.reference = "meta2_database_" + random_str(4)
self.meta2_database = Meta2Database(self.conf)
self.service_type = 'meta2'
def _get_peers(self):
linked_services = self.api.directory.list(self.account, self.reference)
peers = list()
for service in linked_services['srv']:
if service['type'] == self.service_type:
peers.append(service['host'])
return peers
def _test_move(self, base=None, fixed_dst=True):
if base is None:
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
all_meta2_services = self.conscience.all_services(
self.service_type, True)
if len(all_meta2_services) <= len(current_peers):
self.skipTest("need at least %d more %s"
% (len(current_peers)+1, self.service_type))
expected_peers = list(current_peers)
src = random.choice(current_peers)
expected_peers.remove(src)
dst = None
if fixed_dst:
for service in all_meta2_services:
if service['id'] not in current_peers:
dst = service['id']
expected_peers.append(dst)
moved = self.meta2_database.move(base, src, dst=dst)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
if fixed_dst:
self.assertEqual(dst, moved[0]['dst'])
self.assertIsNone(moved[0]['err'])
new_peers = self._get_peers()
if fixed_dst:
self.assertListEqual(sorted(expected_peers), sorted(new_peers))
else:
for expected_service in expected_peers:
self.assertIn(expected_service, new_peers)
self.assertNotIn(src, new_peers)
self.assertEqual(len(expected_peers)+1, len(new_peers))
if self.service_type == 'meta2':
properties = self.api.container_get_properties(
self.account, self.reference)
peers = properties['system']['sys.peers']
new_peers_bis = peers.split(',')
self.assertListEqual(sorted(new_peers), sorted(new_peers_bis))
return (src, expected_peers)
def test_move(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
self._test_move()
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_with_seq(self):
self.api.container_create(self.account, self.reference)
properties = self.api.container_get_properties(
self.account, self.reference)
base = properties['system']['sys.name']
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
self._test_move(base=base)
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_without_dst(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
self._test_move(fixed_dst=False)
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_with_src_not_used(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
src = None
all_meta2_services = self.conscience.all_services('meta2', True)
for service in all_meta2_services:
if service['id'] not in current_peers:
src = service['id']
if src is None:
self.skipTest("need at least 1 more meta2")
moved = self.meta2_database.move(base, src)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertIsNone(moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_dst_already_used(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
src = random.choice(current_peers)
dst = random.choice(current_peers)
moved = self.meta2_database.move(base, src, dst=dst)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertEqual(dst, moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_invalid_src(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
src = '127.0.0.1:666'
moved = self.meta2_database.move(base, src)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertIsNone(moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_invalid_dst(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
src = random.choice(current_peers)
dst = '127.0.0.1:666'
moved = self.meta2_database.move(base, src, dst=dst)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertEqual(dst, moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_1_missing_base(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
if len(current_peers) <= 1:
self.skipTest('need replicated bases')
to_remove = random.choice(current_peers)
self.admin.remove_base(self.service_type, cid=base,
service_id=to_remove)
self._test_move()
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_with_1_remaining_base(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
if len(current_peers) <= 1:
self.skipTest('need replicated bases')
to_remove = list(current_peers)
to_remove.remove(random.choice(current_peers))
self.admin.remove_base(self.service_type, cid=base,
service_id=to_remove)
self._test_move()
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_sqlx(self):
self.meta2_database = Meta2Database(self.conf, service_type='sqlx')
self.service_type = 'sqlx'
execute('oio-sqlx -O AutoCreate %s/%s/%s '
'"create table foo (a INT, b TEXT)"'
% (self.ns, self.account, self.reference))
self._test_move()
class TestContentVersioning(BaseTestCase):
def setUp(self):
super(TestContentVersioning, self).setUp()
self.api = ObjectStorageApi(self.conf['namespace'])
self.container = random_str(8)
system = {'sys.m2.policy.version': '3'}
self.wait_for_score(('meta2', ))
self.api.container_create(self.account, self.container, system=system)
def test_versioning_enabled(self):
props = self.api.container_get_properties(self.account, self.container)
self.assertEqual('3', props['system']['sys.m2.policy.version'])
def test_list_versions(self):
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content0")
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content1")
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(2, len(objects))
self.assertNotEqual(objects[0]['version'], objects[1]['version'])
def test_container_purge(self):
# many contents
for i in range(0, 4):
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content")
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the container configuration
self.api.container_purge(self.account, self.container)
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the request
self.api.container_purge(self.account, self.container, maxvers=1)
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(1, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
def test_content_purge(self):
# many contents
for i in range(0, 4):
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content")
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the container configuration
self.api.container.content_purge(self.account, self.container,
"versioned")
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the request
self.api.container.content_purge(self.account,
self.container,
"versioned",
maxvers=1)
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(1, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
# other contents
for i in range(0, 4):
self.api.object_create(self.account,
self.container,
obj_name="versioned2",
data="content" + str(i))
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(5, len(objects))
# use the maxvers of the container configuration
self.api.container.content_purge(self.account, self.container,
"versioned")
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(5, len(objects))
def test_delete_exceeding_version(self):
def check_num_objects_and_get_oldest_version(expected_objects,
expected_deleted_aliases,
oldest_version):
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
nb_objects = 0
nb_deleted = 0
new_oldest_version = 0
for obj in objects:
if obj['deleted']:
nb_deleted += 1
else:
nb_objects += 1
if new_oldest_version == 0 \
or new_oldest_version > obj['version']:
new_oldest_version = obj['version']
self.assertEqual(expected_objects, nb_objects)
self.assertEqual(expected_deleted_aliases, nb_deleted)
if oldest_version is not None:
self.assertLess(oldest_version, new_oldest_version)
return new_oldest_version
system = {'sys.m2.policy.version.delete_exceeding': '1'}
self.api.container_set_properties(self.account,
self.container,
system=system)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content0")
oldest_version = check_num_objects_and_get_oldest_version(1, 0, None)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content1")
self.assertEqual(oldest_version,
check_num_objects_and_get_oldest_version(2, 0, None))
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content2")
self.assertEqual(oldest_version,
check_num_objects_and_get_oldest_version(3, 0, None))
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content3")
oldest_version = check_num_objects_and_get_oldest_version(
3, 0, oldest_version)
self.api.object_delete(self.account, self.container, "versioned")
self.assertEqual(oldest_version,
check_num_objects_and_get_oldest_version(3, 1, None))
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content4")
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content5")
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content6")
# FIXME(adu) The deleted alias should be deleted at the same time
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content7")
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
def test_change_flag_delete_exceeding_versions(self):
def check_num_objects(expected):
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(expected, len(objects))
for i in range(5):
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content" + str(i))
check_num_objects(5)
system = {'sys.m2.policy.version.delete_exceeding': '1'}
self.api.container_set_properties(self.account,
self.container,
system=system)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content5")
check_num_objects(3)
for i in range(6, 10):
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content" + str(i))
check_num_objects(3)
system['sys.m2.policy.version.delete_exceeding'] = '0'
self.api.container_set_properties(self.account,
self.container,
system=system)
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content11")
check_num_objects(4)
def test_purge_objects_with_delete_marker(self):
def check_num_objects(expected):
listing = self.api.object_list(self.account,
self.container,
versions=True)
objects = listing['objects']
self.assertEqual(expected, len(objects))
for i in range(5):
self.api.object_create(self.account,
self.container,
obj_name="versioned",
data="content" + str(i))
check_num_objects(5)
self.api.object_delete(self.account, self.container, "versioned")
self.assertRaises(NoSuchObject, self.api.object_locate, self.account,
self.container, "versioned")
check_num_objects(6)
self.api.container.content_purge(self.account, self.container,
"versioned")
self.assertRaises(NoSuchObject, self.api.object_locate, self.account,
self.container, "versioned")
check_num_objects(4)
system = {'sys.m2.keep_deleted_delay': '1'}
self.api.container_set_properties(self.account,
self.container,
system=system)
time.sleep(2)
self.api.container.content_purge(self.account, self.container,
"versioned")
check_num_objects(0)
def test_list_objects(self):
resp = self.api.object_list(self.account, self.container)
self.assertEqual(0, len(list(resp['objects'])))
self.assertFalse(resp.get('truncated'))
def _check_objects(expected_objects, objects):
self.assertEqual(len(expected_objects), len(objects))
for i in range(len(expected_objects)):
self.assertEqual(expected_objects[i]['name'],
objects[i]['name'])
self.assertEqual(int(expected_objects[i]['version']),
int(objects[i]['version']))
self.assertEqual(true_value(expected_objects[i]['deleted']),
true_value(objects[i]['deleted']))
all_versions = dict()
def _create_object(obj_name, all_versions):
self.api.object_create(self.account,
self.container,
obj_name=obj_name,
data="test")
versions = all_versions.get(obj_name, list())
versions.append(
self.api.object_show(self.account, self.container, obj_name))
all_versions[obj_name] = versions
def _delete_object(obj_name, all_versions):
self.api.object_delete(self.account, self.container, obj_name)
versions = all_versions.get(obj_name, list())
versions.append(
self.api.object_show(self.account, self.container, obj_name))
all_versions[obj_name] = versions
def _get_current_objects(all_versions):
current_objects = list()
obj_names = sorted(all_versions.keys())
for obj_name in obj_names:
obj = all_versions[obj_name][-1]
if not true_value(obj['deleted']):
current_objects.append(obj)
return current_objects
def _get_object_versions(all_versions):
object_versions = list()
obj_names = sorted(all_versions.keys())
for obj_name in obj_names:
versions = all_versions[obj_name]
versions.reverse()
object_versions += versions
versions.reverse()
return object_versions
# 0 object
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
# 3 objects with 1 version
for i in range(3):
_create_object("versioned" + str(i), all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects[:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects[:1], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
limit=1)
_check_objects(expected_current_objects[1:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True)
_check_objects(expected_object_versions[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True,
limit=3)
_check_objects(expected_object_versions[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
# 3 objects with 2 versions
for i in range(3):
_create_object("versioned" + str(i), all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects[:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects[:1], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
limit=1)
_check_objects(expected_current_objects[1:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True)
_check_objects(expected_object_versions[2:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True,
limit=3)
_check_objects(expected_object_versions[2:5], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned2', resp['next_marker'])
# 3 objects with 2 versions and 1 object with delete marker
_delete_object("versioned1", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects[:1], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
limit=1)
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True)
_check_objects(expected_object_versions[2:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True,
limit=3)
_check_objects(expected_object_versions[2:5], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
# 3 objects with 2 versions and 2 objects with delete marker
_delete_object("versioned0", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0')
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True)
_check_objects(expected_object_versions[3:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True,
limit=3)
_check_objects(expected_object_versions[3:6], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
# 3 objects with 2 versions and 3 objects with delete marker
_delete_object("versioned2", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0')
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True)
_check_objects(expected_object_versions[3:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True,
limit=3)
_check_objects(expected_object_versions[3:6], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
# 3 objects with 2 versions and 3 objects with delete marker
# (1 current version and 2 non current versions)
_create_object("versioned0", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container, limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
versions=True,
limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account,
self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
limit=1)
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True)
_check_objects(expected_object_versions[4:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account,
self.container,
marker='versioned0',
versions=True,
limit=3)
_check_objects(expected_object_versions[4:7], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
class TestObjectStorageAPI(BaseTestCase):
def setUp(self):
super(TestObjectStorageAPI, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(TestObjectStorageAPI, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
def _create(self, name, metadata=None):
return self.api.container_create(self.account,
name,
properties=metadata)
def _delete(self, name):
self.api.container_delete(self.account, name)
def _clean(self, name, clear=False):
if clear:
# must clean properties before
self.api.container_del_properties(self.account, name, [])
self._delete(name)
def _get_properties(self, name, properties=None):
return self.api.container_get_properties(self.account,
name,
properties=properties)
def _set_properties(self, name, properties=None):
return self.api.container_set_properties(self.account,
name,
properties=properties)
def test_container_show(self):
# container_show on unknown container
name = random_str(32)
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
self._create(name)
# container_show on existing container
res = self.api.container_show(self.account, name)
self.assertIsNot(res['properties'], None)
self._delete(name)
# container_show on deleted container
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
def test_container_create(self):
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# second create
res = self._create(name)
self.assertEqual(res, False)
# clean
self._delete(name)
def test_create_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
res = self._create(name, metadata)
self.assertEqual(res, True)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
def test_container_delete(self):
name = random_str(32)
# container_delete on unknown container
self.assertRaises(exc.NoSuchContainer, self.api.container_delete,
self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# container_delete on existing container
self._delete(name)
# verify deleted
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
# second delete
self.assertRaises(exc.NoSuchContainer, self.api.container_delete,
self.account, name)
# verify deleted
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
def test_container_get_properties(self):
name = random_str(32)
# container_get_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_get_properties, self.account,
name)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
self.assertIsNot(data['system'], None)
self.assertIn("sys.user.name", data['system'])
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_get_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_get_properties, self.account,
name)
def test_container_get_properties_filtered(self):
self.skipTest("Server side properties filtering not implemented")
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
# container_get_properties specify key
key = metadata.keys().pop(0)
data = self.api.container_get_properties(self.account, name, [key])
self.assertEqual({key: metadata[key]}, data['properties'])
# clean
self._clean(name, True)
def test_container_set_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# container_set_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_set_properties, self.account,
name, metadata)
res = self._create(name)
self.assertEqual(res, True)
# container_set_properties on existing container
self.api.container_set_properties(self.account, name, metadata)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# container_set_properties
key = random_str(32)
value = random_str(32)
metadata2 = {key: value}
self._set_properties(name, metadata2)
metadata.update(metadata2)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# container_set_properties overwrite key
key = metadata.keys().pop(0)
value = random_str(32)
metadata3 = {key: value}
metadata.update(metadata3)
self.api.container_set_properties(self.account, name, metadata3)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_set_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_set_properties, self.account,
name, metadata)
def test_del_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# container_del_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_del_properties, self.account,
name, [])
res = self._create(name, metadata)
self.assertEqual(res, True)
key = metadata.keys().pop()
del metadata[key]
# container_del_properties on existing container
self.api.container_del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertNotIn(key, data['properties'])
key = random_str(32)
# We do not check if a property exists before deleting it
# self.assertRaises(
# exc.NoSuchContainer, self.api.container_del_properties,
# self.account, name, [key])
self.api.container_del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_del_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_del_properties, self.account,
name, metadata.keys())
def test_object_create_mime_type(self):
name = random_str(32)
self.api.object_create(self.account,
name,
data="data",
obj_name=name,
mime_type='text/custom')
meta, _ = self.api.object_locate(self.account, name, name)
self.assertEqual(meta['mime_type'], 'text/custom')
def _upload_data(self, name):
chunksize = int(self.conf["chunk_size"])
size = int(chunksize * 12)
data = random_data(int(size))
self.api.object_create(self.account, name, obj_name=name, data=data)
self.created.append((name, name))
_, chunks = self.api.object_locate(self.account, name, name)
logging.debug("Chunks: %s", chunks)
return sort_chunks(chunks, False), data
def _fetch_range(self, name, range_):
if not isinstance(range_[0], tuple):
ranges = (range_, )
else:
ranges = range_
stream = self.api.object_fetch(self.account, name, name,
ranges=ranges)[1]
data = ""
for chunk in stream:
data += chunk
return data
def test_object_fetch_range_start(self):
"""From 0 to somewhere"""
name = random_str(16)
_, data = self._upload_data(name)
end = 666
fdata = self._fetch_range(name, (0, end))
self.assertEqual(len(fdata), end + 1)
self.assertEqual(fdata, data[0:end + 1])
def test_object_fetch_range_end(self):
"""From somewhere to end"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
last = max(chunks.keys())
end = chunks[last][0]['offset'] + chunks[last][0]['size']
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), len(data) - start)
self.assertEqual(fdata, data[start:])
def test_object_fetch_range_metachunk_start(self):
"""From the start of the second metachunk to somewhere"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = chunks[1][0]['offset']
end = start + 666
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_range_metachunk_end(self):
"""From somewhere to end of the first metachunk"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_range_2_metachunks(self):
"""
From somewhere in the first metachunk
to somewhere in the second metachunk
"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = start + chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_several_ranges(self):
"""
Download several ranges at once.
"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = start + chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, ((start, end), (end + 1, end + 2)))
self.assertEqual(len(fdata), end - start + 3)
self.assertEqual(fdata, data[start:end + 3])
# Notice that we download some bytes from the second metachunk
# before some from the first.
fdata = self._fetch_range(
name, ((chunks[0][0]['size'], chunks[0][0]['size'] + 2), (0, 1),
(1, 2), (4, 6)))
self.assertEqual(len(fdata), 10)
self.assertEqual(
fdata, data[chunks[0][0]['size']:chunks[0][0]['size'] + 3] +
data[0:2] + data[1:3] + data[4:7])
def test_object_create_then_append(self):
"""Create an object then append data"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
_, size, _ = self.api.object_create(self.account,
name,
data="2" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 256)
self.assertEqual(data, "1" * 128 + "2" * 128)
def test_object_create_from_append(self):
"""Create an object with append operation"""
name = random_str(16)
self.api.container_create(self.account, name)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name,
append=True)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 128)
self.assertEqual(data, "1" * 128)
def test_container_object_create_from_append(self):
"""Try to create container and object with append operation"""
name = random_str(16)
_chunks, size, checksum = self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
meta = self.api.object_get_properties(self.account, name, name)
self.assertEqual(meta.get('hash', "").lower(), checksum.lower())
def test_container_refresh(self):
account = random_str(32)
# container_refresh on unknown container
name = random_str(32)
self.assertRaises(exc.NoSuchContainer, self.api.container_refresh,
account, name)
self.api.container_create(account, name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on existing container
self.api.container_refresh(account, name)
time.sleep(0.5) # ensure container event have been processed
res = self.api.container_list(account, prefix=name)
name_container, nb_objects, nb_bytes, _ = res[0]
self.assertEqual(name_container, name)
self.assertEqual(nb_objects, 0)
self.assertEqual(nb_bytes, 0)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on existing container with data
self.api.container_refresh(account, name)
time.sleep(0.5) # ensure container event have been processed
res = self.api.container_list(account, prefix=name)
name_container, nb_objects, nb_bytes, _ = res[0]
self.assertEqual(name_container, name)
self.assertEqual(nb_objects, 1)
self.assertEqual(nb_bytes, 4)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on deleted container
self.assertRaises(exc.NoSuchContainer, self.api.container_refresh,
account, name)
self.api.account_delete(account)
def test_container_refresh_user_not_found(self):
name = random_str(32)
self.api.account.container_update(name, name, {"mtime": time.time()})
self.api.container_refresh(name, name)
containers = self.api.container_list(name)
self.assertEqual(len(containers), 0)
self.api.account_delete(name)
def test_account_refresh(self):
# account_refresh on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
# account_refresh on existing account
self.api.account_create(account)
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
name = random_str(32)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 4)
self.assertEqual(res["objects"], 1)
self.assertEqual(res["containers"], 1)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_refresh on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
def test_all_accounts_refresh(self):
# clear accounts
accounts = self.api.account_list()
for account in accounts:
try:
self.api.account_flush(account)
self.api.account_delete(account)
except exc.NoSuchAccount: # account remove in the meantime
pass
# all_accounts_refresh with 0 account
self.api.all_accounts_refresh()
# all_accounts_refresh with 2 account
account1 = random_str(32)
self.api.account_create(account1)
account2 = random_str(32)
self.api.account_create(account2)
self.api.all_accounts_refresh()
res = self.api.account_show(account1)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
res = self.api.account_show(account2)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.account_delete(account1)
self.api.account_delete(account2)
def test_account_flush(self):
# account_flush on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
# account_flush on existing account
name1 = random_str(32)
self.api.container_create(account, name1)
name2 = random_str(32)
self.api.container_create(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_flush(account)
containers = self.api.container_list(account)
self.assertEqual(len(containers), 0)
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.container_delete(account, name1)
self.api.container_delete(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_flush on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
def test_object_create_then_truncate(self):
"""Create an object then truncate data"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
self.api.object_truncate(self.account, name, name, size=64)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 64)
self.assertEqual(data, "1" * 64)
def test_object_create_append_then_truncate(self):
"""Create an object, append data then truncate on chunk boundary"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
_, size, _ = self.api.object_create(self.account,
name,
data="2" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
self.api.object_truncate(self.account, name, name, size=128)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 128)
self.assertEqual(data, "1" * 128)
self.api.object_truncate(self.account, name, name, size=128)
def test_object_create_then_invalid_truncate(self):
"""Create an object, append data then try to truncate outside object
range"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
self.assertRaises(exc.OioException,
self.api.object_truncate,
self.account,
name,
name,
size=-1)
self.assertRaises(exc.OioException,
self.api.object_truncate,
self.account,
name,
name,
size=129)
def test_container_snapshot(self):
name = random_str(16)
self.api.container_create(self.account, name)
test_object = "test_object"
self.api.object_create(self.account,
name,
data="0" * 128,
obj_name=test_object)
# Snapshot cannot have same name and same account
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, self.account, name)
snapshot_name = random_str(16)
self.assertNotEqual(snapshot_name, name)
# Non existing snapshot should work
self.api.container_snapshot(self.account, name, self.account,
snapshot_name)
# Already taken snapshot name should failed
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, self.account, snapshot_name)
# Check Container Frozen so create should failed
self.assertRaises(exc.ServiceBusy,
self.api.object_create,
self.account,
snapshot_name,
data="1" * 128,
obj_name="should_not_be_created")
# fullpath is set on every chunk
chunk_list = self.api.object_locate(self.account, name, test_object)[1]
# check that every chunk is different from the target
snapshot_list = self.api.object_locate(self.account, snapshot_name,
test_object)[1]
for c, t in zip(chunk_list, snapshot_list):
self.assertNotEqual(c['url'], t['url'])
# check target can be used
self.api.object_create(self.account,
name,
data="0" * 128,
obj_name="should_be_created")
# Create and send copy of a object
url_list = [c['url'] for c in chunk_list]
copy_list = self.api._generate_copy(url_list)
# every chunks should have the fullpath
fullpath = self.api._generate_fullpath(self.account, snapshot_name,
'copy', 12456)
self.api._send_copy(url_list, copy_list, fullpath[0])
# check that every copy exists
pool_manager = get_pool_manager()
for c in copy_list:
r = pool_manager.request('HEAD', c)
self.assertEqual(r.status, 200)
self.assertIn(fullpath[0],
r.headers["X-oio-chunk-meta-full-path"].split(','))
# Snapshot on non existing container should failed
self.assertRaises(exc.NoSuchContainer, self.api.container_snapshot,
random_str(16), random_str(16), random_str(16),
random_str(16))
# Snapshot need to have a account
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, None, random_str(16))
# Snapshot need to have a name
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, random_str(16), None)
class TestContainerReplication(BaseTestCase):
"""
Test container replication, especially what happens when one copy
has missed some operations (service has been down and is back up),
or has been lost (database file deleted).
"""
down_cache_opts = {
'client.down_cache.avoid': 'false',
'client.down_cache.shorten': 'true'
}
def setUp(self):
super(TestContainerReplication, self).setUp()
if int(self.conf.get('container_replicas', 1)) < 3:
self.skipTest('Container replication must be enabled')
self.api = ObjectStorageApi(self.ns, pool_manager=self.http_pool)
self.must_restart_meta2 = False
self.wait_for_score(('meta2', ))
self._apply_conf_on_all('meta2', self.__class__.down_cache_opts)
@classmethod
def tearDownClass(cls):
# Be kind with the next test suites
cls._cls_reload_proxy()
time.sleep(3)
cls._cls_reload_meta()
time.sleep(1)
def tearDown(self):
# Start all services
self._service('@' + self.ns, 'start')
super(TestContainerReplication, self).tearDown()
# Restart meta2 after configuration has been reset by parent tearDown
if self.must_restart_meta2:
self._service('@meta2', 'stop')
self._service('@meta2', 'start')
self.wait_for_score(('meta2', ))
def _apply_conf_on_all(self, type_, conf):
all_svc = [x['addr'] for x in self.conf['services'][type_]]
for svc in all_svc:
self.admin.service_set_live_config(svc, conf, request_attempts=4)
def _synchronous_restore_allowed(self):
dump_max_size = int(
self.ns_conf.get('sqliterepo.dump.max_size', 1073741824))
return dump_max_size
def _test_restore_after_missed_diff(self):
cname = 'test_restore_' + random_str(8)
# Create a container
self.api.container_create(self.account, cname)
# Locate the peers
peers = self.api.directory.list(self.account,
cname,
service_type='meta2')
# Stop one peer
kept = peers['srv'][0]['host']
stopped = peers['srv'][1]['host']
self.api.logger.info('Stopping meta2 %s', stopped)
self._service(self.service_to_gridinit_key(stopped, 'meta2'), 'stop')
# Create an object
self.api.object_create_ext(self.account,
cname,
obj_name=cname,
data=cname)
# Start the stopped peer
self.api.logger.info('Starting meta2 %s', stopped)
self._service(self.service_to_gridinit_key(stopped, 'meta2'), 'start')
self.wait_for_score(('meta2', ))
# Create another object
self.api.object_create_ext(self.account,
cname,
obj_name=cname + '_2',
data=cname)
# Check the database has been restored (after a little while)
ref_props = self.api.container_get_properties(
self.account, cname, params={'service_id': kept})
copy_props = self.api.container_get_properties(
self.account, cname, params={'service_id': stopped})
self.assertEqual(ref_props['system'], copy_props['system'])
@flaky(rerun_filter=is_election_error)
def test_disabled_synchronous_restore(self):
"""
Test what happens when the synchronous DB_RESTORE mechanism has been
disabled, and some operations have been missed by a slave.
"""
allowed = self._synchronous_restore_allowed()
if allowed:
# Disable synchronous restore, restart all meta2 services
opts = {'sqliterepo.dump.max_size': 0}
opts.update(self.__class__.down_cache_opts)
self.set_ns_opts(opts)
self._apply_conf_on_all('meta2', opts)
self.must_restart_meta2 = True
self._test_restore_after_missed_diff()
@flaky(rerun_filter=is_election_error)
def test_synchronous_restore(self):
"""
Test DB_RESTORE mechanism (the master send a dump of the whole
database to one of the peers).
"""
if not self._synchronous_restore_allowed():
self.skipTest('Synchronous replication is disabled')
if self.is_running_on_public_ci():
self.skipTest("Too buggy to run on public CI")
self._test_restore_after_missed_diff()
@flaky(rerun_filter=is_election_error)
def test_asynchronous_restore(self):
"""
Test DB_DUMP/DB_PIPEFROM mechanism (a slave peer knows it needs
a fresh copy of the database and asks the master).
"""
cname = 'test_pipefrom_' + random_str(8)
# Create a container
self.api.container_create(self.account, cname)
# Locate the peers
peers = self.api.directory.list(self.account,
cname,
service_type='meta2')
# Stop one peer
kept = peers['srv'][0]['host']
stopped = peers['srv'][1]['host']
self.api.logger.info('Stopping meta2 %s', stopped)
self._service(self.service_to_gridinit_key(stopped, 'meta2'), 'stop')
# Delete the database
vol = [
x['path'] for x in self.conf['services']['meta2']
if x.get('service_id', x['addr']) == stopped
][0]
path = '/'.join((vol, peers['cid'][:3], peers['cid'] + '.1.meta2'))
self.api.logger.info('Removing %s', path)
os.remove(path)
# Start the stopped peer
self.api.logger.info('Starting meta2 %s', stopped)
self._service(self.service_to_gridinit_key(stopped, 'meta2'), 'start')
self.wait_for_score(('meta2', ))
# Create an object (to trigger a database replication)
self.api.object_create_ext(self.account,
cname,
obj_name=cname,
data=cname)
# Check the database has been restored
ref_props = self.api.container_get_properties(
self.account, cname, params={'service_id': kept})
copy_props = self.api.container_get_properties(
self.account, cname, params={'service_id': stopped})
self.assertEqual(ref_props['system'], copy_props['system'])
