class TestBlobConverter(BaseTestCase):
def setUp(self):
super(TestBlobConverter, self).setUp()
self.container = random_str(16)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(
self.account, self.container, self.path, 1)
services = self.conscience.all_services('rawx')
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(
self.account, self.container, obj_name=self.path, data="chunk")
meta, self.chunks = self.api.object_locate(
self.account, self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
def _chunk_path(self, chunk):
url = chunk['url']
volume_id = url.split('/', 3)[2]
chunk_id = url.split('/', 3)[3]
volume = self.rawx_volumes[volume_id]
return volume + '/' + chunk_id[:3] + '/' + chunk_id
def _converter_and_check(self, chunk_volume, chunk_path,
chunk_id_info, expected_raw_meta=None,
expected_errors=0):
conf = self.conf
conf['volume'] = self.rawx_volumes[chunk_volume]
converter = BlobConverter(conf)
converter.safe_convert_chunk(chunk_path)
self.assertEqual(1, converter.total_chunks_processed)
self.assertEqual(1, converter.passes)
self.assertEqual(expected_errors, converter.errors)
checker = Checker(self.ns)
for chunk_id, info in chunk_id_info.iteritems():
account, container, path, version, content_id = info
fullpath = encode_fullpath(
account, container, path, version, content_id)
cid = cid_from_name(account, container)
meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id)
self.assertEqual(meta.get('chunk_id'), chunk_id)
self.assertEqual(meta.get('container_id'), cid)
self.assertEqual(meta.get('content_path'), path)
self.assertEqual(meta.get('content_version'), version)
self.assertEqual(meta.get('content_id'), content_id)
self.assertEqual(meta.get('full_path'), fullpath)
checker.check(Target(
account, container=container, obj=path,
chunk='http://' + converter.volume_id + '/' + chunk_id))
checker.wait()
self.assertTrue(checker.report())
if expected_raw_meta:
self.assertDictEqual(expected_raw_meta, raw_meta)
continue
self.assertNotIn(chunk_xattr_keys['chunk_id'], raw_meta)
self.assertNotIn(chunk_xattr_keys['container_id'], raw_meta)
self.assertNotIn(chunk_xattr_keys['content_path'], raw_meta)
self.assertNotIn(chunk_xattr_keys['content_version'], raw_meta)
self.assertNotIn(chunk_xattr_keys['content_id'], raw_meta)
self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id,
raw_meta)
for k, v in raw_meta.iteritems():
if k.startswith('oio:'):
self.fail('old fullpath always existing')
self.assertEqual(raw_meta[chunk_xattr_keys['oio_version']],
OIO_VERSION)
def test_converter(self):
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_wrong_path(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '+', self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_wrong_content_id(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, '0123456789ABCDEF0123456789ABCDEF')
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_old_fullpath(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id, add_old_fullpath=True)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_old_fullpath_and_wrong_path(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path, self.version, self.content_id,
add_old_fullpath=True)
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '+', self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_wrong_fullpath(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path, 'None', '0123456789ABCDEF0123456789ABCDEF',
add_old_fullpath=True)
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path, self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_linked_chunk(self):
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
for c in linked_chunks:
if chunk_volume == c['url'].split('/')[2]:
linked_chunk_id2 = c['url'].split('/')[3]
break
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
for c in self.chunks:
if linked_chunk_volume == c['url'].split('/')[2]:
chunk_id2 = c['url'].split('/')[3]
break
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id2: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
self._converter_and_check(
linked_chunk_volume, linked_chunk_path,
{chunk_id2: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
def test_converter_old_linked_chunk(self):
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
for c in linked_chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '.link', 'None',
'0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True)
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
for c in linked_chunks:
if chunk_volume == c['url'].split('/')[2]:
linked_chunk_id2 = c['url'].split('/')[3]
break
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
for c in self.chunks:
if linked_chunk_volume == c['url'].split('/')[2]:
chunk_id2 = c['url'].split('/')[3]
break
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id2: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
self._converter_and_check(
linked_chunk_volume, linked_chunk_path,
{chunk_id2: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
def test_converter_old_chunk_with_link_on_same_object(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path)
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, linked_meta['id'])
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
# old xattr not removed
_, expected_raw_meta = read_chunk_metadata(linked_chunk_path,
linked_chunk_id)
expected_raw_meta[chunk_xattr_keys['oio_version']] = OIO_VERSION
self._converter_and_check(
linked_chunk_volume, linked_chunk_path,
{linked_chunk_id: (self.account, self.container,
self.path, linked_meta['version'],
linked_meta['id'])},
expected_raw_meta=expected_raw_meta, expected_errors=1)
def test_converter_old_linked_chunk_with_link_on_same_object(self):
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
for c in linked_chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '.link', 'None',
'0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True)
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id, add_old_fullpath=True)
self.api.object_link(
self.account, self.container, self.path + '.link',
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
for c in linked_chunks:
if chunk_volume == c['url'].split('/')[2]:
linked_chunk_id2 = c['url'].split('/')[3]
break
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
for c in self.chunks:
if linked_chunk_volume == c['url'].split('/')[2]:
chunk_id2 = c['url'].split('/')[3]
break
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id2: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
self._converter_and_check(
linked_chunk_volume, linked_chunk_path,
{chunk_id2: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
def test_converter_with_versioning(self):
self.api.container_set_properties(
self.account, self.container,
system={'sys.m2.policy.version': '2'})
self.api.object_create(
self.account, self.container, obj_name=self.path, data='version')
versioned_meta, versioned_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, versioned_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
versioned_chunk = random.choice(versioned_chunks)
versioned_chunk_volume = versioned_chunk['url'].split('/')[2]
versioned_chunk_id = versioned_chunk['url'].split('/')[3]
versioned_chunk_path = self._chunk_path(versioned_chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
self._converter_and_check(
versioned_chunk_volume, versioned_chunk_path,
{versioned_chunk_id: (self.account, self.container, self.path,
versioned_meta['version'],
versioned_meta['id'])})
def test_converter_old_chunk_with_versioning(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
self.api.container_set_properties(
self.account, self.container,
system={'sys.m2.policy.version': '2'})
self.api.object_create(
self.account, self.container, obj_name=self.path, data='version')
versioned_meta, versioned_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, versioned_meta['id'])
for c in versioned_chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
versioned_meta['version'], versioned_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
versioned_chunk = random.choice(versioned_chunks)
versioned_chunk_volume = versioned_chunk['url'].split('/')[2]
versioned_chunk_id = versioned_chunk['url'].split('/')[3]
versioned_chunk_path = self._chunk_path(versioned_chunk)
self._converter_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
self._converter_and_check(
versioned_chunk_volume, versioned_chunk_path,
{versioned_chunk_id: (self.account, self.container, self.path,
versioned_meta['version'],
versioned_meta['id'])})
class TestPerfectibleContent(BaseTestCase):
def setUp(self):
super(TestPerfectibleContent, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri,
pool_manager=self.http_pool)
# Ensure the tube is not clogged
self.beanstalkd.drain_tube(DEFAULT_IMPROVER_TUBE)
@classmethod
def tearDownClass(cls):
# Be kind with the next test suites
cls._cls_reload_proxy()
time.sleep(3)
cls._cls_reload_meta()
time.sleep(1)
def _aggregate_services(self, type_, key):
"""
Build a dictionary of lists of services indexed by `key`.
:param type_: the type if services to index
:param key: a function
"""
all_svcs = self.conscience.all_services(type_)
out = defaultdict(list)
for svc in all_svcs:
out[key(svc)].append(svc)
return out
def _aggregate_rawx_by_slot(self):
by_slot = self._aggregate_services('rawx',
lambda x: x['tags']
.get('tag.slots', 'rawx')
.rsplit(',', 2)[-1])
if 'rawx-even' not in by_slot or 'rawx-odd' not in by_slot:
self.skip('This test requires "rawx-even" and "rawx-odd" slots')
return by_slot
def _aggregate_rawx_by_place(self):
by_place = self._aggregate_services(
'rawx', lambda x: x['tags']['tag.loc'].rsplit('.', 1)[0])
if len(by_place) < 3:
self.skip('This test requires 3 different 2nd level locations')
return by_place
def _wait_for_event(self, timeout=REASONABLE_EVENT_DELAY):
"""
Wait for an event in the oio-improve tube.
"""
self.beanstalkd.watch(DEFAULT_IMPROVER_TUBE)
try:
job_id, data = self.beanstalkd.reserve(timeout=timeout)
except ResponseError as exc:
logging.warn('No event read from tube %s: %s',
DEFAULT_IMPROVER_TUBE, exc)
self.fail()
self.beanstalkd.delete(job_id)
return Event(json.loads(data))
# This test must be executed first
def test_0_upload_ok(self):
"""Check that no event is emitted when everything is ok."""
# Check we have enough service locations.
self._aggregate_rawx_by_place()
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
self.api.object_create(self.account, container,
obj_name='perfect',
data='whatever',
policy='THREECOPIES',
headers={REQID_HEADER: reqid})
# Wait on the oio-improve beanstalk tube.
self.beanstalkd.watch(DEFAULT_IMPROVER_TUBE)
# Ensure we do not receive any event.
self.assertRaises(ResponseError, self.beanstalkd.reserve,
timeout=REASONABLE_EVENT_DELAY)
def test_upload_warn_dist(self):
"""
Check that an event is emitted when the warning distance is reached.
"""
# Check we have enough service locations.
by_place = self._aggregate_services(
'rawx', lambda x: x['tags']['tag.loc'].rsplit('.', 2)[0])
if len(by_place) < 3:
self.skip('This test requires 3 different 2nd level locations')
return
# Lock all services of the 3rd location.
banned_loc = by_place.keys()[2]
self._lock_services('rawx', by_place[banned_loc])
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
self.api.object_create(self.account, container,
obj_name='perfectible',
data='whatever',
policy='THREECOPIES',
headers={REQID_HEADER: reqid})
# Wait on the oio-improve beanstalk tube.
event = self._wait_for_event()
# Check the content of the event.
self.assertEqual('storage.content.perfectible', event.event_type)
self.assertEqual(reqid, event.reqid)
self.assertEqual(self.account, event.url['account'])
self.assertEqual(container, event.url['user'])
self.assertEqual('perfectible', event.url['path'])
mc = event.data
self.assertEqual(0, mc['pos']) # only one metachunk in this test
lowest_dist = 4
warn_dist = 4
for chunk in mc['chunks']:
qual = chunk['quality']
if qual['final_dist'] < lowest_dist:
lowest_dist = qual['final_dist']
if qual['warn_dist'] < warn_dist:
warn_dist = qual['warn_dist']
self.assertEqual(qual['expected_slot'], qual['final_slot'])
self.assertLessEqual(lowest_dist, warn_dist)
def test_upload_fallback(self):
"""
Test that an event is emitted when a fallback service slot is used.
"""
by_slot = self._aggregate_rawx_by_slot()
if len(by_slot['rawx-odd']) < 3:
self.skip('This test requires at least 3 services '
'in the "rawx-odd" slot')
# Lock all services of the 'rawx-even' slot.
banned_slot = 'rawx-even'
self._lock_services('rawx', by_slot[banned_slot])
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
self.api.object_create(self.account, container,
obj_name='perfectible',
data='whatever',
policy='THREECOPIES',
headers={REQID_HEADER: reqid})
# Wait on the oio-improve beanstalk tube.
event = self._wait_for_event()
# Check the content of the event.
self.assertEqual('storage.content.perfectible', event.event_type)
self.assertEqual(reqid, event.reqid)
self.assertEqual(self.account, event.url['account'])
self.assertEqual(container, event.url['user'])
self.assertEqual('perfectible', event.url['path'])
mc = event.data
self.assertEqual(0, mc['pos']) # only one metachunk in this test
slot_matches = list()
for chunk in mc['chunks']:
qual = chunk['quality']
slot_matches.append(qual['final_slot'] == qual['expected_slot'])
self.assertNotEqual(qual['final_slot'], banned_slot)
self.assertIn(False, slot_matches)
def _call_blob_improver_subprocess(self, run_time=3.0,
stop_after_events=1,
log_level='INFO'):
# FIXME(FVE): find a way to call coverage on the subprocess
blob_improver = subprocess.Popen(
['oio-blob-improver', self.ns,
'--beanstalkd=' + self.conf['queue_addr'],
'--retry-delay=1',
'--log-level=' + log_level,
'--stop-after-events=%d' % stop_after_events])
if SUBPROCESS32:
try:
blob_improver.wait(run_time)
except Exception:
blob_improver.kill()
else:
time.sleep(run_time)
blob_improver.kill()
def test_blob_improver_threecopies(self):
by_slot = self._aggregate_rawx_by_slot()
if len(by_slot['rawx-odd']) < 3:
self.skip('This test requires at least 3 services '
'in the "rawx-odd" slot')
# Ensure the distance between services won't be a problem.
self._aggregate_rawx_by_place()
# Lock all services of the 'rawx-even' slot.
banned_slot = 'rawx-even'
self._lock_services('rawx', by_slot[banned_slot])
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
chunks, _, _ = self.api.object_create(
self.account, container, obj_name='perfectible',
data='whatever', policy='THREECOPIES', reqid=reqid)
# Wait for the "perfectible" event to be emitted,
# but do not consume it.
job, data = self.beanstalkd.wait_for_ready_job(
DEFAULT_IMPROVER_TUBE, timeout=REASONABLE_EVENT_DELAY)
if job:
logging.debug("Expected job data: %s", data)
self.assertIsNotNone(job)
# "Unlock" the services of the 'rawx-even' slot.
self._lock_services('rawx', by_slot[banned_slot], score=100)
self._call_blob_improver_subprocess()
# Check some changes have been done on the object.
_, new_chunks = self.api.object_locate(
self.account, container, 'perfectible')
old_urls = sorted([x['url'] for x in chunks])
new_urls = sorted([x['url'] for x in new_chunks])
logging.debug('Old chunks: %s', old_urls)
logging.debug('New chunks: %s', new_urls)
self.assertNotEqual(old_urls, new_urls)
# Ensure no new "perfectible" event is emitted.
job, data = self.beanstalkd.wait_for_ready_job(
DEFAULT_IMPROVER_TUBE, timeout=REASONABLE_EVENT_DELAY)
if job:
logging.debug("Unexpected job data: %s", data)
self.assertIsNone(job)
class TestBlobMover(BaseTestCase):
def setUp(self):
super(TestBlobMover, self).setUp()
self.container = random_str(16)
self.cid = cid_from_name(self.account, self.container)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.blob_client = BlobClient(self.conf)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(self.account,
self.container,
self.path, 1)
services = self.conscience.all_services('rawx')
if len(chunks) >= len([s for s in services if s['score'] > 0]):
self.skipTest("need at least %d rawx to run" % (len(chunks) + 1))
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(self.account,
self.container,
obj_name=self.path,
data="chunk")
meta, self.chunks = self.api.object_locate(self.account,
self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
def _chunk_path(self, chunk):
url = chunk['url']
volume_id = url.split('/', 3)[2]
chunk_id = url.split('/', 3)[3]
volume = self.rawx_volumes[volume_id]
return volume + '/' + chunk_id[:3] + '/' + chunk_id
def test_move_old_chunk(self):
for c in self.chunks:
convert_to_old_chunk(self._chunk_path(c), self.account,
self.container, self.path, self.version,
self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_headers, chunk_stream = self.blob_client.chunk_get(chunk['url'])
chunks_kept = list(self.chunks)
chunks_kept.remove(chunk)
mover = BlobMoverWorker(self.conf, None,
self.rawx_volumes[chunk_volume])
mover.chunk_move(self._chunk_path(chunk), chunk_id)
_, new_chunks = self.api.object_locate(self.account, self.container,
self.path)
new_chunk = list(new_chunks)
self.assertEqual(len(new_chunks), len(chunks_kept) + 1)
url_kept = [c['url'] for c in chunks_kept]
new_chunk = None
for c in new_chunks:
if c['url'] not in url_kept:
self.assertIsNone(new_chunk)
new_chunk = c
self.assertNotEqual(chunk['real_url'], new_chunk['real_url'])
self.assertNotEqual(chunk['url'], new_chunk['url'])
self.assertEqual(chunk['pos'], new_chunk['pos'])
self.assertEqual(chunk['size'], new_chunk['size'])
self.assertEqual(chunk['hash'], new_chunk['hash'])
new_chunk_headers, new_chunk_stream = self.blob_client.chunk_get(
new_chunk['url'])
self.assertEqual(chunk_stream.read(), new_chunk_stream.read())
fullpath = encode_fullpath(self.account, self.container, self.path,
self.version, self.content_id)
self.assertEqual(fullpath, new_chunk_headers['full_path'])
del new_chunk_headers['full_path']
self.assertNotEqual(chunk_headers['chunk_id'],
new_chunk_headers['chunk_id'])
new_chunk_id = new_chunk['url'].split('/')[3]
self.assertEqual(new_chunk_id, new_chunk_headers['chunk_id'])
del chunk_headers['chunk_id']
del new_chunk_headers['chunk_id']
self.assertEqual(OIO_VERSION, new_chunk_headers['oio_version'])
del chunk_headers['oio_version']
del new_chunk_headers['oio_version']
self.assertEqual(chunk_headers, new_chunk_headers)
class TestObjectStorageApiPerfdata(BaseTestCase):
def setUp(self):
super(TestObjectStorageApiPerfdata, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(TestObjectStorageApiPerfdata, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
def test_object_create_perfdata(self):
perfdata = dict()
container = random_str(8)
obj = random_str(8)
self.api.object_create(self.account,
container,
obj_name=obj,
data=obj,
perfdata=perfdata)
meta, chunks = self.api.object_locate(self.account, container, obj)
self.assertIn('proxy', perfdata)
self.assertIn('resolve', perfdata['proxy'])
self.assertIn('meta2', perfdata['proxy'])
self.assertIn('overall', perfdata['proxy'])
self.assertIn('rawx', perfdata)
if meta['policy'] == 'EC':
self.assertIn('ec', perfdata['rawx'])
for chunk in chunks:
self.assertIn('connect.' + chunk['url'], perfdata['rawx'])
self.assertIn('upload.' + chunk['url'], perfdata['rawx'])
self.assertIn('connect.AVG', perfdata['rawx'])
self.assertIn('connect.SD', perfdata['rawx'])
self.assertIn('connect.RSD', perfdata['rawx'])
self.assertIn('upload.AVG', perfdata['rawx'])
self.assertIn('upload.SD', perfdata['rawx'])
self.assertIn('upload.RSD', perfdata['rawx'])
self.assertIn('overall', perfdata['rawx'])
perfdata.clear()
self.api.object_delete(self.account, container, obj, perfdata=perfdata)
self.assertIn('proxy', perfdata)
self.assertIn('resolve', perfdata['proxy'])
self.assertIn('meta2', perfdata['proxy'])
self.assertIn('overall', perfdata['proxy'])
def test_object_fetch_perfdata(self):
perfdata = dict()
container = random_str(8)
obj = random_str(8)
odata = obj.encode('utf-8')
self.api.object_create(self.account,
container,
obj_name=obj,
data=odata)
meta, chunks = self.api.object_locate(self.account, container, obj)
stg_method = STORAGE_METHODS.load(meta['chunk_method'])
_, stream = self.api.object_fetch(self.account,
container,
obj,
perfdata=perfdata)
self.assertIn('proxy', perfdata)
self.assertIn('resolve', perfdata['proxy'])
self.assertIn('meta2', perfdata['proxy'])
self.assertIn('overall', perfdata['proxy'])
self.assertNotIn('ttfb', perfdata)
self.assertNotIn('ttlb', perfdata)
buf = b''.join(stream)
self.assertEqual(odata, buf)
self.assertIn('rawx', perfdata)
if stg_method.ec:
self.assertIn('ec', perfdata['rawx'])
nb_chunks_to_read = 0
for chunk in chunks:
key = "connect." + chunk['url']
if key in perfdata['rawx']:
nb_chunks_to_read += 1
self.assertLessEqual(stg_method.min_chunks_to_read, nb_chunks_to_read)
self.assertIn('overall', perfdata['rawx'])
self.assertIn('ttfb', perfdata)
self.assertIn('ttlb', perfdata)
self.api.object_delete(self.account, container, obj)
class TestObjectStorageAPI(BaseTestCase):
def setUp(self):
super(TestObjectStorageAPI, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(TestObjectStorageAPI, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
def _create(self, name, metadata=None):
return self.api.container_create(self.account,
name,
properties=metadata)
def _delete(self, name):
self.api.container_delete(self.account, name)
def _clean(self, name, clear=False):
if clear:
# must clean properties before
self.api.container_del_properties(self.account, name, [])
self._delete(name)
def _get_properties(self, name, properties=None):
return self.api.container_get_properties(self.account,
name,
properties=properties)
def _set_properties(self, name, properties=None):
return self.api.container_set_properties(self.account,
name,
properties=properties)
def test_container_show(self):
# container_show on unknown container
name = random_str(32)
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
self._create(name)
# container_show on existing container
res = self.api.container_show(self.account, name)
self.assertIsNot(res['properties'], None)
self._delete(name)
# container_show on deleted container
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
def test_container_create(self):
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# second create
res = self._create(name)
self.assertEqual(res, False)
# clean
self._delete(name)
def test_create_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
res = self._create(name, metadata)
self.assertEqual(res, True)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
def test_container_delete(self):
name = random_str(32)
# container_delete on unknown container
self.assertRaises(exc.NoSuchContainer, self.api.container_delete,
self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# container_delete on existing container
self._delete(name)
# verify deleted
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
# second delete
self.assertRaises(exc.NoSuchContainer, self.api.container_delete,
self.account, name)
# verify deleted
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
def test_container_get_properties(self):
name = random_str(32)
# container_get_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_get_properties, self.account,
name)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
self.assertIsNot(data['system'], None)
self.assertIn("sys.user.name", data['system'])
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_get_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_get_properties, self.account,
name)
def test_container_get_properties_filtered(self):
self.skipTest("Server side properties filtering not implemented")
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
# container_get_properties specify key
key = metadata.keys().pop(0)
data = self.api.container_get_properties(self.account, name, [key])
self.assertEqual({key: metadata[key]}, data['properties'])
# clean
self._clean(name, True)
def test_container_set_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# container_set_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_set_properties, self.account,
name, metadata)
res = self._create(name)
self.assertEqual(res, True)
# container_set_properties on existing container
self.api.container_set_properties(self.account, name, metadata)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# container_set_properties
key = random_str(32)
value = random_str(32)
metadata2 = {key: value}
self._set_properties(name, metadata2)
metadata.update(metadata2)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# container_set_properties overwrite key
key = metadata.keys().pop(0)
value = random_str(32)
metadata3 = {key: value}
metadata.update(metadata3)
self.api.container_set_properties(self.account, name, metadata3)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_set_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_set_properties, self.account,
name, metadata)
def test_del_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# container_del_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_del_properties, self.account,
name, [])
res = self._create(name, metadata)
self.assertEqual(res, True)
key = metadata.keys().pop()
del metadata[key]
# container_del_properties on existing container
self.api.container_del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertNotIn(key, data['properties'])
key = random_str(32)
# We do not check if a property exists before deleting it
# self.assertRaises(
# exc.NoSuchContainer, self.api.container_del_properties,
# self.account, name, [key])
self.api.container_del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_del_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_del_properties, self.account,
name, metadata.keys())
def test_object_create_mime_type(self):
name = random_str(32)
self.api.object_create(self.account,
name,
data="data",
obj_name=name,
mime_type='text/custom')
meta, _ = self.api.object_locate(self.account, name, name)
self.assertEqual(meta['mime_type'], 'text/custom')
def _upload_data(self, name):
chunksize = int(self.conf["chunk_size"])
size = int(chunksize * 12)
data = random_data(int(size))
self.api.object_create(self.account, name, obj_name=name, data=data)
self.created.append((name, name))
_, chunks = self.api.object_locate(self.account, name, name)
logging.debug("Chunks: %s", chunks)
return sort_chunks(chunks, False), data
def _fetch_range(self, name, range_):
if not isinstance(range_[0], tuple):
ranges = (range_, )
else:
ranges = range_
stream = self.api.object_fetch(self.account, name, name,
ranges=ranges)[1]
data = ""
for chunk in stream:
data += chunk
return data
def test_object_fetch_range_start(self):
"""From 0 to somewhere"""
name = random_str(16)
_, data = self._upload_data(name)
end = 666
fdata = self._fetch_range(name, (0, end))
self.assertEqual(len(fdata), end + 1)
self.assertEqual(fdata, data[0:end + 1])
def test_object_fetch_range_end(self):
"""From somewhere to end"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
last = max(chunks.keys())
end = chunks[last][0]['offset'] + chunks[last][0]['size']
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), len(data) - start)
self.assertEqual(fdata, data[start:])
def test_object_fetch_range_metachunk_start(self):
"""From the start of the second metachunk to somewhere"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = chunks[1][0]['offset']
end = start + 666
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_range_metachunk_end(self):
"""From somewhere to end of the first metachunk"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_range_2_metachunks(self):
"""
From somewhere in the first metachunk
to somewhere in the second metachunk
"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = start + chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_several_ranges(self):
"""
Download several ranges at once.
"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = start + chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, ((start, end), (end + 1, end + 2)))
self.assertEqual(len(fdata), end - start + 3)
self.assertEqual(fdata, data[start:end + 3])
# Notice that we download some bytes from the second metachunk
# before some from the first.
fdata = self._fetch_range(
name, ((chunks[0][0]['size'], chunks[0][0]['size'] + 2), (0, 1),
(1, 2), (4, 6)))
self.assertEqual(len(fdata), 10)
self.assertEqual(
fdata, data[chunks[0][0]['size']:chunks[0][0]['size'] + 3] +
data[0:2] + data[1:3] + data[4:7])
def test_object_create_then_append(self):
"""Create an object then append data"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
_, size, _ = self.api.object_create(self.account,
name,
data="2" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 256)
self.assertEqual(data, "1" * 128 + "2" * 128)
def test_object_create_from_append(self):
"""Create an object with append operation"""
name = random_str(16)
self.api.container_create(self.account, name)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name,
append=True)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 128)
self.assertEqual(data, "1" * 128)
def test_container_object_create_from_append(self):
"""Try to create container and object with append operation"""
name = random_str(16)
_chunks, size, checksum = self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
meta = self.api.object_get_properties(self.account, name, name)
self.assertEqual(meta.get('hash', "").lower(), checksum.lower())
def test_container_refresh(self):
account = random_str(32)
# container_refresh on unknown container
name = random_str(32)
self.assertRaises(exc.NoSuchContainer, self.api.container_refresh,
account, name)
self.api.container_create(account, name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on existing container
self.api.container_refresh(account, name)
time.sleep(0.5) # ensure container event have been processed
res = self.api.container_list(account, prefix=name)
name_container, nb_objects, nb_bytes, _ = res[0]
self.assertEqual(name_container, name)
self.assertEqual(nb_objects, 0)
self.assertEqual(nb_bytes, 0)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on existing container with data
self.api.container_refresh(account, name)
time.sleep(0.5) # ensure container event have been processed
res = self.api.container_list(account, prefix=name)
name_container, nb_objects, nb_bytes, _ = res[0]
self.assertEqual(name_container, name)
self.assertEqual(nb_objects, 1)
self.assertEqual(nb_bytes, 4)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on deleted container
self.assertRaises(exc.NoSuchContainer, self.api.container_refresh,
account, name)
self.api.account_delete(account)
def test_container_refresh_user_not_found(self):
name = random_str(32)
self.api.account.container_update(name, name, {"mtime": time.time()})
self.api.container_refresh(name, name)
containers = self.api.container_list(name)
self.assertEqual(len(containers), 0)
self.api.account_delete(name)
def test_account_refresh(self):
# account_refresh on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
# account_refresh on existing account
self.api.account_create(account)
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
name = random_str(32)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 4)
self.assertEqual(res["objects"], 1)
self.assertEqual(res["containers"], 1)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_refresh on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
def test_all_accounts_refresh(self):
# clear accounts
accounts = self.api.account_list()
for account in accounts:
try:
self.api.account_flush(account)
self.api.account_delete(account)
except exc.NoSuchAccount: # account remove in the meantime
pass
# all_accounts_refresh with 0 account
self.api.all_accounts_refresh()
# all_accounts_refresh with 2 account
account1 = random_str(32)
self.api.account_create(account1)
account2 = random_str(32)
self.api.account_create(account2)
self.api.all_accounts_refresh()
res = self.api.account_show(account1)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
res = self.api.account_show(account2)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.account_delete(account1)
self.api.account_delete(account2)
def test_account_flush(self):
# account_flush on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
# account_flush on existing account
name1 = random_str(32)
self.api.container_create(account, name1)
name2 = random_str(32)
self.api.container_create(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_flush(account)
containers = self.api.container_list(account)
self.assertEqual(len(containers), 0)
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.container_delete(account, name1)
self.api.container_delete(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_flush on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
def test_object_create_then_truncate(self):
"""Create an object then truncate data"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
self.api.object_truncate(self.account, name, name, size=64)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 64)
self.assertEqual(data, "1" * 64)
def test_object_create_append_then_truncate(self):
"""Create an object, append data then truncate on chunk boundary"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
_, size, _ = self.api.object_create(self.account,
name,
data="2" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
self.api.object_truncate(self.account, name, name, size=128)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 128)
self.assertEqual(data, "1" * 128)
self.api.object_truncate(self.account, name, name, size=128)
def test_object_create_then_invalid_truncate(self):
"""Create an object, append data then try to truncate outside object
range"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
self.assertRaises(exc.OioException,
self.api.object_truncate,
self.account,
name,
name,
size=-1)
self.assertRaises(exc.OioException,
self.api.object_truncate,
self.account,
name,
name,
size=129)
class TestContentRebuildFilter(BaseTestCase):
def setUp(self):
super(TestContentRebuildFilter, self).setUp()
self.namespace = self.conf['namespace']
self.gridconf = {"namespace": self.namespace}
self.container = "TestContentRebuildFilter%f" % time.time()
self.ref = self.container
self.container_client = ContainerClient(self.conf)
self.container_client.container_create(self.account, self.container)
syst = self.container_client.container_get_properties(
self.account, self.container)['system']
self.container_id = syst['sys.name'].split('.', 1)[0]
self.object_storage_api = ObjectStorageApi(namespace=self.namespace)
queue_addr = choice(self.conf['services']['beanstalkd'])['addr']
self.queue_url = queue_addr
self.conf['queue_url'] = 'beanstalk://' + self.queue_url
self.conf['tube'] = BlobRebuilder.DEFAULT_BEANSTALKD_WORKER_TUBE
self.notify_filter = NotifyFilter(app=_App, conf=self.conf)
bt = Beanstalk.from_url(self.conf['queue_url'])
bt.drain_tube(BlobRebuilder.DEFAULT_BEANSTALKD_WORKER_TUBE)
bt.close()
def _create_event(self, content_name, present_chunks, missing_chunks,
content_id):
event = {}
event["when"] = time.time()
event["event"] = "storage.content.broken"
event["data"] = {"present_chunks": present_chunks,
"missing_chunks": missing_chunks}
event["url"] = {"ns": self.namespace, "account": self.account,
"user": self.container, "path": content_name,
"id": self.container_id, "content": content_id}
return event
def _is_chunks_created(self, previous, after, pos_created):
remain = list(after)
for p in previous:
for r in remain:
if p["url"] == r["url"]:
remain.remove(r)
break
if len(remain) != len(pos_created):
return False
for r in remain:
if r["pos"] in pos_created:
remain.remove(r)
else:
return False
return True
def _rebuild(self, event, job_id=0):
self.blob_rebuilder = subprocess.Popen(
['oio-blob-rebuilder', self.namespace,
'--beanstalkd=' + self.queue_url])
time.sleep(3)
self.blob_rebuilder.kill()
def _remove_chunks(self, chunks, content_id):
if not chunks:
return
for chunk in chunks:
chunk['id'] = chunk['url']
chunk['content'] = content_id
chunk['type'] = 'chunk'
self.container_client.container_raw_delete(
self.account, self.container, data=chunks)
def _check_rebuild(self, content_name, chunks, missing_pos, meta,
chunks_to_remove, chunk_created=True):
self._remove_chunks(chunks_to_remove, meta['id'])
event = self._create_event(content_name, chunks, missing_pos,
meta['id'])
self.notify_filter.process(env=event, cb=None)
self._rebuild(event)
_, after = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
self.assertIs(chunk_created,
self._is_chunks_created(chunks, after, missing_pos))
def test_nothing_missing(self):
content_name = "test_nothing_missing"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks_to_remove = []
for chunk in chunks:
chunk.pop('score', None)
missing_pos = []
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove, chunk_created=True)
def test_missing_1_chunk(self):
content_name = "test_missing_1_chunk"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name
)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_last_chunk(self):
content_name = "test_missing_last_chunk"
data = random_str(1024 * 1024 * 4)
self.object_storage_api.object_create(account=self.account,
container=self.container,
data=data,
policy="THREECOPIES",
obj_name=content_name
)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["3"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_2_chunks(self):
content_name = "test_missing_2_chunks"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name
)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
for i in range(0, 2):
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0", "0"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_all_chunks(self):
content_name = "test_missing_all_chunks"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="SINGLE",
obj_name=content_name
)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove, chunk_created=False)
def test_missing_all_chunks_of_a_pos(self):
content_name = "test_missing_2_chunks"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="THREECOPIES",
obj_name=content_name
)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
for i in range(0, 3):
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove, chunk_created=False)
def test_missing_multiple_chunks(self):
content_name = "test_missing_multiple_chunks"
data = random_str(1024 * 1024 * 4)
self.object_storage_api.object_create(account=self.account,
container=self.container,
data=data,
policy="THREECOPIES",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(9))
chunks_to_remove.append(chunks.pop(6))
chunks_to_remove.append(chunks.pop(4))
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0", "1", "2", "3"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_1_chunk_ec(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_1_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name
)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_m_chunk_ec(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_m_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
for i in range(0, 3):
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1", "0.2", "0.3"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_m_chunk_ec_2(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_m_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(3))
chunks_to_remove.append(chunks.pop(5))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1", "0.5", "0.8"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove)
def test_missing_m1_chunk_ec(self):
if len(self.conf['services']['rawx']) < 9:
self.skipTest("Not enough rawx. "
"EC tests needs at least 9 rawx to run")
content_name = "test_missing_m1_chunk_ec"
self.object_storage_api.object_create(account=self.account,
container=self.container,
data="test",
policy="EC",
obj_name=content_name)
meta, chunks = self.object_storage_api.object_locate(
container=self.container, obj=content_name,
account=self.account)
chunks = list(chunks)
chunks_to_remove = []
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(0))
chunks_to_remove.append(chunks.pop(0))
for chunk in chunks:
chunk.pop('score', None)
missing_pos = ["0.1", "0.2", "0.3", "0.4"]
self._check_rebuild(content_name, chunks, missing_pos, meta,
chunks_to_remove, chunk_created=False)
class TestContentVersioning(BaseTestCase):
def setUp(self):
super(TestContentVersioning, self).setUp()
self.api = ObjectStorageApi(self.conf['namespace'])
self.container = random_str(8)
system = {'sys.m2.policy.version': '3'}
self.api.container_create(self.account, self.container, system=system)
def test_versioning_enabled(self):
props = self.api.container_get_properties(
self.account, self.container)
self.assertEqual('3', props['system']['sys.m2.policy.version'])
def test_list_versions(self):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(2, len(objects))
self.assertNotEqual(objects[0]['version'], objects[1]['version'])
def test_container_purge(self):
# many contents
for i in range(0, 4):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the container configuration
self.api.container_purge(self.account, self.container)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the request
self.api.container_purge(self.account, self.container, maxvers=1)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(1, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
def test_content_purge(self):
# many contents
for i in range(0, 4):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the container configuration
self.api.container.content_purge(self.account, self.container,
"versioned")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
oldest_version = min(objects, key=lambda x: x['version'])
# use the maxvers of the request
self.api.container.content_purge(self.account, self.container,
"versioned", maxvers=1)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(1, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
# other contents
for i in range(0, 4):
self.api.object_create(self.account, self.container,
obj_name="versioned2",
data="content"+str(i))
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(5, len(objects))
# use the maxvers of the container configuration
self.api.container.content_purge(self.account, self.container,
"versioned")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(5, len(objects))
def test_delete_exceeding_version(self):
def check_num_objects_and_get_oldest_version(
expected_objects, expected_deleted_aliases, oldest_version):
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
nb_objects = 0
nb_deleted = 0
new_oldest_version = 0
for obj in objects:
if obj['deleted']:
nb_deleted += 1
else:
nb_objects += 1
if new_oldest_version == 0 \
or new_oldest_version > obj['version']:
new_oldest_version = obj['version']
self.assertEqual(expected_objects, nb_objects)
self.assertEqual(expected_deleted_aliases, nb_deleted)
if oldest_version is not None:
self.assertLess(oldest_version,
new_oldest_version)
return new_oldest_version
system = {'sys.m2.policy.version.delete_exceeding': '1'}
self.api.container_set_properties(self.account, self.container,
system=system)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
oldest_version = check_num_objects_and_get_oldest_version(
1, 0, None)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
self.assertEqual(oldest_version,
check_num_objects_and_get_oldest_version(2, 0, None))
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content2")
self.assertEqual(oldest_version,
check_num_objects_and_get_oldest_version(3, 0, None))
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content3")
oldest_version = check_num_objects_and_get_oldest_version(
3, 0, oldest_version)
self.api.object_delete(self.account, self.container, "versioned")
self.assertEqual(oldest_version,
check_num_objects_and_get_oldest_version(3, 1, None))
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content4")
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content5")
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content6")
# FIXME(adu) The deleted alias should be deleted at the same time
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content7")
oldest_version = check_num_objects_and_get_oldest_version(
3, 1, oldest_version)
def test_change_flag_delete_exceeding_versions(self):
def check_num_objects(expected):
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(expected, len(objects))
for i in range(5):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content"+str(i))
check_num_objects(5)
system = {'sys.m2.policy.version.delete_exceeding': '1'}
self.api.container_set_properties(self.account, self.container,
system=system)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content5")
check_num_objects(3)
for i in range(6, 10):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content"+str(i))
check_num_objects(3)
system['sys.m2.policy.version.delete_exceeding'] = '0'
self.api.container_set_properties(self.account, self.container,
system=system)
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content11")
check_num_objects(4)
def test_purge_objects_with_delete_marker(self):
def check_num_objects(expected):
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(expected, len(objects))
for i in range(5):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content"+str(i))
check_num_objects(5)
self.api.object_delete(self.account, self.container, "versioned")
self.assertRaises(NoSuchObject, self.api.object_locate, self.account,
self.container, "versioned")
check_num_objects(6)
self.api.container.content_purge(
self.account, self.container, "versioned")
self.assertRaises(NoSuchObject, self.api.object_locate, self.account,
self.container, "versioned")
check_num_objects(4)
system = {'sys.m2.keep_deleted_delay': '1'}
self.api.container_set_properties(self.account, self.container,
system=system)
time.sleep(2)
self.api.container.content_purge(
self.account, self.container, "versioned")
check_num_objects(0)
def test_list_objects(self):
resp = self.api.object_list(self.account, self.container)
self.assertEqual(0, len(list(resp['objects'])))
self.assertFalse(resp.get('truncated'))
def _check_objects(expected_objects, objects):
self.assertEqual(len(expected_objects), len(objects))
for i in range(len(expected_objects)):
self.assertEqual(
expected_objects[i]['name'], objects[i]['name'])
self.assertEqual(
int(expected_objects[i]['version']),
int(objects[i]['version']))
self.assertEqual(
true_value(expected_objects[i]['deleted']),
true_value(objects[i]['deleted']))
all_versions = dict()
def _create_object(obj_name, all_versions):
self.api.object_create(
self.account, self.container, obj_name=obj_name, data="test")
versions = all_versions.get(obj_name, list())
versions.append(self.api.object_show(
self.account, self.container, obj_name))
all_versions[obj_name] = versions
def _delete_object(obj_name, all_versions):
self.api.object_delete(
self.account, self.container, obj_name)
versions = all_versions.get(obj_name, list())
versions.append(self.api.object_show(
self.account, self.container, obj_name))
all_versions[obj_name] = versions
def _get_current_objects(all_versions):
current_objects = list()
obj_names = sorted(all_versions.keys())
for obj_name in obj_names:
obj = all_versions[obj_name][-1]
if not true_value(obj['deleted']):
current_objects.append(obj)
return current_objects
def _get_object_versions(all_versions):
object_versions = list()
obj_names = sorted(all_versions.keys())
for obj_name in obj_names:
versions = all_versions[obj_name]
versions.reverse()
object_versions += versions
versions.reverse()
return object_versions
# 0 object
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
# 3 objects with 1 version
for i in range(3):
_create_object("versioned"+str(i), all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects[:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects[:1], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', limit=1)
_check_objects(expected_current_objects[1:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True)
_check_objects(expected_object_versions[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True,
limit=3)
_check_objects(expected_object_versions[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
# 3 objects with 2 versions
for i in range(3):
_create_object("versioned"+str(i), all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects[:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects[:1], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', limit=1)
_check_objects(expected_current_objects[1:2], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True)
_check_objects(expected_object_versions[2:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True,
limit=3)
_check_objects(expected_object_versions[2:5], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned2', resp['next_marker'])
# 3 objects with 2 versions and 1 object with delete marker
_delete_object("versioned1", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects[:1], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', limit=1)
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True)
_check_objects(expected_object_versions[2:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True,
limit=3)
_check_objects(expected_object_versions[2:5], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
# 3 objects with 2 versions and 2 objects with delete marker
_delete_object("versioned0", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0')
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True)
_check_objects(expected_object_versions[3:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True,
limit=3)
_check_objects(expected_object_versions[3:6], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
# 3 objects with 2 versions and 3 objects with delete marker
_delete_object("versioned2", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0')
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True)
_check_objects(expected_object_versions[3:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True,
limit=3)
_check_objects(expected_object_versions[3:6], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
# 3 objects with 2 versions and 3 objects with delete marker
# (1 current version and 2 non current versions)
_create_object("versioned0", all_versions)
expected_current_objects = _get_current_objects(all_versions)
expected_object_versions = _get_object_versions(all_versions)
resp = self.api.object_list(self.account, self.container)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=3)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=2)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
limit=1)
_check_objects(expected_current_objects, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True)
_check_objects(expected_object_versions, list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
versions=True, limit=3)
_check_objects(expected_object_versions[:3], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned0', resp['next_marker'])
resp = self.api.object_list(self.account, self.container,
marker='versioned0')
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', limit=1)
_check_objects(expected_current_objects[1:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True)
_check_objects(expected_object_versions[4:], list(resp['objects']))
self.assertFalse(resp.get('truncated'))
resp = self.api.object_list(self.account, self.container,
marker='versioned0', versions=True,
limit=3)
_check_objects(expected_object_versions[4:7], list(resp['objects']))
self.assertTrue(resp.get('truncated'))
self.assertEqual('versioned1', resp['next_marker'])
class TestContentVersioning(BaseTestCase):
def setUp(self):
super(TestContentVersioning, self).setUp()
self.api = ObjectStorageApi(self.conf['namespace'])
self.container = random_str(8)
system = {'sys.m2.policy.version': '3'}
self.api.container_create(self.account, self.container, system=system)
def test_versioning_enabled(self):
props = self.api.container_get_properties(
self.account, self.container)
self.assertEqual('3', props['system']['sys.m2.policy.version'])
def test_list_versions(self):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(2, len(objects))
self.assertNotEqual(objects[0]['version'], objects[1]['version'])
def test_purge(self):
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content0")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content1")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content2")
self.api.object_create(self.account, self.container,
obj_name="versioned", data="content3")
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(4, len(objects))
oldest_version = min(objects, lambda x: x['version'])
self.api.container.container_purge(self.account, self.container)
listing = self.api.object_list(self.account, self.container,
versions=True)
objects = listing['objects']
self.assertEqual(3, len(objects))
self.assertNotIn(oldest_version, [x['version'] for x in objects])
class TestContainerDownload(BaseTestCase):
def setUp(self):
super(TestContainerDownload, self).setUp()
if self.is_running_on_public_ci():
self.skipTest("Too buggy to run on public CI")
# FIXME: should we use direct API from BaseTestCase
# or still container.client ?
self.conn = ObjectStorageApi(self.ns)
self._streaming = 'http://' + self.get_service_url('container')[2]
self._cnt = random_container()
self._uri = self.make_uri('dump')
self._data = {}
self.conn.container_create(self.account, self._cnt)
self.raw = ""
self._slo = []
def make_uri(self, action, account=None, container=None):
account = account or self.account
container = container or self._cnt
return '%s/v1.0/container/%s?acct=%s&ref=%s' % (
self._streaming, action, account, container)
def tearDown(self):
for name in self._data:
self.conn.object_delete(self.account, self._cnt, name)
self.conn.container_delete(self.account, self._cnt)
super(TestContainerDownload, self).tearDown()
def _create_data(self,
name=gen_names,
metadata=None,
size=513,
append=False):
for idx, _name in itertools.islice(name(), 5):
mime = random.choice(MIMETYPE)
if append and size > 0:
data = gen_data(size // 2 * idx)
entry = {'data': data, 'meta': None, 'mime': mime}
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data,
mime_type=mime)
data = gen_data(size // 2 * idx)
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data,
mime_type=mime,
append=True)
entry['data'] += data
else:
data = gen_data(size * idx)
entry = {'data': data, 'meta': None, 'mime': mime}
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data,
mime_type=mime)
if metadata:
entry['meta'] = dict()
for _ in xrange(10):
key, val = metadata()
entry['meta'][key] = val
self.conn.object_update(self.account, self._cnt, _name,
entry['meta'])
self._data[_name] = entry
def _create_s3_slo(self, name=gen_names, metadata=None):
# create a fake S3 bucket with a SLO object
chunksize = 10000
parts = 5
res = []
full_data = b''
self.conn.container_create(self.account, self._cnt + '+segments')
_name = "toto"
etag = rand_str(50)
part_number = 1
for size in [chunksize] * parts + [444]:
data = gen_data(size)
res.append({
'bytes':
size,
'content_type':
'application/octect-stream',
'hash':
md5(data).hexdigest().upper(),
'last_modified':
'2017-06-21T12:42:47.000000',
'name':
'/%s+segments/%s/%s/%d' % (self._cnt, _name, etag, part_number)
})
self.conn.object_create(self.account,
"%s+segments" % self._cnt,
obj_name='%s/%s/%d' %
(_name, etag, part_number),
data=data)
full_data += data
part_number += 1
self._data[_name] = {
'data': full_data,
'meta': {
'x-static-large-object': 'true',
'x-object-sysmeta-slo-etag': etag,
'x-object-sysmeta-slo-size': str(len(full_data))
}
}
self._slo.append(_name)
data = json.dumps(res)
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data)
self.conn.object_update(self.account, self._cnt, _name,
self._data[_name]['meta'])
def _check_tar(self, data):
raw = BytesIO(data)
tar = tarfile.open(fileobj=raw, ignore_zeros=True)
info = list(self._data.keys())
for entry in tar.getnames():
if entry == CONTAINER_MANIFEST:
# skip special entry
continue
self.assertIn(entry, info)
tmp = tar.extractfile(entry)
self.assertEqual(self._data[entry]['data'], tmp.read())
info.remove(entry)
self.assertEqual(info, [])
return tar
def _check_container(self, cnt):
ret = self.conn.object_list(account=self.account, container=cnt)
names = self._data.keys()
for obj in ret['objects']:
name = obj['name']
self.assertIn(name, self._data)
self.assertEqual(obj['size'], len(self._data[name]['data']))
_, data = self.conn.object_fetch(self.account, cnt, name)
raw = b''.join(data)
self.assertEqual(
md5(raw).hexdigest(),
md5(self._data[name]['data']).hexdigest())
meta = self.conn.object_get_properties(self.account, cnt, name)
self.assertEqual(meta['properties'], self._data[name]['meta'])
names.remove(name)
self.assertEqual(len(names), 0)
def _simple_download(self,
name=gen_names,
metadata=None,
size=513,
append=False):
self._create_data(name=name,
metadata=metadata,
size=size,
append=append)
ret = requests.get(self._uri)
self.assertGreater(len(ret.content), 0)
self.assertEqual(ret.status_code, 200)
self.raw = ret.content
return self._check_tar(ret.content)
def _check_metadata(self, tar):
for entry in tar.getnames():
if entry == CONTAINER_MANIFEST:
# skip special entry
continue
headers = tar.getmember(entry).pax_headers
keys = list(headers.keys())
for key, val in self._data[entry]['meta'].items():
key = u"SCHILY.xattr.user." + key.decode('utf-8')
self.assertIn(key, headers)
self.assertEqual(val.decode('utf-8'), headers[key])
keys.remove(key)
#
self.assertEqual(self._data[entry]['mime'], headers['mime_type'])
keys.remove('mime_type')
#
self.assertEqual(keys, [])
def test_missing_container(self):
ret = requests.get(self._streaming + '/' + random_container("ms-"))
self.assertEqual(ret.status_code, 404)
def test_invalid_url(self):
ret = requests.get(self._streaming)
self.assertEqual(ret.status_code, 404)
ret = requests.head(self._streaming + '/' + random_container('inv') +
'/' + random_container('inv'))
self.assertEqual(ret.status_code, 404)
def test_download_empty_container(self):
ret = requests.get(self._uri)
self.assertEqual(ret.status_code, 204)
def test_simple_download(self):
self._simple_download()
def test_check_head(self):
self._create_data()
get = requests.get(self._uri)
head = requests.head(self._uri)
self.assertEqual(get.headers['content-length'],
head.headers['content-length'])
def test_download_per_range(self):
self._create_data()
org = requests.get(self._uri)
data = []
for idx in xrange(0, int(org.headers['content-length']), 512):
ret = requests.get(
self._uri, headers={'Range': 'bytes=%d-%d' % (idx, idx + 511)})
self.assertEqual(ret.status_code, 206)
self.assertEqual(len(ret.content), 512)
self.assertEqual(ret.content, org.content[idx:idx + 512])
data.append(ret.content)
data = b''.join(data)
self.assertGreater(len(data), 0)
self.assertEqual(md5(data).hexdigest(), md5(org.content).hexdigest())
def test_invalid_range(self):
self._create_data()
ranges = ((-512, 511), (512, 0), (1, 3), (98888, 99999))
for start, end in ranges:
ret = requests.get(self._uri,
headers={'Range': 'bytes=%d-%d' % (start, end)})
self.assertEqual(
ret.status_code, 416,
"Invalid error code for range %d-%d" % (start, end))
ret = requests.get(self._uri,
headers={'Range': 'bytes=0-511, 512-1023'})
self.assertEqual(ret.status_code, 416)
def test_file_metadata(self):
tar = self._simple_download(metadata=gen_metadata)
self._check_metadata(tar)
def test_container_metadata(self):
key, val = gen_metadata()
ret = self.conn.container_update(self.account, self._cnt, {key: val})
ret = self.conn.container_show(self.account, self._cnt)
ret = requests.get(self._uri)
self.assertEqual(ret.status_code, 200)
raw = BytesIO(ret.content)
tar = tarfile.open(fileobj=raw, ignore_zeros=True)
self.assertIn(CONTAINER_PROPERTIES, tar.getnames())
data = json.load(tar.extractfile(CONTAINER_PROPERTIES))
self.assertIn(key, data)
self.assertEqual(val, data[key])
def test_charset_file(self):
self._simple_download(name=gen_charset_names)
@unittest.skip("wip")
def test_byte_metadata(self):
tar = self._simple_download(metadata=gen_byte_metadata)
self._check_metadata(tar)
def test_charset_metadata(self):
tar = self._simple_download(metadata=gen_charset_metadata)
self._check_metadata(tar)
@attr('s3')
def test_s3_simple_download(self):
self._create_s3_slo()
ret = requests.get(self._uri)
self.assertGreater(len(ret.content), 0)
self.assertEqual(ret.status_code, 200)
self.raw = ret.content
raw = BytesIO(ret.content)
tar = tarfile.open(fileobj=raw, ignore_zeros=True)
info = list(self._data.keys())
for entry in tar.getnames():
if entry == CONTAINER_MANIFEST:
# skip special entry
continue
self.assertIn(entry, info)
tmp = tar.extractfile(entry)
self.assertEqual(self._data[entry]['data'], tmp.read())
info.remove(entry)
self.assertEqual(len(info), 0)
return tar
@attr('s3')
def test_s3_range_download(self):
self._create_s3_slo()
org = requests.get(self._uri)
self.assertEqual(org.status_code, 200)
data = []
for idx in xrange(0, int(org.headers['content-length']), 512):
ret = requests.get(
self._uri, headers={'Range': 'bytes=%d-%d' % (idx, idx + 511)})
self.assertEqual(ret.status_code, 206)
self.assertEqual(len(ret.content), 512)
self.assertEqual(ret.content, org.content[idx:idx + 512])
data.append(ret.content)
data = b''.join(data)
self.assertGreater(len(data), 0)
self.assertEqual(md5(data).hexdigest(), md5(org.content).hexdigest())
@attr('s3')
def test_s3_check_slo_metadata_download(self):
self._create_s3_slo()
org = requests.get(self.make_uri('dump'))
self.assertEqual(org.status_code, 200)
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=org.content)
self.assertEqual(org.status_code, 200)
res = self.conn.object_get_properties(self.account, cnt, self._slo[0])
props = res['properties']
self.assertNotIn('x-static-large-object', props)
self.assertNotIn('x-object-sysmeta-slo-size', props)
self.assertNotIn('x-object-sysmeta-slo-etag', props)
@attr('simple')
def test_simple_restore(self):
self._create_data(metadata=gen_metadata)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=org.content)
self.assertEqual(res.status_code, 201)
self._check_container(cnt)
@attr('restore')
def test_multipart_restore(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
size = 1014 * 1024
parts = [
org.content[x:x + size] for x in xrange(0, len(org.content), size)
]
uri = self.make_uri('restore', container=cnt)
start = 0
for part in parts:
hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)}
res = requests.put(uri, data=part, headers=hdrs)
start += len(part)
self.assertIn(res.status_code, [201, 206])
self._check_container(cnt)
@attr('restore')
def test_multipart_invalid_restore(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
uri = self.make_uri('restore', container=cnt)
size = 1014 * 1024
parts = [
org.content[x:x + size] for x in xrange(0, len(org.content), size)
]
start = 0
for part in parts:
hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)}
res = requests.put(uri, data=part, headers=hdrs)
self.assertIn(res.status_code, [201, 206])
start += len(part)
# only unfinished restoration expose X-Consumed-Size
if res.status_code == 206:
res = requests.head(uri)
self.assertEqual(int(res.headers['X-Consumed-Size']), start)
inv = requests.put(uri, data=part, headers=hdrs)
self.assertEqual(inv.status_code, 422)
if res.status_code == 206:
res = requests.head(uri)
self.assertEqual(int(res.headers['X-Consumed-Size']), start)
uri = self.make_uri('restore', container=rand_str(20))
hdrs = {'Range': 'bytes=%d-%d' % (size, size + len(parts[1]) - 1)}
res = requests.put(uri, data=part, headers=hdrs)
self.assertEqual(res.status_code, 422)
self._check_container(cnt)
@attr('concurrency')
def test_multipart_concurrency(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
uri = self.make_uri('restore', container=cnt)
size = divmod(len(org.content) // 3, 512)[0] * 512
parts = [
org.content[x:x + size] for x in xrange(0, len(org.content), size)
]
start = 0
class StreamWithContentLength(Thread):
"""Thread to send data with delays to restore API"""
def __init__(self, data, headers):
self._count = 0
self._data = data
self._hdrs = headers
super(StreamWithContentLength, self).__init__()
def __len__(self):
return len(self._data)
def read(self, *args):
if self._count < len(self._data):
time.sleep(0.5)
data = self._data[self._count:self._count + size // 3]
self._count += len(data)
return data
return b''
def run(self):
self._ret = requests.put(uri, data=self, headers=self._hdrs)
for idx, part in enumerate(parts):
hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)}
if idx == 0:
res = requests.put(uri, data=part, headers=hdrs)
self.assertIn(res.status_code, [201, 206])
else:
# launch Thread and simulate slow bandwidth
thr = StreamWithContentLength(part, hdrs)
thr.start()
# send data on same range
time.sleep(0.5)
res = requests.put(uri, data=part, headers=hdrs)
self.assertEqual(res.status_code, 422)
thr.join()
self.assertIn(thr._ret.status_code, [201, 206])
start += len(part)
self._check_container(cnt)
@attr('disconnected')
def test_broken_connectivity(self):
# TODO remove this ASAP, as soon as the test has been fixed
if self.is_running_on_public_ci():
self.skipTest("Too buggy to run on public CI")
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
class FakeStream(object):
"""Send data and simulate a connectivity issue"""
def __init__(self, data, size):
self._count = 0
self._data = data
self._size = size
def __len__(self):
return len(self._data)
def read(self, *args):
if self._count < self._size:
data = self._data[self._count:self._count + size // 3]
self._count += len(data)
return data
if self._count == len(self._data):
return b''
raise Exception("break connection")
def wait_lock():
"""When the lock is gone, return current consumed size"""
nb = 0
while True:
time.sleep(0.1)
req = requests.head(uri)
if (req.status_code == 200 and req.headers.get(
'X-Upload-In-Progress', '1') == '0'):
print("Tried before lock free", nb)
print("Got consumed-size", req.headers['X-Consumed-Size'])
return int(req.headers['X-Consumed-Size'])
nb += 1
self.assertLess(nb, 10)
uri = self.make_uri('restore', container=cnt)
block = 1000 * 512
start = 0
cut = False
while True:
if start:
start = wait_lock()
stop = min(len(org.content), start + block)
hdrs = {'Range': 'bytes=%d-%d' % (start, stop - 1)}
size = stop - start
if cut:
size = block // 2
cut = not cut
try:
ret = requests.put(uri,
headers=hdrs,
data=FakeStream(org.content[start:stop],
size))
except Exception:
pass
else:
self.assertIn(
ret.status_code, (201, 206),
"Unexpected %d HTTP response: %s" %
(ret.status_code, ret.content))
start += size
if ret.status_code == 201:
break
result = requests.get(self.make_uri('dump', container=cnt))
self._check_tar(result.content)
@attr('rawtar')
def test_rawtar(self):
"""Create a normal tar archive and restore it"""
raw = BytesIO()
tarfile = TarFile(mode='w', fileobj=raw)
testdata = (rand_str(20) * 5000).encode('utf-8')
inf = TarInfo("simpletar")
fileraw = BytesIO()
fileraw.write(testdata)
inf.size = len(testdata)
fileraw.seek(0)
tarfile.addfile(inf, fileobj=fileraw)
tarfile.close()
raw.seek(0)
data = raw.read()
cnt = rand_str(20)
ret = requests.put(self.make_uri("restore", container=cnt), data=data)
self.assertEqual(ret.status_code, 201)
meta, stream = self.conn.object_fetch(self.account, cnt, "simpletar")
self.assertEqual(
md5(b''.join(stream)).hexdigest(),
md5(testdata).hexdigest())
@attr('invalid')
def test_checksums(self):
"""Check restore operation with invalid tar"""
# TODO remove this ASAP, as soon as the test has been fixed
if self.is_running_on_public_ci():
self.skipTest("Too buggy to run on public CI")
tar = self._simple_download(append=True)
manifest = json.load(tar.extractfile(CONTAINER_MANIFEST),
object_pairs_hook=OrderedDict)
# => add random bytes inside each file (either header and data)
for entry in manifest:
if entry['name'] == CONTAINER_MANIFEST:
# CONTAINER_MANIFEST does not have checksum at this time
continue
inv = self.raw
# Test with tar entry
# checksum tar doesn't work very well with SCHILY attributes
# so only apply changes on regular block entry
idx = entry['start_block'] * BLOCKSIZE \
+ random.randint(0, BLOCKSIZE)
# + random.randint(0, entry['hdr_blocks'] * BLOCKSIZE)
while self.raw[idx] == inv[idx]:
inv = (inv[:idx] +
chr(random.randint(0, 255)).encode('utf-8') +
inv[idx + 1:])
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=inv)
self.assertEqual(res.status_code, 400)
# skip emty file
if entry['size'] == 0:
continue
# Test with data blocks
inv = self.raw
idx = (entry['start_block'] + entry['hdr_blocks']) * BLOCKSIZE \
+ random.randint(0, entry['size'] - 1)
while self.raw[idx] == inv[idx]:
inv = inv[:idx] + chr(random.randint(0, 255)) + inv[idx + 1:]
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=inv)
self.assertEqual(res.status_code, 400)
class TestBlobConverter(BaseTestCase):
def setUp(self):
super(TestBlobConverter, self).setUp()
self.container = random_str(16)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(
self.account, self.container, self.path, size=1)
services = self.conscience.all_services('rawx')
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(
self.account, self.container, obj_name=self.path, data="chunk")
meta, self.chunks = self.api.object_locate(
self.account, self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
self.container_id = cid_from_name(self.account, self.container)
def tearDown(self):
try:
self.api.object_delete(self.account, self.container, self.path)
except Exception:
pass
super(TestBlobConverter, self).tearDown()
def _chunk_path(self, chunk):
url = chunk['url']
chunk_id = url.split('/', 3)[3]
volume = self.rawx_volumes[self._chunk_volume_id(chunk)]
return volume + '/' + chunk_id[:3] + '/' + chunk_id
def _chunk_volume_id(self, chunk):
return chunk['url'].split('/', 3)[2]
def _deindex_chunk(self, chunk):
rdir = RdirClient(self.conf, pool_manager=self.conscience.pool_manager)
url = chunk['url']
volume_id = url.split('/', 3)[2]
chunk_id = url.split('/', 3)[3]
rdir.chunk_delete(volume_id, self.container_id,
self.content_id, chunk_id)
def _convert_and_check(self, chunk_volume, chunk_path,
chunk_id_info, expected_raw_meta=None,
expected_errors=0):
conf = self.conf
conf['volume'] = self.rawx_volumes[chunk_volume]
converter = BlobConverter(conf, logger=self.logger)
converter.safe_convert_chunk(chunk_path)
self.assertEqual(1, converter.total_chunks_processed)
self.assertEqual(1, converter.passes)
self.assertEqual(expected_errors, converter.errors)
checker = Checker(self.ns)
for chunk_id, info in chunk_id_info.items():
account, container, path, version, content_id = info
fullpath = encode_fullpath(
account, container, path, version, content_id)
cid = cid_from_name(account, container)
meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id)
self.assertEqual(meta.get('chunk_id'), chunk_id)
self.assertEqual(meta.get('container_id'), cid)
self.assertEqual(meta.get('content_path'), path)
self.assertEqual(meta.get('content_version'), version)
self.assertEqual(meta.get('content_id'), content_id)
self.assertEqual(meta.get('full_path'), fullpath)
checker.check(Target(
account, container=container, obj=path,
chunk='http://' + converter.volume_id + '/' + chunk_id))
for _ in checker.run():
pass
self.assertTrue(checker.report())
if expected_raw_meta:
self.assertDictEqual(expected_raw_meta, raw_meta)
continue
self.assertNotIn(chunk_xattr_keys['chunk_id'], raw_meta)
self.assertNotIn(chunk_xattr_keys['container_id'], raw_meta)
self.assertNotIn(chunk_xattr_keys['content_path'], raw_meta)
self.assertNotIn(chunk_xattr_keys['content_version'], raw_meta)
self.assertNotIn(chunk_xattr_keys['content_id'], raw_meta)
self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id,
raw_meta)
for k in raw_meta.keys():
if k.startswith('oio:'):
self.fail('old fullpath always existing')
self.assertEqual(raw_meta[chunk_xattr_keys['oio_version']],
OIO_VERSION)
def _test_converter_single_chunk(self, chunk, expected_errors=0):
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)},
expected_errors=expected_errors)
def test_converter(self):
chunk = random.choice(self.chunks)
self._test_converter_single_chunk(chunk)
def test_converter_old_chunk(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_wrong_path(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '+', self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_wrong_content_id(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, '0123456789ABCDEF0123456789ABCDEF')
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_old_fullpath(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id, add_old_fullpath=True)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_old_fullpath_and_wrong_path(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path, self.version, self.content_id,
add_old_fullpath=True)
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '+', self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_old_chunk_with_wrong_fullpath(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path, 'None', '0123456789ABCDEF0123456789ABCDEF',
add_old_fullpath=True)
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path, self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
def test_converter_linked_chunk(self):
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
for c in linked_chunks:
if chunk_volume == c['url'].split('/')[2]:
linked_chunk_id2 = c['url'].split('/')[3]
break
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
for c in self.chunks:
if linked_chunk_volume == c['url'].split('/')[2]:
chunk_id2 = c['url'].split('/')[3]
break
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id2: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
self._convert_and_check(
linked_chunk_volume, linked_chunk_path,
{chunk_id2: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
def test_converter_old_linked_chunk(self):
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
for c in linked_chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '.link', 'None',
'0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True)
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
for c in linked_chunks:
if chunk_volume == c['url'].split('/')[2]:
linked_chunk_id2 = c['url'].split('/')[3]
break
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
for c in self.chunks:
if linked_chunk_volume == c['url'].split('/')[2]:
chunk_id2 = c['url'].split('/')[3]
break
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id2: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
self._convert_and_check(
linked_chunk_volume, linked_chunk_path,
{chunk_id2: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
def test_converter_old_chunk_with_link_on_same_object(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path)
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, linked_meta['id'])
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
# old xattr not removed
_, expected_raw_meta = read_chunk_metadata(linked_chunk_path,
linked_chunk_id)
self._convert_and_check(
linked_chunk_volume, linked_chunk_path,
{linked_chunk_id: (self.account, self.container,
self.path, linked_meta['version'],
linked_meta['id'])},
expected_raw_meta=expected_raw_meta, expected_errors=1)
def test_converter_old_linked_chunk_with_link_on_same_object(self):
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
for c in linked_chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container,
self.path + '.link', 'None',
'0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True)
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id, add_old_fullpath=True)
self.api.object_link(
self.account, self.container, self.path + '.link',
self.account, self.container, self.path + '.link')
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path + '.link')
self.assertNotEqual(self.content_id, linked_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
for c in linked_chunks:
if chunk_volume == c['url'].split('/')[2]:
linked_chunk_id2 = c['url'].split('/')[3]
break
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
for c in self.chunks:
if linked_chunk_volume == c['url'].split('/')[2]:
chunk_id2 = c['url'].split('/')[3]
break
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id2: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
self._convert_and_check(
linked_chunk_volume, linked_chunk_path,
{chunk_id2: (self.account, self.container, self.path, self.version,
self.content_id),
linked_chunk_id: (self.account, self.container,
self.path + '.link', linked_meta['version'],
linked_meta['id'])})
def test_converter_with_versioning(self):
self.api.container_set_properties(
self.account, self.container,
system={'sys.m2.policy.version': '2'})
self.api.object_create(
self.account, self.container, obj_name=self.path, data='version')
versioned_meta, versioned_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, versioned_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
versioned_chunk = random.choice(versioned_chunks)
versioned_chunk_volume = versioned_chunk['url'].split('/')[2]
versioned_chunk_id = versioned_chunk['url'].split('/')[3]
versioned_chunk_path = self._chunk_path(versioned_chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
self._convert_and_check(
versioned_chunk_volume, versioned_chunk_path,
{versioned_chunk_id: (self.account, self.container, self.path,
versioned_meta['version'],
versioned_meta['id'])})
def test_converter_old_chunk_with_versioning(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
self.api.container_set_properties(
self.account, self.container,
system={'sys.m2.policy.version': '2'})
self.api.object_create(
self.account, self.container, obj_name=self.path, data='version')
versioned_meta, versioned_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, versioned_meta['id'])
for c in versioned_chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
versioned_meta['version'], versioned_meta['id'])
chunk = random.choice(self.chunks)
chunk_volume = chunk['url'].split('/')[2]
chunk_id = chunk['url'].split('/')[3]
chunk_path = self._chunk_path(chunk)
versioned_chunk = random.choice(versioned_chunks)
versioned_chunk_volume = versioned_chunk['url'].split('/')[2]
versioned_chunk_id = versioned_chunk['url'].split('/')[3]
versioned_chunk_path = self._chunk_path(versioned_chunk)
self._convert_and_check(
chunk_volume, chunk_path,
{chunk_id: (self.account, self.container, self.path, self.version,
self.content_id)})
self._convert_and_check(
versioned_chunk_volume, versioned_chunk_path,
{versioned_chunk_id: (self.account, self.container, self.path,
versioned_meta['version'],
versioned_meta['id'])})
def test_converter_file_not_found(self):
"""
Test what happens when the BlobConverter encounters a chunk with
neither a fullpath extended attribute, not any of the legacy
attributes.
"""
victim = random.choice(self.chunks)
path = self._chunk_path(victim)
chunk_volume = victim['url'].split('/')[2]
os.remove(path)
with patch('oio.blob.converter.BlobConverter.recover_chunk_fullpath') \
as recover:
self._convert_and_check(chunk_volume, path, {}, expected_errors=1)
recover.assert_not_called()
def test_recover_missing_old_fullpath(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
victim = random.choice(self.chunks)
self._test_converter_single_chunk(victim)
def test_recover_missing_content_path(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id, add_old_fullpath=True)
victim = random.choice(self.chunks)
path = self._chunk_path(victim)
remove_xattr(path, chunk_xattr_keys['content_path'])
self._test_converter_single_chunk(victim)
def test_recover_missing_old_fullpath_and_content_path(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
victim = random.choice(self.chunks)
path = self._chunk_path(victim)
remove_xattr(path, chunk_xattr_keys['content_path'])
self._test_converter_single_chunk(victim)
def test_recover_missing_fullpath(self):
"""
Test what happens when the BlobConverter encounters a chunk with
neither a fullpath extended attribute, not any of the legacy
attributes.
"""
victim = random.choice(self.chunks)
path = self._chunk_path(victim)
remove_fullpath_xattr(path)
self._test_converter_single_chunk(victim)
def test_recover_missing_fullpath_not_indexed(self):
"""
Test what happens when the BlobConverter encounters a chunk with
neither a fullpath extended attribute, not any of the legacy
attributes, and the chunk does not appear in rdir.
"""
victim = random.choice(self.chunks)
path = self._chunk_path(victim)
remove_fullpath_xattr(path)
self._deindex_chunk(victim)
conf = dict(self.conf)
conf['volume'] = self.rawx_volumes[self._chunk_volume_id(victim)]
converter = BlobConverter(conf)
self.assertRaises(KeyError, converter.recover_chunk_fullpath, path)
def test_recover_missing_fullpath_orphan_chunk(self):
"""
Test what happens when the BlobConverter encounters a chunk with
neither a fullpath extended attribute, not any of the legacy
attributes, and the chunk does not appear in object description.
"""
victim = random.choice(self.chunks)
path = self._chunk_path(victim)
remove_fullpath_xattr(path)
cbean = {
'content': self.content_id,
'hash': victim['hash'],
'id': victim['url'],
'size': victim['size'],
'pos': victim['pos'],
'type': 'chunk'
}
self.api.container.container_raw_delete(
self.account, self.container, data=[cbean])
conf = dict(self.conf)
conf['volume'] = self.rawx_volumes[self._chunk_volume_id(victim)]
converter = BlobConverter(conf)
self.assertRaises(OrphanChunk, converter.recover_chunk_fullpath, path)
class TestMeta2Database(BaseTestCase):
def setUp(self):
super(TestMeta2Database, self).setUp()
self.api = ObjectStorageApi(self.ns)
self.account = "test_meta2_database"
self.reference = "meta2_database_" + random_str(4)
self.meta2_database = Meta2Database(self.conf)
self.service_type = 'meta2'
def _get_peers(self):
linked_services = self.api.directory.list(self.account, self.reference)
peers = list()
for service in linked_services['srv']:
if service['type'] == self.service_type:
peers.append(service['host'])
return peers
def _test_move(self, base=None, fixed_dst=True):
if base is None:
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
all_meta2_services = self.conscience.all_services(
self.service_type, True)
if len(all_meta2_services) <= len(current_peers):
self.skipTest("need at least %d more %s"
% (len(current_peers)+1, self.service_type))
expected_peers = list(current_peers)
src = random.choice(current_peers)
expected_peers.remove(src)
dst = None
if fixed_dst:
for service in all_meta2_services:
if service['id'] not in current_peers:
dst = service['id']
expected_peers.append(dst)
moved = self.meta2_database.move(base, src, dst=dst)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
if fixed_dst:
self.assertEqual(dst, moved[0]['dst'])
self.assertIsNone(moved[0]['err'])
new_peers = self._get_peers()
if fixed_dst:
self.assertListEqual(sorted(expected_peers), sorted(new_peers))
else:
for expected_service in expected_peers:
self.assertIn(expected_service, new_peers)
self.assertNotIn(src, new_peers)
self.assertEqual(len(expected_peers)+1, len(new_peers))
if self.service_type == 'meta2':
properties = self.api.container_get_properties(
self.account, self.reference)
peers = properties['system']['sys.peers']
new_peers_bis = peers.split(',')
self.assertListEqual(sorted(new_peers), sorted(new_peers_bis))
return (src, expected_peers)
def test_move(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
self._test_move()
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_with_seq(self):
self.api.container_create(self.account, self.reference)
properties = self.api.container_get_properties(
self.account, self.reference)
base = properties['system']['sys.name']
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
self._test_move(base=base)
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_without_dst(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
self._test_move(fixed_dst=False)
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_with_src_not_used(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
src = None
all_meta2_services = self.conscience.all_services('meta2', True)
for service in all_meta2_services:
if service['id'] not in current_peers:
src = service['id']
if src is None:
self.skipTest("need at least 1 more meta2")
moved = self.meta2_database.move(base, src)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertIsNone(moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_dst_already_used(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
src = random.choice(current_peers)
dst = random.choice(current_peers)
moved = self.meta2_database.move(base, src, dst=dst)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertEqual(dst, moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_invalid_src(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
src = '127.0.0.1:666'
moved = self.meta2_database.move(base, src)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertIsNone(moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_invalid_dst(self):
self.api.container_create(self.account, self.reference)
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
src = random.choice(current_peers)
dst = '127.0.0.1:666'
moved = self.meta2_database.move(base, src, dst=dst)
moved = list(moved)
self.assertEqual(1, len(moved))
self.assertTrue(moved[0]['base'].startswith(base))
self.assertEqual(src, moved[0]['src'])
self.assertEqual(dst, moved[0]['dst'])
self.assertIsNotNone(moved[0]['err'])
def test_move_with_1_missing_base(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
if len(current_peers) <= 1:
self.skipTest('need replicated bases')
to_remove = random.choice(current_peers)
self.admin.remove_base(self.service_type, cid=base,
service_id=to_remove)
self._test_move()
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
def test_move_with_1_remaining_base(self):
self.api.container_create(self.account, self.reference)
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test1")
base = cid_from_name(self.account, self.reference)
current_peers = self._get_peers()
if len(current_peers) <= 1:
self.skipTest('need replicated bases')
to_remove = list(current_peers)
to_remove.remove(random.choice(current_peers))
self.admin.remove_base(self.service_type, cid=base,
service_id=to_remove)
self._test_move()
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test1")
self.api.object_create(self.account, self.reference,
data="move meta2", obj_name="test2")
for _ in range(0, 5):
self.api.object_show(self.account, self.reference, "test2")
