def _test_index_chunk(self, alias="toto"):
# create a fake chunk
chunk_path, container_id, content_id, chunk_id = self._create_chunk(
self.rawx_conf['path'], alias)
# index the chunk
indexer = BlobIndexer(self.conf)
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=1234)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
self.assertEqual(check_value['mtime'], 1234)
# index a chunk already indexed
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=4567)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
self.assertEqual(check_value['mtime'], 4567)
def _test_index_chunk(self, alias="toto"):
# create a fake chunk
chunk_path, container_id, content_id, chunk_id = self._create_chunk(
self.rawx_conf['path'], alias)
# index the chunk
indexer = BlobIndexer(self.conf)
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=1234)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
# _rdir_get returns unicode
self.assertEqual(check_value['content_nbchunks'], 1)
self.assertEqual(check_value['chunk_hash'], 32 * '0')
self.assertEqual(check_value['content_size'], 4)
self.assertEqual(check_value['content_path'], alias.decode("utf8"))
self.assertEqual(check_value['chunk_position'], '0')
self.assertEqual(check_value['chunk_size'], 4)
self.assertEqual(check_value['mtime'], 1234)
self.assertEqual(check_value['content_version'], 0)
# index a chunk already indexed
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=4567)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
self.assertEqual(check_value['mtime'], 4567)
def setUp(self):
super(TestBlobIndexer, self).setUp()
self.rdir_client = RdirClient(self.conf)
self.blob_client = BlobClient(self.conf)
_, self.rawx_path, rawx_addr, _ = \
self.get_service_url('rawx')
services = self.conscience.all_services('rawx')
self.rawx_id = None
for rawx in services:
if rawx_addr == rawx['addr']:
self.rawx_id = rawx['tags'].get('tag.service_id', None)
if self.rawx_id is None:
self.rawx_id = rawx_addr
conf = self.conf.copy()
conf['volume'] = self.rawx_path
self.blob_indexer = BlobIndexer(conf)
# clear rawx/rdir
chunk_files = paths_gen(self.rawx_path)
for chunk_file in chunk_files:
os.remove(chunk_file)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
def test_index_chunk_missing_xattr(self):
# create a fake chunk
chunk_path, container_id, content_id, chunk_id = self._create_chunk(
self.rawx_conf['path'])
# remove mandatory xattr
xattr.removexattr(chunk_path, 'user.grid.chunk.hash')
# try to index the chunk
indexer = BlobIndexer(self.conf)
self.assertRaises(FaultyChunk, indexer.update_index, chunk_path)
def _test_index_chunk(self, alias="toto"):
# create a fake chunk
chunk_path, container_id, content_id, chunk_id = self._create_chunk(
self.rawx_conf['path'], alias)
# index the chunk
indexer = BlobIndexer(self.conf)
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=1234)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
self.assertEqual(check_value['mtime'], 1234)
# index a chunk already indexed
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=4567)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
self.assertEqual(check_value['mtime'], 4567)
def test_index_chunk_with_wrong_paths(self):
indexer = BlobIndexer(self.conf)
# try to index the chunks
indexer.index_pass()
successes = indexer.successes
errors = indexer.errors
# create fake chunks
chunk_path1, _, _, _ = self._create_chunk(self.rawx_conf['path'] +
".pending")
chunk_path2, _, _, _ = self._create_chunk(self.rawx_conf['path'][:-1] +
'G')
chunk_path3, _, _, _ = self._create_chunk(self.rawx_conf['path'] + '0')
chunk_path4, _, _, _ = self._create_chunk(self.rawx_conf['path'][:-1])
# try to index the chunks
indexer.index_pass()
self.assertEqual(indexer.successes, successes)
self.assertEqual(indexer.errors, errors)
os.remove(chunk_path1)
os.remove(chunk_path2)
os.remove(chunk_path3)
os.remove(chunk_path4)
def test_index_chunk_with_wrong_paths(self):
indexer = BlobIndexer(self.conf)
# try to index the chunks
indexer.index_pass()
successes = indexer.successes
errors = indexer.errors
# create fake chunks
chunk_path1, _, _, _ = self._create_chunk(
self.rawx_conf['path'], suffix=".pending")
chunk_path2, _, _, _ = self._create_chunk(
self.rawx_conf['path'][:-1] + 'G')
chunk_path3, _, _, _ = self._create_chunk(
self.rawx_conf['path'] + '0')
chunk_path4, _, _, _ = self._create_chunk(
self.rawx_conf['path'][:-1])
# try to index the chunks
indexer.index_pass()
# All chunks we have created have invalid names,
# the success count should not increase.
# The indexer should print a warning, and ignore them.
self.assertEqual(indexer.successes, successes)
self.assertEqual(indexer.errors, errors)
os.remove(chunk_path1)
os.remove(chunk_path2)
os.remove(chunk_path3)
os.remove(chunk_path4)
def test_index_chunk_missing_xattr(self):
# create a fake chunk
chunk_path, container_id, content_id, chunk_id = self._create_chunk(
self.rawx_conf['path'])
# remove mandatory xattr
# pylint: disable=no-member
xattr.removexattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'])
# try to index the chunk
indexer = BlobIndexer(self.conf)
self.assertRaises(FaultyChunk, indexer.update_index, chunk_path,
chunk_id)
os.remove(chunk_path)
def _test_index_chunk(self, alias="toto"):
# create a fake chunk
chunk_path, container_id, content_id, chunk_id = self._create_chunk(
self.rawx_conf['path'], alias)
# index the chunk
indexer = BlobIndexer(self.conf)
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=1234)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
# _rdir_get returns unicode
self.assertEqual(check_value['content_nbchunks'], 1)
self.assertEqual(check_value['chunk_hash'], 32 * '0')
self.assertEqual(check_value['content_size'], 4)
self.assertEqual(check_value['content_path'], alias.decode("utf8"))
self.assertEqual(check_value['chunk_position'], '0')
self.assertEqual(check_value['chunk_size'], 4)
self.assertEqual(check_value['mtime'], 1234)
self.assertEqual(check_value['content_version'], 0)
# index a chunk already indexed
with mock.patch('oio.blob.indexer.time.time',
mock.MagicMock(return_value=4567)):
indexer.update_index(chunk_path)
# check rdir
check_value = self._rdir_get(self.rawx_conf['addr'], container_id,
content_id, chunk_id)
self.assertIsNotNone(check_value)
self.assertEqual(check_value['mtime'], 4567)
class TestBlobIndexer(BaseTestCase):
def setUp(self):
super(TestBlobIndexer, self).setUp()
self.rdir_client = RdirClient(self.conf)
self.blob_client = BlobClient(self.conf)
_, self.rawx_path, rawx_addr, _ = \
self.get_service_url('rawx')
services = self.conscience.all_services('rawx')
self.rawx_id = None
for rawx in services:
if rawx_addr == rawx['addr']:
self.rawx_id = rawx['tags'].get('tag.service_id', None)
if self.rawx_id is None:
self.rawx_id = rawx_addr
conf = self.conf.copy()
conf['volume'] = self.rawx_path
self.blob_indexer = BlobIndexer(conf)
# clear rawx/rdir
chunk_files = paths_gen(self.rawx_path)
for chunk_file in chunk_files:
os.remove(chunk_file)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
def _put_chunk(self):
account = random_str(16)
container = random_str(16)
cid = cid_from_name(account, container)
content_path = random_str(16)
content_version = 1234567890
content_id = random_id(32)
fullpath = encode_fullpath(account, container, content_path,
content_version, content_id)
chunk_id = random_chunk_id()
data = random_buffer(string.printable, 100)
meta = {
'full_path': fullpath,
'container_id': cid,
'content_path': content_path,
'version': content_version,
'id': content_id,
'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'policy': 'TESTPOLICY',
'chunk_hash': md5(data).hexdigest().upper(),
'oio_version': OIO_VERSION,
'chunk_pos': 0,
'metachunk_hash': md5().hexdigest(),
'metachunk_size': 1024
}
self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id,
meta, data)
sleep(1) # ensure chunk event have been processed
return account, container, cid, content_path, content_version, \
content_id, chunk_id
def _delete_chunk(self, chunk_id):
self.blob_client.chunk_delete('http://' + self.rawx_id + '/' +
chunk_id)
sleep(1) # ensure chunk event have been processed
def _link_chunk(self, target_chunk_id):
account = random_str(16)
container = random_str(16)
cid = cid_from_name(account, container)
content_path = random_str(16)
content_version = 1234567890
content_id = random_id(32)
fullpath = encode_fullpath(account, container, content_path,
content_version, content_id)
_, link = self.blob_client.chunk_link(
'http://' + self.rawx_id + '/' + target_chunk_id, None, fullpath)
chunk_id = link.split('/')[-1]
sleep(1) # ensure chunk event have been processed
return account, container, cid, content_path, content_version, \
content_id, chunk_id
def _chunk_path(self, chunk_id):
return self.rawx_path + '/' + chunk_id[:3] + '/' + chunk_id
def test_blob_indexer(self):
_, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \
self._put_chunk()
chunks = list(self.rdir_client.chunk_fetch(self.rawx_id))
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
self.assertEqual(expected_chunk_id, chunk_id)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
self.blob_indexer.index_pass()
self.assertEqual(1, self.blob_indexer.successes)
self.assertEqual(0, self.blob_indexer.errors)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
self.assertEqual(expected_chunk_id, chunk_id)
self._delete_chunk(expected_chunk_id)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(0, len(chunks))
def test_blob_indexer_with_old_chunk(self):
expected_account, expected_container, expected_cid, \
expected_content_path, expected_content_version, \
expected_content_id, expected_chunk_id = self._put_chunk()
chunks = list(self.rdir_client.chunk_fetch(self.rawx_id))
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
self.assertEqual(expected_chunk_id, chunk_id)
convert_to_old_chunk(self._chunk_path(chunk_id), expected_account,
expected_container, expected_content_path,
expected_content_version, expected_content_id)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
self.blob_indexer.index_pass()
self.assertEqual(1, self.blob_indexer.successes)
self.assertEqual(0, self.blob_indexer.errors)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
self.assertEqual(expected_chunk_id, chunk_id)
self._delete_chunk(expected_chunk_id)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(0, len(chunks))
def test_blob_indexer_with_linked_chunk(self):
_, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \
self._put_chunk()
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
self.assertEqual(expected_chunk_id, chunk_id)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
self.blob_indexer.index_pass()
self.assertEqual(1, self.blob_indexer.successes)
self.assertEqual(0, self.blob_indexer.errors)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
self.assertEqual(expected_chunk_id, chunk_id)
_, _, linked_cid, _, _, linked_content_id, linked_chunk_id = \
self._link_chunk(expected_chunk_id)
self.rdir_client.admin_clear(self.rawx_id, clear_all=True)
self.blob_indexer.index_pass()
self.assertEqual(2, self.blob_indexer.successes)
self.assertEqual(0, self.blob_indexer.errors)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(2, len(chunks))
self.assertNotEqual(chunks[0][2], chunks[1][2])
for chunk in chunks:
cid, content_id, chunk_id, _ = chunk
if chunk_id == expected_chunk_id:
self.assertEqual(expected_cid, cid)
self.assertEqual(expected_content_id, content_id)
else:
self.assertEqual(linked_cid, cid)
self.assertEqual(linked_content_id, content_id)
self.assertEqual(linked_chunk_id, chunk_id)
self._delete_chunk(expected_chunk_id)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(1, len(chunks))
cid, content_id, chunk_id, _ = chunks[0]
self.assertEqual(linked_cid, cid)
self.assertEqual(linked_content_id, content_id)
self.assertEqual(linked_chunk_id, chunk_id)
self._delete_chunk(linked_chunk_id)
chunks = self.rdir_client.chunk_fetch(self.rawx_id)
chunks = list(chunks)
self.assertEqual(0, len(chunks))