class TestBlobIndexer(BaseTestCase): def setUp(self): super(TestBlobIndexer, self).setUp() self.rdir_client = RdirClient(self.conf) self.blob_client = BlobClient(self.conf) _, self.rawx_path, rawx_addr, _ = \ self.get_service_url('rawx') services = self.conscience.all_services('rawx') self.rawx_id = None for rawx in services: if rawx_addr == rawx['addr']: self.rawx_id = rawx['tags'].get('tag.service_id', None) if self.rawx_id is None: self.rawx_id = rawx_addr conf = self.conf.copy() conf['volume'] = self.rawx_path self.blob_indexer = BlobIndexer(conf) # clear rawx/rdir chunk_files = paths_gen(self.rawx_path) for chunk_file in chunk_files: os.remove(chunk_file) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) def _put_chunk(self): account = random_str(16) container = random_str(16) cid = cid_from_name(account, container) content_path = random_str(16) content_version = 1234567890 content_id = random_id(32) fullpath = encode_fullpath(account, container, content_path, content_version, content_id) chunk_id = random_chunk_id() data = random_buffer(string.printable, 100) meta = { 'full_path': fullpath, 'container_id': cid, 'content_path': content_path, 'version': content_version, 'id': content_id, 'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3', 'policy': 'TESTPOLICY', 'chunk_hash': md5(data).hexdigest().upper(), 'oio_version': OIO_VERSION, 'chunk_pos': 0, 'metachunk_hash': md5().hexdigest(), 'metachunk_size': 1024 } self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id, meta, data) sleep(1) # ensure chunk event have been processed return account, container, cid, content_path, content_version, \ content_id, chunk_id def _delete_chunk(self, chunk_id): self.blob_client.chunk_delete('http://' + self.rawx_id + '/' + chunk_id) sleep(1) # ensure chunk event have been processed def _link_chunk(self, target_chunk_id): account = random_str(16) container = random_str(16) cid = cid_from_name(account, container) content_path = random_str(16) content_version = 1234567890 content_id = random_id(32) fullpath = encode_fullpath(account, container, content_path, content_version, content_id) _, link = self.blob_client.chunk_link( 'http://' + self.rawx_id + '/' + target_chunk_id, None, fullpath) chunk_id = link.split('/')[-1] sleep(1) # ensure chunk event have been processed return account, container, cid, content_path, content_version, \ content_id, chunk_id def _chunk_path(self, chunk_id): return self.rawx_path + '/' + chunk_id[:3] + '/' + chunk_id def test_blob_indexer(self): _, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \ self._put_chunk() chunks = list(self.rdir_client.chunk_fetch(self.rawx_id)) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks)) def test_blob_indexer_with_old_chunk(self): expected_account, expected_container, expected_cid, \ expected_content_path, expected_content_version, \ expected_content_id, expected_chunk_id = self._put_chunk() chunks = list(self.rdir_client.chunk_fetch(self.rawx_id)) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) convert_to_old_chunk(self._chunk_path(chunk_id), expected_account, expected_container, expected_content_path, expected_content_version, expected_content_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks)) def test_blob_indexer_with_linked_chunk(self): _, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \ self._put_chunk() chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) _, _, linked_cid, _, _, linked_content_id, linked_chunk_id = \ self._link_chunk(expected_chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(2, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(2, len(chunks)) self.assertNotEqual(chunks[0][2], chunks[1][2]) for chunk in chunks: cid, content_id, chunk_id, _ = chunk if chunk_id == expected_chunk_id: self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) else: self.assertEqual(linked_cid, cid) self.assertEqual(linked_content_id, content_id) self.assertEqual(linked_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(linked_cid, cid) self.assertEqual(linked_content_id, content_id) self.assertEqual(linked_chunk_id, chunk_id) self._delete_chunk(linked_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks))
class TestRdirClient(BaseTestCase): @classmethod def setUpClass(cls): super(TestRdirClient, cls).setUpClass() cls._service('@indexer', 'stop') @classmethod def tearDownClass(cls): super(TestRdirClient, cls).tearDownClass() cls._service('@indexer', 'start') def _push_chunks(self): max_mtime = 16 self.incident_date = random.randrange(2, max_mtime - 1) expected_entries = list() for _ in range(4): cid = random_id(64) for _ in range(random.randrange(2, 5)): content_id = random_id(32) for _ in range(random.randrange(2, 5)): chunk_id = random_id(63) mtime = random.randrange(0, max_mtime + 1) if mtime <= self.incident_date: chunk_id += '0' else: chunk_id += '1' self.rdir.chunk_push(self.rawx_id, cid, content_id, chunk_id, mtime=mtime) entry = (cid, content_id, chunk_id, {'mtime': mtime}) expected_entries.append(entry) self.expected_entries = sorted(expected_entries) def setUp(self): super(TestRdirClient, self).setUp() self.rawx_conf = random.choice(self.conf['services']['rawx']) self.rawx_id = self.rawx_conf.get('service_id', self.rawx_conf['addr']) self.rdir = RdirClient(self.conf) self.rdir.admin_clear(self.rawx_id, clear_all=True) self._push_chunks() self.rdir._direct_request = Mock(side_effect=self.rdir._direct_request) def _assert_chunk_fetch(self, expected_entries, entries, limit=0): self.assertListEqual(expected_entries, list(entries)) nb_requests = 1 if limit > 0 and len(expected_entries) > 0: nb_requests = int(math.ceil(len(expected_entries) / float(limit))) self.assertEqual(nb_requests, self.rdir._direct_request.call_count) self.rdir._direct_request.reset_mock() def test_chunk_fetch(self): entries = self.rdir.chunk_fetch(self.rawx_id) self._assert_chunk_fetch(self.expected_entries, entries) def test_chunk_fetch_with_limit(self): entries = self.rdir.chunk_fetch(self.rawx_id, limit=2) self._assert_chunk_fetch(self.expected_entries, entries, limit=2) def test_chunk_fetch_with_container_id(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] entries = self.rdir.chunk_fetch(self.rawx_id, container_id=cid) self._assert_chunk_fetch(expected_entries_cid, entries) def test_chunk_fetch_with_container_id_limit(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] entries = self.rdir.chunk_fetch(self.rawx_id, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_cid, entries, limit=2) def test_chunk_fetch_with_start_after(self): start_after_index = random.randrange(0, len(self.expected_entries)) start_after = '|'.join(self.expected_entries[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after) self._assert_chunk_fetch(self.expected_entries[start_after_index + 1:], entries) def test_chunk_fetch_with_start_after_limit(self): start_after_index = random.randrange(0, len(self.expected_entries)) start_after = '|'.join(self.expected_entries[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, limit=2) self._assert_chunk_fetch(self.expected_entries[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_start_after_container_id(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] start_after_index = random.randrange(0, len(expected_entries_cid)) start_after = '|'.join(expected_entries_cid[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, container_id=cid) self._assert_chunk_fetch(expected_entries_cid[start_after_index + 1:], entries) def test_chunk_fetch_with_start_after_container_id_limit(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] start_after_index = random.randrange(0, len(expected_entries_cid)) start_after = '|'.join(expected_entries_cid[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_cid[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_rebuild_no_incident(self): entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True) self._assert_chunk_fetch(list(), entries) def test_chunk_fetch_with_rebuild(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True) self._assert_chunk_fetch(expected_entries_rebuild, entries) def test_chunk_fetch_with_rebuild_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, limit=2) self._assert_chunk_fetch(expected_entries_rebuild, entries, limit=2) def test_chunk_fetch_with_rebuild_container_id(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid) self._assert_chunk_fetch(expected_entries_rebuild_cid, entries) def test_chunk_fetch_with_rebuild_start_after(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] if expected_entries_rebuild: start_after_index = random.randrange(0, len(expected_entries_rebuild)) start_after = '|'.join( expected_entries_rebuild[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, start_after=start_after) self._assert_chunk_fetch( expected_entries_rebuild[start_after_index + 1:], entries) def test_chunk_fetch_with_rebuild_contaier_id_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_rebuild_cid, entries, limit=2) def test_chunk_fetch_with_rebuild_container_id_start_after(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] if expected_entries_rebuild_cid: start_after_index = random.randrange( 0, len(expected_entries_rebuild_cid)) start_after = '|'.join( expected_entries_rebuild_cid[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, start_after=start_after) self._assert_chunk_fetch( expected_entries_rebuild_cid[start_after_index + 1:], entries) def test_chunk_fetch_with_rebuild_start_after_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] if expected_entries_rebuild: start_after_index = random.randrange(0, len(expected_entries_rebuild)) start_after = '|'.join( expected_entries_rebuild[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, start_after=start_after, limit=2) self._assert_chunk_fetch(expected_entries_rebuild[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_rebuild_container_id_start_after_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] if expected_entries_rebuild_cid: start_after_index = random.randrange( 0, len(expected_entries_rebuild_cid)) start_after = '|'.join( expected_entries_rebuild_cid[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, start_after=start_after, limit=2) self._assert_chunk_fetch( expected_entries_rebuild_cid[start_after_index + 1:], entries, limit=2) def _assert_chunk_status(self, expected_entries, status, max=0, incident=False): expected_status = dict() expected_status['chunk'] = {'total': len(expected_entries)} expected_status['container'] = dict() for entry in expected_entries: expected_status['container'][entry[0]]['total'] = \ expected_status['container'].setdefault( entry[0], dict()).get('total', 0) + 1 if incident: expected_entries_rebuild = [ entry for entry in expected_entries if entry[2][-1] == '0' ] expected_status['chunk']['to_rebuild'] = \ len(expected_entries_rebuild) for entry in expected_entries_rebuild: expected_status['container'][entry[0]]['to_rebuild'] = \ expected_status['container'][entry[0]].get( 'to_rebuild', 0) + 1 self.assertDictEqual(expected_status, status) nb_requests = 1 if max > 0 and len(expected_entries) > 0: nb_requests = int(math.ceil(len(expected_entries) / float(max))) self.assertEqual(nb_requests, self.rdir._direct_request.call_count) self.rdir._direct_request.reset_mock() def test_chunk_status(self): status = self.rdir.status(self.rawx_id) self._assert_chunk_status(self.expected_entries, status) def test_chunk_status_with_max(self): status = self.rdir.status(self.rawx_id, max=2) self._assert_chunk_status(self.expected_entries, status, max=2) def test_chunk_status_with_prefix(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid) self._assert_chunk_status(expected_entries_cid, status) def test_chunk_status_with_prefix_max(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid, status, max=2) def test_chunk_status_with_marker(self): marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status) def test_chunk_status_with_marker_max(self): marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, max=2) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, max=2) def test_chunk_status_marker_prefix(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status) def test_chunk_status_with_marker_prefix_max(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, max=2) def test_chunk_status_with_incident(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() status = self.rdir.status(self.rawx_id) self._assert_chunk_status(self.expected_entries, status, incident=True) def test_chunk_status_with_incident_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() status = self.rdir.status(self.rawx_id, max=2) self._assert_chunk_status(self.expected_entries, status, incident=True, max=2) def test_chunk_status_with_incident_prefix(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid) self._assert_chunk_status(expected_entries_cid, status, incident=True) def test_chunk_status_with_incident_prefix_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid, status, incident=True, max=2) def test_chunk_status_with_incident_marker(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, incident=True) def test_chunk_status_with_incident_marker_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, max=2) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, incident=True, max=2) def test_chunk_status_with_incident_marker_prefix(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, incident=True) def test_chunk_status_with_incident_marker_prefix_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, incident=True, max=2)