class TestRdirClient(BaseTestCase): def setUp(self): super(TestRdirClient, self).setUp() self.namespace = self.conf['namespace'] self.rdir_client = RdirClient({'namespace': self.namespace}) self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567") def tearDown(self): super(TestRdirClient, self).tearDown() del self.rdir_client def test_fetch_one_req_post(self): self.rdir_client._direct_request = Mock( side_effect=[ ( Mock(), [ ["container1|content1|chunk1", {'mtime': 10}], ["container2|content2|chunk2", {'mtime': 20}] ] ) ]) gen = self.rdir_client.chunk_fetch("volume", limit=2) self.assertEqual(gen.next(), ("container1", "content1", "chunk1", {'mtime': 10})) self.assertEqual(gen.next(), ("container2", "content2", "chunk2", {'mtime': 20})) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 2) def test_fetch_multi_req(self): self.rdir_client._direct_request = Mock( side_effect=[ ( Mock(), [ ["container1|content1|chunk1", {'mtime': 10}], ["container2|content2|chunk2", {'mtime': 20}] ] ), ( Mock(), [ ["container3|content3|chunk3", {'mtime': 30}] ] ) ]) gen = self.rdir_client.chunk_fetch("volume", limit=2) # print(gen.next()) # print(gen.next()) # print(gen.next()) self.assertEqual(gen.next(), ("container1", "content1", "chunk1", {'mtime': 10})) self.assertEqual(gen.next(), ("container2", "content2", "chunk2", {'mtime': 20})) self.assertEqual(gen.next(), ("container3", "content3", "chunk3", {'mtime': 30})) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 3)
class TestRdirClient(BaseTestCase): def setUp(self): super(TestRdirClient, self).setUp() self.namespace = self.conf["namespace"] self.rdir_client = RdirClient({"namespace": self.namespace}) self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567") self.container_id_1 = random_id(64) self.container_id_2 = random_id(64) self.container_id_3 = random_id(64) self.content_id_1 = random_id(32) self.content_id_2 = random_id(32) self.content_id_3 = random_id(32) self.chunk_id_1 = random_id(64) self.chunk_id_2 = random_id(64) self.chunk_id_3 = random_id(64) def tearDown(self): super(TestRdirClient, self).tearDown() del self.rdir_client def test_fetch_one_req_post(self): self.rdir_client._direct_request = Mock( side_effect=[ ( Mock(), [ ["%s|%s|%s" % (self.container_id_1, self.content_id_1, self.chunk_id_1), {"mtime": 10}], ["%s|%s|%s" % (self.container_id_2, self.content_id_2, self.chunk_id_2), {"mtime": 20}], ], ) ] ) gen = self.rdir_client.chunk_fetch("volume", limit=2) self.assertEqual(gen.next(), (self.container_id_1, self.content_id_1, self.chunk_id_1, {"mtime": 10})) self.assertEqual(gen.next(), (self.container_id_2, self.content_id_2, self.chunk_id_2, {"mtime": 20})) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 2) def test_fetch_multi_req(self): self.rdir_client._direct_request = Mock( side_effect=[ ( Mock(), [ ["%s|%s|%s" % (self.container_id_1, self.content_id_1, self.chunk_id_1), {"mtime": 10}], ["%s|%s|%s" % (self.container_id_2, self.content_id_2, self.chunk_id_2), {"mtime": 20}], ], ), (Mock(), [["%s|%s|%s" % (self.container_id_3, self.content_id_3, self.chunk_id_3), {"mtime": 30}]]), ] ) gen = self.rdir_client.chunk_fetch("volume", limit=2) self.assertEqual(gen.next(), (self.container_id_1, self.content_id_1, self.chunk_id_1, {"mtime": 10})) self.assertEqual(gen.next(), (self.container_id_2, self.content_id_2, self.chunk_id_2, {"mtime": 20})) self.assertEqual(gen.next(), (self.container_id_3, self.content_id_3, self.chunk_id_3, {"mtime": 30})) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 3)
class TestRdirClient(BaseTestCase): def setUp(self): super(TestRdirClient, self).setUp() self.namespace = self.conf['namespace'] self.rdir_client = RdirClient({'namespace': self.namespace}) self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567") def tearDown(self): super(TestRdirClient, self).tearDown() del self.rdir_client def test_fetch_one_req_post(self): self.rdir_client._direct_request = Mock( side_effect=[(Mock(), [["container1|content1|chunk1", { 'mtime': 10 }], ["container2|content2|chunk2", { 'mtime': 20 }]])]) gen = self.rdir_client.chunk_fetch("volume", limit=2) self.assertEqual(gen.next(), ("container1", "content1", "chunk1", { 'mtime': 10 })) self.assertEqual(gen.next(), ("container2", "content2", "chunk2", { 'mtime': 20 })) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 2) def test_fetch_multi_req(self): self.rdir_client._direct_request = Mock(side_effect=[( Mock(), [["container1|content1|chunk1", { 'mtime': 10 }], ["container2|content2|chunk2", { 'mtime': 20 }]]), (Mock(), [["container3|content3|chunk3", { 'mtime': 30 }]])]) gen = self.rdir_client.chunk_fetch("volume", limit=2) # print(gen.next()) # print(gen.next()) # print(gen.next()) self.assertEqual(gen.next(), ("container1", "content1", "chunk1", { 'mtime': 10 })) self.assertEqual(gen.next(), ("container2", "content2", "chunk2", { 'mtime': 20 })) self.assertEqual(gen.next(), ("container3", "content3", "chunk3", { 'mtime': 30 })) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 3)
def test_rebuild_chunk(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) rebuilder.chunk_rebuild(content.container_id, content.content_id, content.chunks[0].id) # check meta2 information _, res = self.container_client.content_show(acct=content.account, ref=content.container_name, content=content.content_id) new_chunk_info = None for c in res: if (c['url'] != content.chunks[0].url and c['url'] != content.chunks[1].url): new_chunk_info = c new_chunk_id = new_chunk_info['url'].split('/')[-1] self.assertEqual(new_chunk_info['hash'], content.chunks[0].hash) self.assertEqual(new_chunk_info['pos'], content.chunks[0].pos) self.assertEqual(new_chunk_info['size'], content.chunks[0].size) # check chunk information meta, stream = self.blob_client.chunk_get(new_chunk_info['url']) self.assertEqual(meta['content_size'], str(content.chunks[0].size)) self.assertEqual(meta['content_path'], content.content_name) self.assertEqual(meta['content_cid'], content.container_id) self.assertEqual(meta['content_id'], content.content_id) self.assertEqual(meta['chunk_id'], new_chunk_id) self.assertEqual(meta['chunk_pos'], content.chunks[0].pos) self.assertEqual(meta['content_version'], content.version) self.assertEqual(meta['chunk_hash'], content.chunks[0].hash) self.assertEqual(stream.next(), content.chunks[0].data) # check rtime flag in rdir rdir_client = RdirClient(self.gridconf) res = rdir_client.chunk_fetch(self.conf['rawx'][0]['addr']) key = (content.container_id, content.content_id, content.chunks[0].id) for i_container, i_content, i_chunk, i_value in res: if (i_container, i_content, i_chunk) == key: check_value = i_value self.assertIsNotNone(check_value.get('rtime'))
class TestIndexerCrawler(BaseTestCase): def setUp(self): super(TestIndexerCrawler, self).setUp() self.namespace = self.conf['namespace'] self.gridconf = {"namespace": self.namespace} self.rdir_client = RdirClient(self.gridconf) def tearDown(self): super(TestIndexerCrawler, self).tearDown() def _create_chunk(self, rawx_path): container_id = generate_id(64) content_id = generate_id(64) chunk_id = generate_id(64) chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:2]) if not os.path.isdir(chunk_dir): os.makedirs(chunk_dir) chunk_path = "%s/%s" % (chunk_dir, chunk_id) with open(chunk_path, "w") as f: f.write("toto") xattr.setxattr(chunk_path, 'user.grid.chunk.hash', 32 * '0') xattr.setxattr(chunk_path, 'user.grid.chunk.id', chunk_id) xattr.setxattr(chunk_path, 'user.grid.chunk.position', '0') xattr.setxattr(chunk_path, 'user.grid.chunk.size', '4') xattr.setxattr(chunk_path, 'user.grid.content.container', container_id) xattr.setxattr(chunk_path, 'user.grid.content.id', content_id) xattr.setxattr(chunk_path, 'user.grid.content.nbchunk', '1') xattr.setxattr(chunk_path, 'user.grid.content.path', 'toto') xattr.setxattr(chunk_path, 'user.grid.content.size', '4') xattr.setxattr(chunk_path, 'user.grid.content.mime_type', 'application/octet-stream') xattr.setxattr(chunk_path, 'user.grid.content.storage_policy', 'TESTPOLICY') xattr.setxattr(chunk_path, 'user.grid.content.chunk_method', 'bytes') xattr.setxattr(chunk_path, 'user.grid.content.version', '0') return chunk_path, container_id, content_id, chunk_id def _rdir_get(self, rawx_addr, container_id, content_id, chunk_id): data = self.rdir_client.chunk_fetch(rawx_addr) key = (container_id, content_id, chunk_id) for i_container, i_content, i_chunk, i_value in data: if (i_container, i_content, i_chunk) == key: return i_value return None def test_index_chunk(self): rawx_conf = self.conf['rawx'][0] # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( rawx_conf['path']) # index the chunk indexer = BlobIndexerWorker(self.gridconf, None, rawx_conf['path']) with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=1234)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(rawx_conf['addr'], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['content_nbchunks'], 1) self.assertEqual(check_value['chunk_hash'], 32 * '0') self.assertEqual(check_value['content_size'], 4) self.assertEqual(check_value['content_path'], 'toto') self.assertEqual(check_value['chunk_position'], '0') self.assertEqual(check_value['chunk_size'], 4) self.assertEqual(check_value['mtime'], 1234) self.assertEqual(check_value['content_version'], 0) # index a chunk already indexed with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=4567)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(rawx_conf['addr'], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 4567) def test_index_chunk_missing_xattr(self): rawx_conf = self.conf['rawx'][0] # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( rawx_conf['path']) # remove mandatory xattr xattr.removexattr(chunk_path, 'user.grid.chunk.hash') # try to index the chunk indexer = BlobIndexerWorker(self.gridconf, None, rawx_conf['path']) self.assertRaises(FaultyChunk, indexer.update_index, chunk_path)
class TestIndexerCrawler(BaseTestCase): def setUp(self): super(TestIndexerCrawler, self).setUp() self.namespace = self.conf['namespace'] self.rawx_conf = self.conf['services']['rawx'][0] self.conf = { "namespace": self.namespace, "volume": self.rawx_conf['path'] } self.rdir_client = RdirClient(self.conf) def tearDown(self): super(TestIndexerCrawler, self).tearDown() def _create_chunk(self, rawx_path, alias="toto"): container_id = random_id(64) content_id = random_id(32) chunk_id = random_id(64) chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3]) if not os.path.isdir(chunk_dir): os.makedirs(chunk_dir) chunk_path = "%s/%s" % (chunk_dir, chunk_id) with open(chunk_path, "w") as f: f.write("toto") xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * '0') xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], chunk_id) xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '0') xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '4') xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['container_id'], container_id) xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['content_id'], content_id) xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['content_path'], alias) xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['content_policy'], 'TESTPOLICY') xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'], 'plain/nb_copy=3') xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['content_version'], '0') xattr.setxattr(chunk_path, 'user.' + chunk_xattr_keys['oio_version'], OIO_VERSION) return chunk_path, container_id, content_id, chunk_id def _rdir_get(self, rawx_addr, container_id, content_id, chunk_id): data = self.rdir_client.chunk_fetch(rawx_addr) key = (container_id, content_id, chunk_id) for i_container, i_content, i_chunk, i_value in data: if (i_container, i_content, i_chunk) == key: return i_value return None def _test_index_chunk(self, alias="toto"): # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( self.rawx_conf['path'], alias) # index the chunk indexer = BlobIndexer(self.conf) with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=1234)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(self.rawx_conf['addr'], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 1234) # index a chunk already indexed with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=4567)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(self.rawx_conf['addr'], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 4567) def test_index_chunk(self): return self._test_index_chunk() def test_index_unicode_chunk(self): return self._test_index_chunk('a%%%s%d%xàç"\r\n{0}€ 1+1=2/\\$\t_') def test_index_chunk_missing_xattr(self): # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( self.rawx_conf['path']) # remove mandatory xattr xattr.removexattr(chunk_path, 'user.' + chunk_xattr_keys['container_id']) # try to index the chunk indexer = BlobIndexer(self.conf) self.assertRaises(FaultyChunk, indexer.update_index, chunk_path) os.remove(chunk_path)
class TestRdirClient(BaseTestCase): @classmethod def setUpClass(cls): super(TestRdirClient, cls).setUpClass() cls._service('@indexer', 'stop') @classmethod def tearDownClass(cls): super(TestRdirClient, cls).tearDownClass() cls._service('@indexer', 'start') def _push_chunks(self): max_mtime = 16 self.incident_date = random.randrange(2, max_mtime - 1) expected_entries = list() for _ in range(4): cid = random_id(64) for _ in range(random.randrange(2, 5)): content_id = random_id(32) for _ in range(random.randrange(2, 5)): chunk_id = random_id(63) mtime = random.randrange(0, max_mtime + 1) if mtime <= self.incident_date: chunk_id += '0' else: chunk_id += '1' self.rdir.chunk_push(self.rawx_id, cid, content_id, chunk_id, mtime=mtime) entry = (cid, content_id, chunk_id, {'mtime': mtime}) expected_entries.append(entry) self.expected_entries = sorted(expected_entries) def setUp(self): super(TestRdirClient, self).setUp() self.rawx_conf = random.choice(self.conf['services']['rawx']) self.rawx_id = self.rawx_conf.get('service_id', self.rawx_conf['addr']) self.rdir = RdirClient(self.conf) self.rdir.admin_clear(self.rawx_id, clear_all=True) self._push_chunks() self.rdir._direct_request = Mock(side_effect=self.rdir._direct_request) def _assert_chunk_fetch(self, expected_entries, entries, limit=0): self.assertListEqual(expected_entries, list(entries)) nb_requests = 1 if limit > 0 and len(expected_entries) > 0: nb_requests = int(math.ceil(len(expected_entries) / float(limit))) self.assertEqual(nb_requests, self.rdir._direct_request.call_count) self.rdir._direct_request.reset_mock() def test_chunk_fetch(self): entries = self.rdir.chunk_fetch(self.rawx_id) self._assert_chunk_fetch(self.expected_entries, entries) def test_chunk_fetch_with_limit(self): entries = self.rdir.chunk_fetch(self.rawx_id, limit=2) self._assert_chunk_fetch(self.expected_entries, entries, limit=2) def test_chunk_fetch_with_container_id(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] entries = self.rdir.chunk_fetch(self.rawx_id, container_id=cid) self._assert_chunk_fetch(expected_entries_cid, entries) def test_chunk_fetch_with_container_id_limit(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] entries = self.rdir.chunk_fetch(self.rawx_id, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_cid, entries, limit=2) def test_chunk_fetch_with_start_after(self): start_after_index = random.randrange(0, len(self.expected_entries)) start_after = '|'.join(self.expected_entries[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after) self._assert_chunk_fetch(self.expected_entries[start_after_index + 1:], entries) def test_chunk_fetch_with_start_after_limit(self): start_after_index = random.randrange(0, len(self.expected_entries)) start_after = '|'.join(self.expected_entries[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, limit=2) self._assert_chunk_fetch(self.expected_entries[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_start_after_container_id(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] start_after_index = random.randrange(0, len(expected_entries_cid)) start_after = '|'.join(expected_entries_cid[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, container_id=cid) self._assert_chunk_fetch(expected_entries_cid[start_after_index + 1:], entries) def test_chunk_fetch_with_start_after_container_id_limit(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] start_after_index = random.randrange(0, len(expected_entries_cid)) start_after = '|'.join(expected_entries_cid[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_cid[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_rebuild_no_incident(self): entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True) self._assert_chunk_fetch(list(), entries) def test_chunk_fetch_with_rebuild(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True) self._assert_chunk_fetch(expected_entries_rebuild, entries) def test_chunk_fetch_with_rebuild_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, limit=2) self._assert_chunk_fetch(expected_entries_rebuild, entries, limit=2) def test_chunk_fetch_with_rebuild_container_id(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid) self._assert_chunk_fetch(expected_entries_rebuild_cid, entries) def test_chunk_fetch_with_rebuild_start_after(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] if expected_entries_rebuild: start_after_index = random.randrange(0, len(expected_entries_rebuild)) start_after = '|'.join( expected_entries_rebuild[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, start_after=start_after) self._assert_chunk_fetch( expected_entries_rebuild[start_after_index + 1:], entries) def test_chunk_fetch_with_rebuild_contaier_id_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_rebuild_cid, entries, limit=2) def test_chunk_fetch_with_rebuild_container_id_start_after(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] if expected_entries_rebuild_cid: start_after_index = random.randrange( 0, len(expected_entries_rebuild_cid)) start_after = '|'.join( expected_entries_rebuild_cid[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, start_after=start_after) self._assert_chunk_fetch( expected_entries_rebuild_cid[start_after_index + 1:], entries) def test_chunk_fetch_with_rebuild_start_after_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] if expected_entries_rebuild: start_after_index = random.randrange(0, len(expected_entries_rebuild)) start_after = '|'.join( expected_entries_rebuild[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, start_after=start_after, limit=2) self._assert_chunk_fetch(expected_entries_rebuild[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_rebuild_container_id_start_after_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] if expected_entries_rebuild_cid: start_after_index = random.randrange( 0, len(expected_entries_rebuild_cid)) start_after = '|'.join( expected_entries_rebuild_cid[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, start_after=start_after, limit=2) self._assert_chunk_fetch( expected_entries_rebuild_cid[start_after_index + 1:], entries, limit=2) def _assert_chunk_status(self, expected_entries, status, max=0, incident=False): expected_status = dict() expected_status['chunk'] = {'total': len(expected_entries)} expected_status['container'] = dict() for entry in expected_entries: expected_status['container'][entry[0]]['total'] = \ expected_status['container'].setdefault( entry[0], dict()).get('total', 0) + 1 if incident: expected_entries_rebuild = [ entry for entry in expected_entries if entry[2][-1] == '0' ] expected_status['chunk']['to_rebuild'] = \ len(expected_entries_rebuild) for entry in expected_entries_rebuild: expected_status['container'][entry[0]]['to_rebuild'] = \ expected_status['container'][entry[0]].get( 'to_rebuild', 0) + 1 self.assertDictEqual(expected_status, status) nb_requests = 1 if max > 0 and len(expected_entries) > 0: nb_requests = int(math.ceil(len(expected_entries) / float(max))) self.assertEqual(nb_requests, self.rdir._direct_request.call_count) self.rdir._direct_request.reset_mock() def test_chunk_status(self): status = self.rdir.status(self.rawx_id) self._assert_chunk_status(self.expected_entries, status) def test_chunk_status_with_max(self): status = self.rdir.status(self.rawx_id, max=2) self._assert_chunk_status(self.expected_entries, status, max=2) def test_chunk_status_with_prefix(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid) self._assert_chunk_status(expected_entries_cid, status) def test_chunk_status_with_prefix_max(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid, status, max=2) def test_chunk_status_with_marker(self): marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status) def test_chunk_status_with_marker_max(self): marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, max=2) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, max=2) def test_chunk_status_marker_prefix(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status) def test_chunk_status_with_marker_prefix_max(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, max=2) def test_chunk_status_with_incident(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() status = self.rdir.status(self.rawx_id) self._assert_chunk_status(self.expected_entries, status, incident=True) def test_chunk_status_with_incident_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() status = self.rdir.status(self.rawx_id, max=2) self._assert_chunk_status(self.expected_entries, status, incident=True, max=2) def test_chunk_status_with_incident_prefix(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid) self._assert_chunk_status(expected_entries_cid, status, incident=True) def test_chunk_status_with_incident_prefix_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid, status, incident=True, max=2) def test_chunk_status_with_incident_marker(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, incident=True) def test_chunk_status_with_incident_marker_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, max=2) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, incident=True, max=2) def test_chunk_status_with_incident_marker_prefix(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, incident=True) def test_chunk_status_with_incident_marker_prefix_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, incident=True, max=2)
class BlobRebuilder(Tool): """ Rebuild chunks. """ DEFAULT_BEANSTALKD_WORKER_TUBE = 'oio-rebuild' DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE = 'oio-rebuild' DEFAULT_RDIR_FETCH_LIMIT = 100 DEFAULT_RDIR_TIMEOUT = 60.0 DEFAULT_ALLOW_FROZEN_CT = False DEFAULT_ALLOW_SAME_RAWX = True DEFAULT_TRY_CHUNK_DELETE = False DEFAULT_DRY_RUN = False def __init__(self, conf, input_file=None, service_id=None, **kwargs): super(BlobRebuilder, self).__init__(conf, **kwargs) # counters self.bytes_processed = 0 self.total_bytes_processed = 0 # input self.input_file = input_file self.rawx_id = service_id # rawx/rdir self.rdir_client = RdirClient(self.conf, logger=self.logger) self.rdir_fetch_limit = int_value(self.conf.get('rdir_fetch_limit'), self.DEFAULT_RDIR_FETCH_LIMIT) self.rdir_shuffle_chunks = true_value(conf.get('rdir_shuffle_chunks')) self.rdir_timeout = float_value(conf.get('rdir_timeout'), self.DEFAULT_RDIR_TIMEOUT) @staticmethod def items_from_task_event(task_event): namespace = task_event['url']['ns'] container_id = task_event['url']['id'] content_id = task_event['url']['content'] for chunk_id_or_pos in task_event['data']['missing_chunks']: yield namespace, container_id, content_id, str(chunk_id_or_pos) @staticmethod def task_event_from_item(item): namespace, container_id, content_id, chunk_id_or_pos = item return \ { 'when': time.time(), 'event': EventTypes.CONTENT_BROKEN, 'url': { 'ns': namespace, 'id': container_id, 'content': content_id }, 'data': { 'missing_chunks': [ chunk_id_or_pos ] } } @staticmethod def tasks_res_from_res_event(res_event): namespace = res_event['url']['ns'] container_id = res_event['url']['id'] content_id = res_event['url']['content'] for chunk_rebuilt in res_event['data']['chunks_rebuilt']: yield (namespace, container_id, content_id, str(chunk_rebuilt['chunk_id_or_pos'])), \ chunk_rebuilt['bytes_processed'], chunk_rebuilt['error'] @staticmethod def res_event_from_task_res(task_res): item, bytes_processed, error = task_res namespace, container_id, content_id, chunk_id_or_pos = item return \ { 'when': time.time(), 'event': EventTypes.CONTENT_REBUILT, 'url': { 'ns': namespace, 'id': container_id, 'content': content_id }, 'data': { 'chunks_rebuilt': [{ 'chunk_id_or_pos': chunk_id_or_pos, 'bytes_processed': bytes_processed, 'error': error }] } } @staticmethod def string_from_item(item): namespace, container_id, content_id, chunk_id_or_pos = item return '%s|%s|%s|%s' % (namespace, container_id, content_id, chunk_id_or_pos) def _fetch_items_from_input_file(self): with open(self.input_file, 'r') as ifile: for line in ifile: stripped = line.strip() if not stripped or stripped.startswith('#'): continue container_id, content_id, chunk_id_or_pos = \ stripped.split('|', 3)[:3] yield self.namespace, container_id, content_id, \ chunk_id_or_pos def _fetch_items_from_rawx_id(self): lost_chunks = self.rdir_client.chunk_fetch( self.rawx_id, limit=self.rdir_fetch_limit, rebuild=True, shuffle=self.rdir_shuffle_chunks, timeout=self.rdir_timeout) for container_id, content_id, chunk_id, _ in lost_chunks: yield self.namespace, container_id, content_id, chunk_id def _fetch_items(self): if self.input_file: return self._fetch_items_from_input_file() if self.rawx_id: return self._fetch_items_from_rawx_id() def _empty_generator(): return yield # pylint: disable=unreachable return _empty_generator() def update_counters(self, task_res): super(BlobRebuilder, self).update_counters(task_res) _, bytes_processed, _ = task_res if bytes_processed is not None: self.bytes_processed += bytes_processed def _update_total_counters(self): chunks_processed, total_chunks_processed, errors, total_errors = \ super(BlobRebuilder, self)._update_total_counters() bytes_processed = self.bytes_processed self.bytes_processed = 0 self.total_bytes_processed += bytes_processed return chunks_processed, total_chunks_processed, \ bytes_processed, self.total_bytes_processed, \ errors, total_errors def _get_report(self, status, end_time, counters): chunks_processed, total_chunks_processed, \ bytes_processed, total_bytes_processed, \ errors, total_errors = counters time_since_last_report = (end_time - self.last_report) or 0.00001 total_time = (end_time - self.start_time) or 0.00001 report = ( '%(status)s ' 'last_report=%(last_report)s %(time_since_last_report).2fs ' 'chunks=%(chunks)d %(chunks_rate).2f/s ' 'bytes=%(bytes)d %(bytes_rate).2fB/s ' 'errors=%(errors)d %(errors_rate).2f%% ' 'start_time=%(start_time)s %(total_time).2fs ' 'total_chunks=%(total_chunks)d %(total_chunks_rate).2f/s ' 'total_bytes=%(total_bytes)d %(total_bytes_rate).2fB/s ' 'total_errors=%(total_errors)d %(total_errors_rate).2f%%' % { 'status': status, 'last_report': datetime.fromtimestamp(int(self.last_report)).isoformat(), 'time_since_last_report': time_since_last_report, 'chunks': chunks_processed, 'chunks_rate': chunks_processed / time_since_last_report, 'bytes': bytes_processed, 'bytes_rate': bytes_processed / time_since_last_report, 'errors': errors, 'errors_rate': 100 * errors / float(chunks_processed or 1), 'start_time': datetime.fromtimestamp(int(self.start_time)).isoformat(), 'total_time': total_time, 'total_chunks': total_chunks_processed, 'total_chunks_rate': total_chunks_processed / total_time, 'total_bytes': total_bytes_processed, 'total_bytes_rate': total_bytes_processed / total_time, 'total_errors': total_errors, 'total_errors_rate': 100 * total_errors / float(total_chunks_processed or 1) }) if self.total_expected_items is not None: progress = 100 * total_chunks_processed / \ float(self.total_expected_items or 1) report += ' progress=%d/%d %.2f%%' % \ (total_chunks_processed, self.total_expected_items, progress) return report def create_worker(self, queue_workers, queue_reply): return BlobRebuilderWorker(self, queue_workers, queue_reply) def _load_total_expected_items(self): if self.rawx_id: try: info = self.rdir_client.status(self.rawx_id, read_timeout=self.rdir_timeout) self.total_expected_items = info.get('chunk', dict()).get( 'to_rebuild', None) except Exception as exc: self.logger.warn( 'Failed to fetch the total chunks to rebuild: %s', exc) def run(self): if self.rawx_id: self.rdir_client.admin_lock(self.rawx_id, "rebuilder on %s" % gethostname(), timeout=self.rdir_timeout) success = super(BlobRebuilder, self).run() if self.rawx_id: self.rdir_client.admin_unlock(self.rawx_id, timeout=self.rdir_timeout) return success
class BlobRebuilderWorker(object): def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value( conf.get('dry_run', False)) self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value( conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value( conf.get('rdir_fetch_limit'), 100) self.rdir_client = RdirClient(conf) self.content_factory = ContentFactory(conf) def rebuilder_pass_with_lock(self): self.rdir_client.admin_lock(self.volume, "rebuilder on %s" % gethostname()) try: self.rebuilder_pass() finally: self.rdir_client.admin_unlock(self.volume) def rebuilder_pass(self): start_time = report_time = time.time() total_errors = 0 rebuilder_time = 0 chunks = self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True) for container_id, content_id, chunk_id, data in chunks: loop_time = time.time() if self.dry_run: self.dryrun_chunk_rebuild(container_id, content_id, chunk_id) else: self.safe_chunk_rebuild(container_id, content_id, chunk_id) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(rebuilder_time).2f' '%(rebuilder_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'rebuilder_time': rebuilder_time, 'rebuilder_rate': rebuilder_time / (now - start_time) } ) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now rebuilder_time += (now - loop_time) elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(rebuilder_time).2f ' '%(rebuilder_rate).2f' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'rebuilder_time': rebuilder_time, 'rebuilder_rate': rebuilder_time / elapsed } ) def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info("[dryrun] Rebuilding " "container %s, content %s, chunk %s", container_id, content_id, chunk_id) self.passes += 1 def safe_chunk_rebuild(self, container_id, content_id, chunk_id): try: self.chunk_rebuild(container_id, content_id, chunk_id) except Exception as e: self.errors += 1 self.logger.error('ERROR while rebuilding chunk %s|%s|%s) : %s', container_id, content_id, chunk_id, e) self.passes += 1 def chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info('Rebuilding (container %s, content %s, chunk %s)', container_id, content_id, chunk_id) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') chunk = content.chunks.filter(id=chunk_id).one() if chunk is None: raise OrphanChunk("Chunk not found in content") chunk_size = chunk.size content.rebuild_chunk(chunk_id) self.rdir_client.chunk_push(self.volume, container_id, content_id, chunk_id, rtime=int(time.time())) self.bytes_processed += chunk_size self.total_bytes_processed += chunk_size
class TestIndexerCrawler(BaseTestCase): def setUp(self): super(TestIndexerCrawler, self).setUp() self.namespace = self.conf['namespace'] self.rawx_conf = self.conf['services']['rawx'][0] self.conf = {"namespace": self.namespace, "volume": self.rawx_conf['path']} self.rdir_client = RdirClient(self.conf) def tearDown(self): super(TestIndexerCrawler, self).tearDown() def _create_chunk(self, rawx_path, alias="toto", suffix=''): cname = random_str(8) container_id = cid_from_name(self.account, cname) content_id = random_id(32) chunk_id = random_id(64) chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3]) if not os.path.isdir(chunk_dir): os.makedirs(chunk_dir) chunk_path = "%s/%s%s" % (chunk_dir, chunk_id, suffix) with open(chunk_path, "w") as chunk_file: chunk_file.write("toto") # pylint: disable=no-member xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * '0') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], chunk_id) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '0') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '4') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_policy'], 'TESTPOLICY') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'], 'plain/nb_copy=3') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_version'], '1') # Old (oio-sds < 4.2) extended attributes xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['container_id'], container_id) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_id'], content_id) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_path'], alias) # New (oio-sds >= 4.2) extended attributes xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['oio_version'], OIO_VERSION) fullpath = encode_fullpath(self.account, cname, alias, 1, content_id) xattr.setxattr( chunk_path, 'user.%s%s' % (CHUNK_XATTR_CONTENT_FULLPATH_PREFIX, chunk_id), fullpath) return chunk_path, container_id, content_id, chunk_id def _get_service_id_or_addr(self): return self.rawx_conf.get('service_id', self.rawx_conf['addr']) def _rdir_get(self, rawx_addr, container_id, content_id, chunk_id): data = self.rdir_client.chunk_fetch(rawx_addr) key = (container_id, content_id, chunk_id) for i_container, i_content, i_chunk, i_value in data: if (i_container, i_content, i_chunk) == key: return i_value return None def _test_index_chunk(self, alias="toto"): # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( self.rawx_conf['path'], alias) # index the chunk indexer = BlobIndexer(self.conf) with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=1234)): indexer.update_index(chunk_path, chunk_id) # check rdir check_value = self._rdir_get(self._get_service_id_or_addr(), container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 1234) # index a chunk already indexed with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=4567)): indexer.update_index(chunk_path, chunk_id) # check rdir check_value = self._rdir_get(self._get_service_id_or_addr(), container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 4567) def test_index_chunk(self): return self._test_index_chunk() def test_index_unicode_chunk(self): return self._test_index_chunk('a%%%s%d%xàç"\r\n{0}€ 1+1=2/\\$\t_') def test_index_chunk_missing_xattr(self): # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( self.rawx_conf['path']) # remove mandatory xattr # pylint: disable=no-member xattr.removexattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_pos']) # try to index the chunk indexer = BlobIndexer(self.conf) self.assertRaises(FaultyChunk, indexer.update_index, chunk_path, chunk_id) os.remove(chunk_path) def test_index_chunk_with_wrong_paths(self): indexer = BlobIndexer(self.conf) # try to index the chunks indexer.index_pass() successes = indexer.successes errors = indexer.errors # create fake chunks chunk_path1, _, _, _ = self._create_chunk( self.rawx_conf['path'], suffix=".pending") chunk_path2, _, _, _ = self._create_chunk( self.rawx_conf['path'][:-1] + 'G') chunk_path3, _, _, _ = self._create_chunk( self.rawx_conf['path'] + '0') chunk_path4, _, _, _ = self._create_chunk( self.rawx_conf['path'][:-1]) # try to index the chunks indexer.index_pass() # All chunks we have created have invalid names, # the success count should not increase. # The indexer should print a warning, and ignore them. self.assertEqual(indexer.successes, successes) self.assertEqual(indexer.errors, errors) os.remove(chunk_path1) os.remove(chunk_path2) os.remove(chunk_path3) os.remove(chunk_path4)
class RawxDecommissionJob(XcuteRdirJob): JOB_TYPE = 'rawx-decommission' TASK_CLASS = RawxDecommissionTask DEFAULT_RAWX_TIMEOUT = 60.0 DEFAULT_MIN_CHUNK_SIZE = 0 DEFAULT_MAX_CHUNK_SIZE = 0 DEFAULT_USAGE_TARGET = 0 DEFAULT_USAGE_CHECK_INTERVAL = 60.0 @classmethod def sanitize_params(cls, job_params): sanitized_job_params, _ = super(RawxDecommissionJob, cls).sanitize_params(job_params) # specific configuration service_id = job_params.get('service_id') if not service_id: raise ValueError('Missing service ID') sanitized_job_params['service_id'] = service_id sanitized_job_params['rawx_timeout'] = float_value( job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT) sanitized_job_params['min_chunk_size'] = int_value( job_params.get('min_chunk_size'), cls.DEFAULT_MIN_CHUNK_SIZE) sanitized_job_params['max_chunk_size'] = int_value( job_params.get('max_chunk_size'), cls.DEFAULT_MAX_CHUNK_SIZE) excluded_rawx = job_params.get('excluded_rawx') if excluded_rawx: excluded_rawx = excluded_rawx.split(',') else: excluded_rawx = list() sanitized_job_params['excluded_rawx'] = excluded_rawx sanitized_job_params['usage_target'] = int_value( job_params.get('usage_target'), cls.DEFAULT_USAGE_TARGET) sanitized_job_params['usage_check_interval'] = float_value( job_params.get('usage_check_interval'), cls.DEFAULT_USAGE_CHECK_INTERVAL) return sanitized_job_params, 'rawx/%s' % service_id def __init__(self, conf, logger=None): super(RawxDecommissionJob, self).__init__(conf, logger=logger) self.rdir_client = RdirClient(self.conf, logger=self.logger) self.conscience_client = ConscienceClient(self.conf, logger=self.logger) def get_usage(self, service_id): services = self.conscience_client.all_services('rawx', full=True) for service in services: if service_id == service['tags'].get('tag.service_id', service['addr']): return 100 - service['tags']['stat.space'] raise ValueError('No rawx service this ID (%s)' % service_id) def get_tasks(self, job_params, marker=None): service_id = job_params['service_id'] usage_target = job_params['usage_target'] usage_check_interval = job_params['usage_check_interval'] if usage_target > 0: now = time.time() current_usage = self.get_usage(service_id) if current_usage <= usage_target: self.logger.info( 'current usage %.2f%%: target already reached (%.2f%%)', current_usage, usage_target) return last_usage_check = now chunk_infos = self.get_chunk_infos(job_params, marker=marker) for container_id, content_id, chunk_id, _ in chunk_infos: task_id = '|'.join((container_id, content_id, chunk_id)) yield task_id, { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } if usage_target <= 0: continue now = time.time() if now - last_usage_check < usage_check_interval: continue current_usage = self.get_usage(service_id) if current_usage > usage_target: last_usage_check = now continue self.logger.info('current usage %.2f%%: target reached (%.2f%%)', current_usage, usage_target) return def get_total_tasks(self, job_params, marker=None): service_id = job_params['service_id'] usage_target = job_params['usage_target'] current_usage = self.get_usage(service_id) if current_usage <= usage_target: return kept_chunks_ratio = 1 - (usage_target / float(current_usage)) chunk_infos = self.get_chunk_infos(job_params, marker=marker) i = 0 for i, (container_id, content_id, chunk_id, _) \ in enumerate(chunk_infos, 1): if i % 1000 == 0: yield ('|'.join((container_id, content_id, chunk_id)), int(math.ceil(1000 * kept_chunks_ratio))) remaining = int(math.ceil(i % 1000 * kept_chunks_ratio)) if remaining > 0: yield '|'.join((container_id, content_id, chunk_id)), remaining def get_chunk_infos(self, job_params, marker=None): service_id = job_params['service_id'] rdir_fetch_limit = job_params['rdir_fetch_limit'] rdir_timeout = job_params['rdir_timeout'] chunk_infos = self.rdir_client.chunk_fetch(service_id, timeout=rdir_timeout, limit=rdir_fetch_limit, start_after=marker) return chunk_infos
class BlobRebuilderWorker(object): def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value( conf.get('dry_run', False)) self.report_interval = int_value( conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value( conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value( conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value( conf.get('rdir_fetch_limit'), 100) self.blob_client = BlobClient() self.container_client = ContainerClient(conf) self.rdir_client = RdirClient(conf) def rebuilder_pass_with_lock(self): self.rdir_client.admin_lock(self.volume, "rebuilder on %s" % gethostname()) try: self.rebuilder_pass() finally: self.rdir_client.admin_unlock(self.volume) def rebuilder_pass(self): start_time = report_time = time.time() total_errors = 0 rebuilder_time = 0 chunks = self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True) for container_id, content_id, chunk_id, data in chunks: loop_time = time.time() if self.dry_run: self.dryrun_chunk_rebuild(container_id, content_id, chunk_id) else: self.safe_chunk_rebuild(container_id, content_id, chunk_id) self.chunks_run_time = ratelimit( self.chunks_run_time, self.max_chunks_per_second ) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( '%(start_time)s ' '%(passes)d ' '%(errors)d ' '%(c_rate).2f ' '%(b_rate).2f ' '%(total).2f ' '%(rebuilder_time).2f' '%(rebuilder_rate).2f' % { 'start_time': time.ctime(report_time), 'passes': self.passes, 'errors': self.errors, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'rebuilder_time': rebuilder_time, 'rebuilder_rate': rebuilder_time / (now - start_time) } ) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now rebuilder_time += (now - loop_time) elapsed = (time.time() - start_time) or 0.000001 self.logger.info( '%(elapsed).02f ' '%(errors)d ' '%(chunk_rate).2f ' '%(bytes_rate).2f ' '%(rebuilder_time).2f ' '%(rebuilder_rate).2f' % { 'elapsed': elapsed, 'errors': total_errors + self.errors, 'chunk_rate': self.total_chunks_processed / elapsed, 'bytes_rate': self.total_bytes_processed / elapsed, 'rebuilder_time': rebuilder_time, 'rebuilder_rate': rebuilder_time / elapsed } ) def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info("[dryrun] Rebuilding " "container %s, content %s, chunk %s" % (container_id, content_id, chunk_id)) self.passes += 1 def safe_chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info('Rebuilding (container %s, content %s, chunk %s)' % (container_id, content_id, chunk_id)) try: self.chunk_rebuild(container_id, content_id, chunk_id) except Exception as e: self.errors += 1 self.logger.error('ERROR while rebuilding chunk %s|%s|%s) : %s', container_id, content_id, chunk_id, e) self.passes += 1 def _meta2_get_chunks_at_pos(self, container_id, content_id, chunk_id): current_chunk_url = 'http://%s/%s' % (self.volume, chunk_id) try: data = self.container_client.content_show( cid=container_id, content=content_id) except exc.NotFound: raise exc.OrphanChunk('Content not found') current_chunk = None for c in data: if c['url'] == current_chunk_url: current_chunk = c break if not current_chunk: raise exc.OrphanChunk('Chunk not found in content') duplicate_chunks = [] for c in data: if c['pos'] == current_chunk['pos'] \ and c['url'] != current_chunk['url']: duplicate_chunks.append(c) if len(duplicate_chunks) == 0: raise exc.UnrecoverableContent('No copy of missing chunk') return current_chunk, duplicate_chunks def _meta2_get_spare_chunk(self, container_id, content_id, notin, broken): spare_data = {'notin': notin, 'broken': [broken], 'size': 0} try: spare_resp = self.container_client.content_spare( cid=container_id, content=content_id, data=spare_data) except ClientException as e: raise exc.SpareChunkException('No spare chunk (%s)' % e.message) return spare_resp['chunks'][0] def _meta2_replace_chunk(self, container_id, content_id, current_chunk, new_chunk): old = [{'type': 'chunk', 'id': current_chunk['url'], 'hash': current_chunk['hash'], 'size': current_chunk['size'], 'pos': current_chunk['pos'], 'content': content_id}] new = [{'type': 'chunk', 'id': new_chunk['id'], 'hash': current_chunk['hash'], 'size': current_chunk['size'], 'pos': current_chunk['pos'], 'content': content_id}] update_data = {'old': old, 'new': new} self.container_client.container_raw_update( cid=container_id, data=update_data) # TODO rain support def chunk_rebuild(self, container_id, content_id, chunk_id): current_chunk, duplicate_chunks = self._meta2_get_chunks_at_pos( container_id, content_id, chunk_id) spare_chunk = self._meta2_get_spare_chunk( container_id, content_id, duplicate_chunks, current_chunk) uploaded = False for src in duplicate_chunks: try: self.blob_client.chunk_copy(src['url'], spare_chunk['id']) self.logger.debug('copy chunk from %s to %s', src['url'], spare_chunk['id']) uploaded = True break except Exception as e: self.logger.debug('Failed to copy chunk from %s to %s: %s', src['url'], spare_chunk['id'], type(e)) if not uploaded: raise exc.UnrecoverableContent('No copy available ' 'of missing chunk') self._meta2_replace_chunk(container_id, content_id, current_chunk, spare_chunk) self.rdir_client.chunk_push(self.volume, container_id, content_id, chunk_id, rtime=int(time.time())) self.bytes_processed += current_chunk['size'] self.total_bytes_processed += current_chunk['size']
class RawxRebuildJob(XcuteRdirJob): JOB_TYPE = 'rawx-rebuild' TASK_CLASS = RawxRebuildTask DEFAULT_RAWX_TIMEOUT = 60.0 DEFAULT_DRY_RUN = False DEFAULT_ALLOW_SAME_RAWX = True DEFAULT_TRY_CHUNK_DELETE = False DEFAULT_ALLOW_FROZEN_CT = False DEFAULT_DECLARE_INCIDENT_DATE = False @classmethod def sanitize_params(cls, job_params): sanitized_job_params, _ = super(RawxRebuildJob, cls).sanitize_params(job_params) # specific configuration service_id = job_params.get('service_id') if not service_id: raise ValueError('Missing service ID') sanitized_job_params['service_id'] = service_id sanitized_job_params['rawx_timeout'] = float_value( job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT) sanitized_job_params['dry_run'] = boolean_value( job_params.get('dry_run'), cls.DEFAULT_DRY_RUN) sanitized_job_params['allow_same_rawx'] = boolean_value( job_params.get('allow_same_rawx'), cls.DEFAULT_ALLOW_SAME_RAWX) sanitized_job_params['try_chunk_delete'] = boolean_value( job_params.get('try_chunk_delete'), cls.DEFAULT_TRY_CHUNK_DELETE) sanitized_job_params['allow_frozen_container'] = boolean_value( job_params.get('allow_frozen_container'), cls.DEFAULT_ALLOW_FROZEN_CT) set_specific_incident_date = int_value( job_params.get('set_specific_incident_date'), None) if set_specific_incident_date is None: set_incident_date = boolean_value( job_params.get('set_incident_date'), cls.DEFAULT_DECLARE_INCIDENT_DATE) if set_incident_date: set_specific_incident_date = int(time.time()) else: set_incident_date = True sanitized_job_params['set_incident_date'] = set_incident_date sanitized_job_params['set_specific_incident_date'] = \ set_specific_incident_date return sanitized_job_params, 'rawx/%s' % service_id def __init__(self, conf, logger=None): super(RawxRebuildJob, self).__init__(conf, logger=logger) self.rdir_client = RdirClient(self.conf, logger=self.logger) def prepare(self, job_params): service_id = job_params['service_id'] rdir_timeout = job_params['rdir_timeout'] set_incident_date = job_params['set_incident_date'] set_specific_incident_date = job_params['set_specific_incident_date'] if not set_incident_date: return self.rdir_client.admin_incident_set(service_id, set_specific_incident_date, timeout=rdir_timeout) def get_tasks(self, job_params, marker=None): chunk_infos = self.get_chunk_infos(job_params, marker=marker) for container_id, content_id, chunk_id, _ in chunk_infos: task_id = '|'.join((container_id, content_id, chunk_id)) yield task_id, { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } def get_total_tasks(self, job_params, marker=None): chunk_infos = self.get_chunk_infos(job_params, marker=marker) i = 0 for i, (container_id, content_id, chunk_id, _) \ in enumerate(chunk_infos, 1): if i % 1000 == 0: yield '|'.join((container_id, content_id, chunk_id)), 1000 remaining = i % 1000 if remaining > 0: yield '|'.join((container_id, content_id, chunk_id)), remaining def get_chunk_infos(self, job_params, marker=None): service_id = job_params['service_id'] rdir_fetch_limit = job_params['rdir_fetch_limit'] rdir_timeout = job_params['rdir_timeout'] chunk_infos = self.rdir_client.chunk_fetch(service_id, rebuild=True, timeout=rdir_timeout, limit=rdir_fetch_limit, start_after=marker) return chunk_infos
class BlobRebuilder(Rebuilder): def __init__(self, conf, logger, volume, try_chunk_delete=False, beanstalkd_addr=None, **kwargs): super(BlobRebuilder, self).__init__(conf, logger, **kwargs) self.volume = volume self.rdir_client = RdirClient(conf, logger=self.logger) self.try_chunk_delete = try_chunk_delete self.beanstalkd_addr = beanstalkd_addr self.beanstalkd_tube = conf.get('beanstalkd_tube', DEFAULT_REBUILDER_TUBE) self.beanstalk = None self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100) def _fetch_chunks_from_event(self, job_id, data): env = json.loads(data) for chunk_pos in env['data']['missing_chunks']: yield [ env['url']['id'], env['url']['content'], str(chunk_pos), None ] def _connect_to_beanstalk(self): self.logger.debug('Connecting to %s', self.beanstalkd_addr) self.beanstalk = Beanstalk.from_url(self.beanstalkd_addr) self.logger.debug('Using tube %s', self.beanstalkd_tube) self.beanstalk.use(self.beanstalkd_tube) self.beanstalk.watch(self.beanstalkd_tube) def _handle_beanstalk_event(self, conn_error): try: job_id, data = self.beanstalk.reserve() if conn_error: self.logger.warn("beanstalk reconnected") except ConnectionError: if not conn_error: self.logger.warn("beanstalk connection error") raise try: for chunk in self._fetch_chunks_from_event(job_id, data): yield chunk self.beanstalk.delete(job_id) except Exception: self.logger.exception("handling event %s (bury)", job_id) self.beanstalk.bury(job_id) def _fetch_chunks_from_beanstalk(self): conn_error = False while 1: try: self._connect_to_beanstalk() for chunk in self._handle_beanstalk_event(conn_error): conn_error = False yield chunk except ConnectionError as exc: self.logger.warn('Disconnected: %s', exc) if 'Invalid URL' in str(exc): raise conn_error = True time.sleep(1.0) def _fetch_chunks_from_file(self): with open(self.input_file, 'r') as ifile: for line in ifile: stripped = line.strip() if stripped and not stripped.startswith('#'): yield stripped.split('|', 3)[:3] + [None] def _fetch_chunks(self): if self.input_file: return self._fetch_chunks_from_file() elif self.beanstalkd_addr: return self._fetch_chunks_from_beanstalk() else: return self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True) def rebuilder_pass_with_lock(self): self.rdir_client.admin_lock(self.volume, "rebuilder on %s" % gethostname()) try: self.rebuilder_pass() finally: self.rdir_client.admin_unlock(self.volume) def _create_worker(self, **kwargs): return BlobRebuilderWorker(self.conf, self.logger, self.volume, self.try_chunk_delete) def _fill_queue(self, queue, **kwargs): chunks = self._fetch_chunks() for chunk in chunks: queue.put(chunk) def _init_info(self, **kwargs): return 0 def _compute_info(self, worker, total_bytes_processed, **kwargs): total_bytes_processed += worker.total_bytes_processed return total_bytes_processed def _get_report(self, start_time, end_time, passes, errors, waiting_time, rebuilder_time, elapsed, total_chunks_processed, total_bytes_processed, **kwargs): return ('DONE %(volume)s ' 'started=%(start_time)s ' 'ended=%(end_time)s ' 'elapsed=%(elapsed).2f ' 'passes=%(passes)d ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' 'bytes=%(nb_bytes)d %(b_rate).2fB/s ' 'waiting_time=%(waiting_time).2f ' 'rebuilder_time=%(rebuilder_time).2f ' '(rebuilder: %(success_rate).2f%%)' % { 'volume': self.volume, 'start_time': datetime.fromtimestamp(int(start_time)).isoformat(), 'end_time': datetime.fromtimestamp(int(end_time)).isoformat(), 'elapsed': elapsed, 'passes': passes, 'errors': errors, 'nb_chunks': total_chunks_processed, 'nb_bytes': total_bytes_processed, 'c_rate': total_chunks_processed / elapsed, 'b_rate': total_bytes_processed / elapsed, 'rebuilder_time': rebuilder_time, 'waiting_time': waiting_time, 'success_rate': 100 * ((total_chunks_processed - errors) / float(total_chunks_processed or 1)) })
class BlobRebuilderWorker(object): def __init__(self, conf, logger, volume, input_file=None, try_chunk_delete=False, beanstalkd_addr=None): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value(conf.get('dry_run', False)) self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value(conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100) self.allow_same_rawx = true_value(conf.get('allow_same_rawx')) self.input_file = input_file self.rdir_client = RdirClient(conf, logger=self.logger) self.content_factory = ContentFactory(conf) self.try_chunk_delete = try_chunk_delete self.beanstalkd_addr = beanstalkd_addr self.beanstalkd_tube = conf.get('beanstalkd_tube', 'rebuild') self.beanstalk = None def _fetch_chunks_from_event(self, job_id, data): env = json.loads(data) for chunk_pos in env['data']['missing_chunks']: yield [ env['url']['id'], env['url']['content'], str(chunk_pos), None ] def _connect_to_beanstalk(self): self.beanstalk = Beanstalk.from_url(self.beanstalkd_addr) self.beanstalk.use(self.beanstalkd_tube) self.beanstalk.watch(self.beanstalkd_tube) def _handle_beanstalk_event(self, conn_error): try: job_id, data = self.beanstalk.reserve() if conn_error: self.logger.warn("beanstalk reconnected") except ConnectionError: if not conn_error: self.logger.warn("beanstalk connection error") raise try: for chunk in self._fetch_chunks_from_event(job_id, data): yield chunk self.beanstalk.delete(job_id) except Exception: self.logger.exception("handling event %s (bury)", job_id) self.beanstalk.bury(job_id) def _fetch_chunks_from_beanstalk(self): conn_error = False while 1: try: self._connect_to_beanstalk() for chunk in self._handle_beanstalk_event(conn_error): conn_error = False yield chunk except ConnectionError: conn_error = True time.sleep(1.0) def _fetch_chunks_from_file(self): with open(self.input_file, 'r') as ifile: for line in ifile: stripped = line.strip() if stripped and not stripped.startswith('#'): yield stripped.split('|', 3)[:3] + [None] def _fetch_chunks(self): if self.input_file: return self._fetch_chunks_from_file() elif self.beanstalkd_addr: return self._fetch_chunks_from_beanstalk() else: return self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True) def rebuilder_pass_with_lock(self): self.rdir_client.admin_lock(self.volume, "rebuilder on %s" % gethostname()) try: self.rebuilder_pass() finally: self.rdir_client.admin_unlock(self.volume) def rebuilder_pass(self): start_time = report_time = time.time() rebuilder_time = 0 chunks = self._fetch_chunks() for cid, content_id, chunk_id_or_pos, _ in chunks: loop_time = time.time() if self.dry_run: self.dryrun_chunk_rebuild(cid, content_id, chunk_id_or_pos) else: self.safe_chunk_rebuild(cid, content_id, chunk_id_or_pos) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( 'RUN %(volume)s ' 'started=%(start_time)s ' 'passes=%(passes)d ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' 'bytes=%(nb_bytes)d %(b_rate).2fB/s ' 'elapsed=%(total).2f ' '(rebuilder: %(success_rate).2f%%)' % { 'volume': self.volume, 'start_time': datetime.fromtimestamp(int(report_time)).isoformat(), 'passes': self.passes, 'errors': self.errors, 'nb_chunks': self.total_chunks_processed, 'nb_bytes': self.total_bytes_processed, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'rebuilder_time': rebuilder_time, 'success_rate': 100 * ((self.total_chunks_processed - self.errors) / float(self.total_chunks_processed)) }) report_time = now self.passes = 0 self.bytes_processed = 0 self.last_reported = now rebuilder_time += (now - loop_time) end_time = time.time() elapsed = (end_time - start_time) or 0.000001 self.logger.info( 'DONE %(volume)s ' 'started=%(start_time)s ' 'ended=%(end_time)s ' 'passes=%(passes)d ' 'elapsed=%(elapsed).02f ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' 'bytes=%(nb_bytes)d %(b_rate).2fB/s ' 'elapsed=%(rebuilder_time).2f ' '(rebuilder: %(success_rate).2f%%)' % { 'volume': self.volume, 'start_time': datetime.fromtimestamp(int(start_time)).isoformat(), 'end_time': datetime.fromtimestamp(int(end_time)).isoformat(), 'passes': self.passes, 'elapsed': elapsed, 'errors': self.errors, 'nb_chunks': self.total_chunks_processed, 'nb_bytes': self.total_bytes_processed, 'c_rate': self.total_chunks_processed / elapsed, 'b_rate': self.total_bytes_processed / elapsed, 'rebuilder_time': rebuilder_time, 'success_rate': 100 * ((self.total_chunks_processed - self.errors) / float(self.total_chunks_processed or 1)) }) def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id_or_pos): self.logger.info( "[dryrun] Rebuilding " "container %s, content %s, chunk %s", container_id, content_id, chunk_id_or_pos) self.passes += 1 def safe_chunk_rebuild(self, container_id, content_id, chunk_id_or_pos): try: self.chunk_rebuild(container_id, content_id, chunk_id_or_pos) except Exception as e: self.errors += 1 self.logger.error('ERROR while rebuilding chunk %s|%s|%s: %s', container_id, content_id, chunk_id_or_pos, e) self.passes += 1 def chunk_rebuild(self, container_id, content_id, chunk_id_or_pos): self.logger.info('Rebuilding (container %s, content %s, chunk %s)', container_id, content_id, chunk_id_or_pos) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise OrphanChunk('Content not found: possible orphan chunk') chunk_size = 0 chunk_pos = None if len(chunk_id_or_pos) < 32: chunk_pos = chunk_id_or_pos chunk_id = None metapos = int(chunk_pos.split('.', 1)[0]) chunk_size = content.chunks.filter(metapos=metapos).all()[0].size else: if '/' in chunk_id_or_pos: chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1] else: chunk_id = chunk_id_or_pos chunk = content.chunks.filter(id=chunk_id).one() if chunk is None: raise OrphanChunk(("Chunk not found in content:" "possible orphan chunk")) elif self.volume and chunk.host != self.volume: raise ValueError("Chunk does not belong to this volume") chunk_size = chunk.size content.rebuild_chunk(chunk_id, allow_same_rawx=self.allow_same_rawx, chunk_pos=chunk_pos) if self.try_chunk_delete: try: content.blob_client.chunk_delete(chunk.url) self.logger.info("Chunk %s deleted", chunk.url) except NotFound as exc: self.logger.debug("Chunk %s: %s", chunk.url, exc) # This call does not raise exception if chunk is not referenced if chunk_id is not None: self.rdir_client.chunk_delete(chunk.host, container_id, content_id, chunk_id) self.bytes_processed += chunk_size self.total_bytes_processed += chunk_size
class BlobRebuilder(Rebuilder): def __init__(self, conf, logger, volume, try_chunk_delete=False, beanstalkd_addr=None, **kwargs): super(BlobRebuilder, self).__init__(conf, logger, volume, **kwargs) # rdir self.rdir_client = RdirClient(conf, logger=self.logger) self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100) # rawx self.try_chunk_delete = try_chunk_delete # beanstalk if beanstalkd_addr: self.beanstalkd_listener = BeanstalkdListener( beanstalkd_addr, conf.get('beanstalkd_tube', DEFAULT_REBUILDER_TUBE), self.logger, **kwargs) else: self.beanstalkd_listener = None # counters self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_expected_chunks = None # distributed self.distributed = False def _create_worker(self, **kwargs): return BlobRebuilderWorker(self, try_chunk_delete=self.try_chunk_delete, **kwargs) def _fill_queue(self, queue, **kwargs): chunks = self._fetch_chunks(**kwargs) for chunk in chunks: queue.put(chunk) def _item_to_string(self, chunk, **kwargs): cid, content_id, chunk_id_or_pos, _ = chunk return 'chunk %s|%s|%s' % (cid, content_id, chunk_id_or_pos) def _get_report(self, status, end_time, counters, **kwargs): chunks_processed, bytes_processed, errors, total_chunks_processed, \ total_bytes_processed, total_errors = counters time_since_last_report = (end_time - self.last_report) or 0.00001 total_time = (end_time - self.start_time) or 0.00001 report = ( '%(status)s volume=%(volume)s ' 'last_report=%(last_report)s %(time_since_last_report).2fs ' 'chunks=%(chunks)d %(chunks_rate).2f/s ' 'bytes=%(bytes)d %(bytes_rate).2fB/s ' 'errors=%(errors)d %(errors_rate).2f%% ' 'start_time=%(start_time)s %(total_time).2fs ' 'total_chunks=%(total_chunks)d %(total_chunks_rate).2f/s ' 'total_bytes=%(total_bytes)d %(total_bytes_rate).2fB/s ' 'total_errors=%(total_errors)d %(total_errors_rate).2f%%' % { 'status': status, 'volume': self.volume, 'last_report': datetime.fromtimestamp(int(self.last_report)).isoformat(), 'time_since_last_report': time_since_last_report, 'chunks': chunks_processed, 'chunks_rate': chunks_processed / time_since_last_report, 'bytes': bytes_processed, 'bytes_rate': bytes_processed / time_since_last_report, 'errors': errors, 'errors_rate': 100 * errors / float(chunks_processed or 1), 'start_time': datetime.fromtimestamp(int(self.start_time)).isoformat(), 'total_time': total_time, 'total_chunks': total_chunks_processed, 'total_chunks_rate': total_chunks_processed / total_time, 'total_bytes': total_bytes_processed, 'total_bytes_rate': total_bytes_processed / total_time, 'total_errors': total_errors, 'total_errors_rate': 100 * total_errors / float(total_chunks_processed or 1) }) if self.total_expected_chunks is not None: progress = 100 * total_chunks_processed / \ float(self.total_expected_chunks or 1) report += ' progress=%d/%d %.2f%%' % \ (total_chunks_processed, self.total_expected_chunks, progress) return report def _update_processed_without_lock(self, bytes_processed, error=None, **kwargs): super(BlobRebuilder, self)._update_processed_without_lock(None, error=error, **kwargs) if bytes_processed is not None: self.bytes_processed += bytes_processed def _update_totals_without_lock(self, **kwargs): chunks_processed, errors, total_chunks_processed, total_errors = \ super(BlobRebuilder, self)._update_totals_without_lock(**kwargs) bytes_processed = self.bytes_processed self.bytes_processed = 0 self.total_bytes_processed += bytes_processed return chunks_processed, bytes_processed, errors, \ total_chunks_processed, self.total_bytes_processed, total_errors def _rebuilder_pass(self, **kwargs): return super(BlobRebuilder, self).rebuilder_pass(**kwargs) def rebuilder_pass(self, **kwargs): success = False if self.volume: self.rdir_client.admin_lock(self.volume, "rebuilder on %s" % gethostname()) info = self.rdir_client.status(self.volume) self.total_expected_chunks = info.get('chunk', dict()).get( 'to_rebuild', None) try: success = self._rebuilder_pass(**kwargs) finally: if self.volume: self.rdir_client.admin_unlock(self.volume) return success def _event_from_broken_chunk(self, chunk, reply, **kwargs): cid, content_id, chunk_id_or_pos, _ = chunk event = {} event['when'] = time.time() event['event'] = 'storage.content.broken' event['data'] = {'missing_chunks': [chunk_id_or_pos]} event['url'] = {'ns': self.namespace, 'id': cid, 'content': content_id} event['reply'] = reply return json.dumps(event) def _chunks_from_event(self, job_id, data, **kwargs): decoded = json.loads(data) container_id = decoded['url']['id'] content_id = decoded['url']['content'] more = None reply = decoded.get('reply', None) if reply: more = {'reply': reply} for chunk_id_or_pos in decoded['data']['missing_chunks']: yield [container_id, content_id, str(chunk_id_or_pos), more] def _fetch_events_from_beanstalk(self, **kwargs): return self.beanstalkd_listener.fetch_events(self._chunks_from_event, **kwargs) def _fetch_chunks_from_file(self, **kwargs): with open(self.input_file, 'r') as ifile: for line in ifile: stripped = line.strip() if stripped and not stripped.startswith('#'): yield stripped.split('|', 3)[:3] + [None] def _fetch_chunks(self, **kwargs): if self.input_file: return self._fetch_chunks_from_file(**kwargs) if self.beanstalkd_listener and not self.distributed: return self._fetch_events_from_beanstalk(**kwargs) if self.volume: return self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True, **kwargs) raise ConfigurationException('No source to fetch chunks from')
class Checker(object): def __init__(self, namespace, concurrency=50, error_file=None, rebuild_file=None, check_xattr=True, limit_listings=0, request_attempts=1, logger=None, verbose=False, check_hash=False, min_time_in_error=0.0, required_confirmations=0, beanstalkd_addr=None, beanstalkd_tube=BlobRebuilder.DEFAULT_BEANSTALKD_WORKER_TUBE, cache_size=2**24, **_kwargs): self.pool = GreenPool(concurrency) self.error_file = error_file self.error_sender = None self.check_xattr = bool(check_xattr) self.check_hash = bool(check_hash) self.logger = logger or get_logger( {'namespace': namespace}, name='integrity', verbose=verbose) # Optimisation for when we are only checking one object # or one container. # 0 -> do not limit # 1 -> limit account listings (list of containers) # 2 -> limit container listings (list of objects) self.limit_listings = limit_listings if self.error_file: outfile = open(self.error_file, 'a') self.error_writer = csv.writer(outfile, delimiter=' ') self.rebuild_file = rebuild_file if self.rebuild_file: self.fd = open(self.rebuild_file, 'a') self.rebuild_writer = csv.writer(self.fd, delimiter='|') if beanstalkd_addr: self.error_sender = BeanstalkdSender(beanstalkd_addr, beanstalkd_tube, self.logger) self.api = ObjectStorageApi(namespace, logger=self.logger, max_retries=request_attempts - 1, request_attempts=request_attempts) self.rdir_client = RdirClient({"namespace": namespace}, logger=self.logger) self.accounts_checked = 0 self.containers_checked = 0 self.objects_checked = 0 self.chunks_checked = 0 self.account_not_found = 0 self.container_not_found = 0 self.object_not_found = 0 self.chunk_not_found = 0 self.account_exceptions = 0 self.container_exceptions = 0 self.object_exceptions = 0 self.chunk_exceptions = 0 self.list_cache = CacheDict(cache_size) self.running_tasks = {} self.running_lock = Semaphore(1) self.result_queue = LightQueue(concurrency) self.running = True self.run_time = 0 # Set of targets which must be checked again, to confirm # or deny the issues reported by previous passes. self.delayed_targets = dict() # Minimum time in error and number of confirmations of the error # before triggering a reconstruction action. self.min_time_in_error = min_time_in_error self.required_confirmations = required_confirmations def reset_stats(self): self.accounts_checked = 0 self.containers_checked = 0 self.objects_checked = 0 self.chunks_checked = 0 self.account_not_found = 0 self.container_not_found = 0 self.object_not_found = 0 self.chunk_not_found = 0 self.account_exceptions = 0 self.container_exceptions = 0 self.object_exceptions = 0 self.chunk_exceptions = 0 def _spawn(self, func, target, *args, **kwargs): """ Spawn a task on the internal GreenPool. Discards the task if the pool is no more running. """ if self.running: return self.pool.spawn(func, target, *args, **kwargs) self.logger.info("Discarding %s", target) return None def _spawn_n(self, func, target, *args, **kwargs): """ Spawn a task on the internal GreenPool, do not wait for the result. Discards the task if the pool is no more running. """ if self.running: return self.pool.spawn_n(func, target, *args, **kwargs) self.logger.info("Discarding %s", target) return None def complete_target_from_chunk_metadata(self, target, xattr_meta): """ Complete a Target object from metadata found in chunk's extended attributes. In case the "fullpath" is not available, try to read legacy metadata, and maybe ask meta1 to resolve the CID into account and container names. """ # pylint: disable=unbalanced-tuple-unpacking try: acct, ct, path, vers, content_id = \ decode_fullpath(xattr_meta['full_path']) target.account = acct target.container = ct target.obj = path target.content_id = content_id target.version = vers except KeyError: # No fullpath header, try legacy headers if 'content_path' in xattr_meta: target.obj = xattr_meta['content_path'] if 'content_id' in xattr_meta: target.content_id = xattr_meta['content_id'] if 'content_version' in xattr_meta: target.version = xattr_meta['content_version'] cid = xattr_meta.get('container_id') if cid: try: md = self.api.directory.show(cid=cid) acct = md.get('account') ct = md.get('name') if acct: target.account = acct if ct: target.container = ct except Exception as err: self.logger.warn( "Failed to resolve CID %s into account " "and container names: %s", cid, err) def recover_and_complete_object_meta(self, target, chunk): _, rawx_service, chunk_id = chunk.rsplit('/', 2) # 1. Fetch chunk list from rdir (could be cached). # Unfortunately we cannot seek for a chunk ID. entries = [ x for x in self.rdir_client.chunk_fetch(rawx_service, limit=-1) if x[2] == chunk_id ] if not entries: self.logger.warn('Chunk %s not found in rdir' % chunk_id) return elif len(entries) > 1: self.logger.info('Chunk %s appears in %d objects', chunk_id, len(entries)) # 2. Find content and container IDs target.cid, target.content_id = entries[0][0:2] meta = self.api.object_get_properties(None, None, None, cid=target.cid, content=target.content_id) target.obj = meta['name'] target.version = meta['version'] target.account, target.container = self.api.resolve_cid(target.cid) def send_result(self, target, errors=None, irreparable=False): """ Put an item in the result queue. """ # TODO(FVE): send to an external queue. target.append_result(ItemResult(errors, irreparable)) self.result_queue.put(target) def send_chunk_job(self, target, irreparable=False): """ Send a "content broken" event, to trigger the reconstruction of the chunk. """ item = (self.api.namespace, target.cid, target.content_id, target.chunk) ev_dict = BlobRebuilder.task_event_from_item(item) if irreparable: ev_dict['data']['irreparable'] = irreparable job = json.dumps(ev_dict) self.error_sender.send_job(job) self.error_sender.job_done() # Don't expect any response def write_error(self, target, irreparable=False): if not self.error_file: return error = list() if irreparable: error.append(IRREPARABLE_PREFIX) error.append(target.account) if target.container: error.append(target.container) if target.obj: error.append(target.obj) if target.chunk: error.append(target.chunk) self.error_writer.writerow(error) def write_rebuilder_input(self, target, irreparable=False): error = list() if irreparable: error.append(IRREPARABLE_PREFIX) error.append(target.cid) # FIXME(FVE): ensure we always resolve content_id, # or pass object version along with object name. error.append(target.content_id or target.obj) error.append(target.chunk) self.rebuild_writer.writerow(error) def write_chunk_error(self, target, chunk=None, irreparable=False): if chunk is not None: target = target.copy() target.chunk = chunk self.write_error(target, irreparable=irreparable) if self.rebuild_file: self.write_rebuilder_input(target, irreparable=irreparable) if self.error_sender: self.send_chunk_job(target, irreparable=irreparable) def _check_chunk_xattr(self, target, obj_meta, xattr_meta): """ Check coherency of chunk extended attributes with object metadata. :returns: a list of errors """ errors = list() # Composed position -> erasure coding attr_prefix = 'meta' if '.' in obj_meta['pos'] else '' attr_key = attr_prefix + 'chunk_size' if str(obj_meta['size']) != xattr_meta.get(attr_key): errors.append( "'%s' xattr (%s) differs from size in meta2 (%s)" % (attr_key, xattr_meta.get(attr_key), obj_meta['size'])) attr_key = attr_prefix + 'chunk_hash' if obj_meta['hash'] != xattr_meta.get(attr_key): errors.append( "'%s' xattr (%s) differs from hash in meta2 (%s)" % (attr_key, xattr_meta.get(attr_key), obj_meta['hash'])) return errors def _check_chunk(self, target): """ Execute various checks on a chunk: - does it appear in object's chunk list? - is it reachable? - are its extended attributes coherent? :returns: the list of errors encountered, and the chunk's owner object metadata. """ chunk = target.chunk errors = list() obj_meta = None xattr_meta = None cached = self._get_cached_or_lock(chunk) if cached is not None: return cached + (True, ) self.logger.debug('Checking chunk "%s"', target) try: xattr_meta = self.api.blob_client.chunk_head( chunk, xattr=self.check_xattr, check_hash=self.check_hash) except exc.NotFound as err: self.chunk_not_found += 1 errors.append('Not found: %s' % (err, )) except exc.FaultyChunk as err: self.chunk_exceptions += 1 errors.append('Faulty: %r' % (err, )) except Exception as err: self.chunk_exceptions += 1 errors.append('Check failed: %s' % (err, )) if not target.obj: if xattr_meta: self.complete_target_from_chunk_metadata(target, xattr_meta) else: self.recover_and_complete_object_meta(target, chunk) if target.obj: obj_listing, obj_meta = self.check_obj(target.copy_object()) if chunk not in obj_listing: errors.append('Missing from object listing') db_meta = dict() else: db_meta = obj_listing[chunk] if db_meta and xattr_meta and self.check_xattr: errors.extend( self._check_chunk_xattr(target, db_meta, xattr_meta)) self.list_cache[chunk] = errors, obj_meta self._unlock(chunk) # Do not send errors directly, let the caller do it. # Indeed, it may want to check if the chunks can be repaired or not. self.chunks_checked += 1 return errors, obj_meta, False def check_chunk(self, target): errors, _obj_meta, from_cache = self._check_chunk(target) # If the result comes from the cache, we already reported it. if not from_cache: self.send_result(target, errors, target.irreparable) return errors def _check_metachunk(self, target, stg_met, pos, chunks, recurse=0): """ Check that a metachunk has the right number of chunks. :returns: the list of errors """ required = stg_met.expected_chunks errors = list() chunk_results = list() if len(chunks) < required: missing_chunks = required - len(chunks) if stg_met.ec: subs = {x['num'] for x in chunks} for sub in range(required): if sub not in subs: chkt = target.copy() chkt.chunk = '%d.%d' % (pos, sub) err = "Missing chunk at position %s" % chkt.chunk chunk_results.append((chkt, [err], False)) errors.append(err) else: for _ in range(missing_chunks): chkt = target.copy() chkt.chunk = '%d.%d' % (pos, sub) err = "Missing chunk at position %d" % pos chunk_results.append((chkt, [err], False)) errors.append(err) if recurse > 0: for chunk in chunks: tcopy = target.copy() tcopy.chunk = chunk['url'] chunk_errors, _, from_cache = self._check_chunk(tcopy) chunk_results.append((tcopy, chunk_errors, from_cache)) if chunk_errors: errors.append("Unusable chunk %s at position %s" % (chunk['url'], chunk['pos'])) irreparable = required - len(errors) < stg_met.min_chunks_to_read if irreparable: errors.append( "Unavailable metachunk at position %s " "(%d/%d chunks available, %d/%d required)" % (pos, required - len(errors), stg_met.expected_chunks, stg_met.min_chunks_to_read, stg_met.expected_chunks)) for tgt, errs, from_cache in chunk_results: # If the result comes from the cache, we already reported it. if not from_cache: self.send_result(tgt, errs, irreparable) # Since the "metachunk" is not an official item type, # this method does not report errors itself. Errors will # be reported as object errors. return errors def _check_obj_policy(self, target, obj_meta, chunks, recurse=0): """ Check that the list of chunks of an object matches the object's storage policy. :returns: the list of errors encountered """ stg_met = STORAGE_METHODS.load(obj_meta['chunk_method']) chunks_by_pos = _sort_chunks(chunks, stg_met.ec) tasks = list() for pos, pchunks in iteritems(chunks_by_pos): tasks.append((pos, self._spawn(self._check_metachunk, target.copy(), stg_met, pos, pchunks, recurse=recurse))) errors = list() for pos, task in tasks: if not task and not self.running: errors.append("Pos %d skipped: checker is exiting" % pos) continue try: errors.extend(task.wait()) except Exception as err: errors.append("Check failed: pos %d: %s" % (pos, err)) return errors def check_obj_versions(self, target, versions, recurse=0): """ Run checks of all versions of the targeted object in parallel. """ tasks = list() for ov in versions: tcopy = target.copy_object() tcopy.content_id = ov['id'] tcopy.version = str(ov['version']) tasks.append((tcopy.version, self._spawn(self.check_obj, tcopy, recurse=recurse))) errors = list() for version, task in tasks: if not task and not self.running: errors.append("Version %s skipped: checker is exiting" % version) continue try: task.wait() except Exception as err: errors.append("Check failed: version %s: %s" % (version, err)) if errors: # Send a result with the target without version to tell # we were not able to check all versions of the object. self.send_result(target, errors) def _load_obj_meta(self, target, errors): """ Load object metadata and chunks. :param target: which object to check. :param errors: list of errors that will be appended in case any error occurs. :returns: a tuple with object metadata and a list of chunks. """ try: return self.api.object_locate(target.account, target.container, target.obj, version=target.version, properties=False) except exc.NoSuchObject as err: self.object_not_found += 1 errors.append('Not found: %s' % (err, )) except Exception as err: self.object_exceptions += 1 errors.append('Check failed: %s' % (err, )) return None, [] def _get_cached_or_lock(self, lock_key): # If something is running, wait for it with self.running_lock: event = self.running_tasks.get(lock_key) if event: event.wait() event = None # Maybe get a cached result if lock_key in self.list_cache: return self.list_cache[lock_key] # No cached result, try to compute the thing ourselves while True: with self.running_lock: # Another check while locked if lock_key in self.list_cache: return self.list_cache[lock_key] # Still nothing cached event = self.running_tasks.get(lock_key) if event is None: self.running_tasks[lock_key] = Event() return None event.wait() def _unlock(self, lock_key): with self.running_lock: event = self.running_tasks[lock_key] del self.running_tasks[lock_key] event.send(True) def check_obj(self, target, recurse=0): """ Check one object version. If no version is specified, all versions of the object will be checked. :returns: the result of the check of the most recent version, or the one that is explicitly targeted. """ account = target.account container = target.container obj = target.obj vers = target.version # can be None cached = self._get_cached_or_lock((account, container, obj, vers)) if cached is not None: return cached self.logger.info('Checking object "%s"', target) container_listing, _ = self.check_container(target.copy_container()) errors = list() if obj not in container_listing: errors.append('Missing from container listing') # checksum = None else: versions = container_listing[obj] if vers is None: if target.content_id is None: # No version specified, check all versions self.check_obj_versions(target.copy_object(), versions, recurse=recurse) # Now return the cached result of the most recent version target.content_id = versions[0]['id'] target.version = str(versions[0]['version']) res = self.check_obj(target, recurse=0) self._unlock((account, container, obj, vers)) return res else: for ov in versions: if ov['id'] == target.content_id: vers = str(ov['version']) target.version = vers break else: errors.append('Missing from container listing') # TODO check checksum match # checksum = container_listing[obj]['hash'] pass meta, chunks = self._load_obj_meta(target, errors) chunk_listing = {c['url']: c for c in chunks} if meta: if target.content_id is None: target.content_id = meta['id'] if target.version is None: target.version = str(meta['version']) self.list_cache[(account, container, obj, vers)] = \ (chunk_listing, meta) self.objects_checked += 1 self._unlock((account, container, obj, vers)) # Skip the check if we could not locate the object if meta: errors.extend( self._check_obj_policy(target, meta, chunks, recurse=recurse)) self.send_result(target, errors) return chunk_listing, meta def check_container(self, target, recurse=0): account = target.account container = target.container cached = self._get_cached_or_lock((account, container)) if cached is not None: return cached self.logger.info('Checking container "%s"', target) account_listing = self.check_account(target.copy_account()) errors = list() if container not in account_listing: errors.append('Missing from account listing') marker = None results = [] ct_meta = dict() extra_args = dict() if self.limit_listings > 1 and target.obj: # When we are explicitly checking one object, start the listing # where this object is supposed to be. Do not use a limit, # but an end marker, in order to fetch all versions of the object. extra_args['prefix'] = target.obj extra_args['end_marker'] = target.obj + '\x00' # HACK while True: try: resp = self.api.object_list(account, container, marker=marker, versions=True, **extra_args) except exc.NoSuchContainer as err: self.container_not_found += 1 errors.append('Not found: %s' % (err, )) break except Exception as err: self.container_exceptions += 1 errors.append('Check failed: %s' % (err, )) break truncated = resp.get('truncated', False) if truncated: marker = resp['next_marker'] if resp['objects']: # safeguard, probably useless if not marker: marker = resp['objects'][-1]['name'] results.extend(resp['objects']) if not truncated or self.limit_listings > 1: break else: ct_meta = resp ct_meta.pop('objects') break container_listing = dict() # Save all object versions, with the most recent first for obj in results: container_listing.setdefault(obj['name'], list()).append(obj) for versions in container_listing.values(): versions.sort(key=lambda o: o['version'], reverse=True) if self.limit_listings <= 1: # We just listed the whole container, keep the result in a cache self.containers_checked += 1 self.list_cache[(account, container)] = container_listing, ct_meta self._unlock((account, container)) if recurse > 0: for obj_vers in container_listing.values(): for obj in obj_vers: tcopy = target.copy_object() tcopy.obj = obj['name'] tcopy.content_id = obj['id'] tcopy.version = str(obj['version']) self._spawn_n(self.check_obj, tcopy, recurse - 1) self.send_result(target, errors) return container_listing, ct_meta def check_account(self, target, recurse=0): account = target.account cached = self._get_cached_or_lock(account) if cached is not None: return cached self.logger.info('Checking account "%s"', target) errors = list() marker = None results = [] extra_args = dict() if self.limit_listings > 0 and target.container: # When we are explicitly checking one container, start the listing # where this container is supposed to be, and list only one # container. extra_args['prefix'] = target.container extra_args['limit'] = 1 while True: try: resp = self.api.container_list(account, marker=marker, **extra_args) except Exception as err: self.account_exceptions += 1 errors.append('Check failed: %s' % (err, )) break if resp: marker = resp[-1][0] results.extend(resp) if self.limit_listings > 0: break else: break containers = dict() for container in results: # Name, number of objects, number of bytes containers[container[0]] = (container[1], container[2]) if self.limit_listings <= 0: # We just listed the whole account, keep the result in a cache self.accounts_checked += 1 self.list_cache[account] = containers self._unlock(account) if recurse > 0: for container in containers: tcopy = target.copy_account() tcopy.container = container self._spawn_n(self.check_container, tcopy, recurse - 1) self.send_result(target, errors) return containers def check(self, target, recurse=0): if target.type == 'chunk': self._spawn_n(self.check_chunk, target) elif target.type == 'object': self._spawn_n(self.check_obj, target, recurse) elif target.type == 'container': self._spawn_n(self.check_container, target, recurse) else: self._spawn_n(self.check_account, target, recurse) def check_all_accounts(self, recurse=0): all_accounts = self.api.account_list() for acct in all_accounts: self.check(Target(acct), recurse=recurse) def fetch_results(self, rate_limiter=None): while self.running and not self.result_queue.empty(): res = self.result_queue.get(True) yield res # Rate limiting is done on the result queue for now. # Someday we could implement a submission queue instead of # letting each worker submit tasks to the pool, and do # the rate limiting on this queue. if rate_limiter is not None: self.run_time = rate_limiter(self.run_time) def merge_with_delayed_target(self, target): """ Merge the specified target with a delayed one. :returns: the delayed target, if there is one, with an error log including the errors of the new target. Return the new target otherwise. """ tkey = repr(target) prev_target = self.delayed_targets.get(tkey, target) if prev_target is not target: errors = dict(prev_target.error_log) errors.update(target.error_log) prev_target.error_log = sorted(errors.items()) return prev_target def log_result(self, target): """ Log a check result, if it shows errors. Dispatch the errors to the appropriate destinations (log files, queues, etc.). """ # The result may come from a new target, or from an old target # we checked another time, or both. target = self.merge_with_delayed_target(target) if target.has_errors: time_in_error, confirmations = target.time_in_error() if (time_in_error < self.min_time_in_error or confirmations < self.required_confirmations): self.logger.info("Delaying check for %s, %d/%d confirmations", target, confirmations, self.required_confirmations) self.delayed_targets[repr(target)] = target else: if target.type == 'chunk': self.logger.info( "Writing error for %s, %d/%d confirmations", target, confirmations, self.required_confirmations) self.write_chunk_error(target, irreparable=target.irreparable) else: self.write_error(target, irreparable=target.irreparable) self.delayed_targets.pop(repr(target), None) self.logger.warn( '%s:%s\n%s', target, ' irreparable' if target.irreparable else '', target.latest_error_result().errors_to_str(err_format=' %s')) def run(self, rate_limiter=None): """ Fetch results and write logs until all jobs have finished. :returns: a generator yielding check results. """ while self.running and (self.pool.running() + self.pool.waiting()): for result in self.fetch_results(rate_limiter): self.log_result(result) yield result sleep(0.1) if self.running: self.pool.waitall() # No rate limiting for result in self.fetch_results(): self.log_result(result) yield result self.list_cache = CacheDict(self.list_cache.size) def stop(self): self.logger.info("Stopping") self.running = False def report(self): success = True def _report_stat(name, stat): print("{0:18}: {1}".format(name, stat)) print() print('Report') _report_stat("Accounts checked", self.accounts_checked) if self.account_not_found: success = False _report_stat("Missing accounts", self.account_not_found) if self.account_exceptions: success = False _report_stat("Exceptions", self.account_exceptions) print() _report_stat("Containers checked", self.containers_checked) if self.container_not_found: success = False _report_stat("Missing containers", self.container_not_found) if self.container_exceptions: success = False _report_stat("Exceptions", self.container_exceptions) print() _report_stat("Objects checked", self.objects_checked) if self.object_not_found: success = False _report_stat("Missing objects", self.object_not_found) if self.object_exceptions: success = False _report_stat("Exceptions", self.object_exceptions) print() _report_stat("Chunks checked", self.chunks_checked) if self.chunk_not_found: success = False _report_stat("Missing chunks", self.chunk_not_found) if self.chunk_exceptions: success = False _report_stat("Exceptions", self.chunk_exceptions) return success
class BlobRebuilderWorker(object): def __init__(self, conf, logger, volume): self.conf = conf self.logger = logger or get_logger(conf) self.volume = volume self.run_time = 0 self.passes = 0 self.errors = 0 self.last_reported = 0 self.chunks_run_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_chunks_processed = 0 self.dry_run = true_value(conf.get('dry_run', False)) self.report_interval = int_value(conf.get('report_interval'), 3600) self.max_chunks_per_second = int_value(conf.get('chunks_per_second'), 30) self.max_bytes_per_second = int_value(conf.get('bytes_per_second'), 10000000) self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100) self.allow_same_rawx = true_value(conf.get('allow_same_rawx')) self.rdir_client = RdirClient(conf) self.content_factory = ContentFactory(conf) def rebuilder_pass_with_lock(self): self.rdir_client.admin_lock(self.volume, "rebuilder on %s" % gethostname()) try: self.rebuilder_pass() finally: self.rdir_client.admin_unlock(self.volume) def rebuilder_pass(self): start_time = report_time = time.time() total_errors = 0 rebuilder_time = 0 chunks = self.rdir_client.chunk_fetch(self.volume, limit=self.rdir_fetch_limit, rebuild=True) for container_id, content_id, chunk_id, data in chunks: loop_time = time.time() if self.dry_run: self.dryrun_chunk_rebuild(container_id, content_id, chunk_id) else: self.safe_chunk_rebuild(container_id, content_id, chunk_id) self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second) self.total_chunks_processed += 1 now = time.time() if now - self.last_reported >= self.report_interval: self.logger.info( 'RUN %(volume)s ' 'started=%(start_time)s ' 'passes=%(passes)d ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' 'bytes=%(nb_bytes)d %(b_rate).2fB/s ' 'elapsed=%(total).2f ' '(rebuilder: %(rebuilder_rate).2f%%)' % { 'volume': self.volume, 'start_time': datetime.fromtimestamp(int(report_time)).isoformat(), 'passes': self.passes, 'errors': self.errors, 'nb_chunks': self.total_chunks_processed, 'nb_bytes': self.total_bytes_processed, 'c_rate': self.passes / (now - report_time), 'b_rate': self.bytes_processed / (now - report_time), 'total': (now - start_time), 'rebuilder_time': rebuilder_time, 'rebuilder_rate': 100.0 * rebuilder_time / float(now - start_time) }) report_time = now total_errors += self.errors self.passes = 0 self.bytes_processed = 0 self.last_reported = now rebuilder_time += (now - loop_time) end_time = time.time() elapsed = (end_time - start_time) or 0.000001 self.logger.info( 'DONE %(volume)s ' 'started=%(start_time)s ' 'ended=%(end_time)s ' 'elapsed=%(elapsed).02f ' 'errors=%(errors)d ' 'chunks=%(nb_chunks)d %(c_rate).2f/s ' 'bytes=%(nb_bytes)d %(b_rate).2fB/s ' 'elapsed=%(rebuilder_time).2f ' '(rebuilder: %(rebuilder_rate).2f%%)' % { 'volume': self.volume, 'start_time': datetime.fromtimestamp( int(start_time)).isoformat(), 'end_time': datetime.fromtimestamp(int(end_time)).isoformat(), 'elapsed': elapsed, 'errors': total_errors + self.errors, 'nb_chunks': self.total_chunks_processed, 'nb_bytes': self.total_bytes_processed, 'c_rate': self.total_chunks_processed / elapsed, 'b_rate': self.total_bytes_processed / elapsed, 'rebuilder_time': rebuilder_time, 'rebuilder_rate': 100.0 * rebuilder_time / float(elapsed) }) def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info( "[dryrun] Rebuilding " "container %s, content %s, chunk %s", container_id, content_id, chunk_id) self.passes += 1 def safe_chunk_rebuild(self, container_id, content_id, chunk_id): try: self.chunk_rebuild(container_id, content_id, chunk_id) except Exception as e: self.errors += 1 self.logger.error('ERROR while rebuilding chunk %s|%s|%s) : %s', container_id, content_id, chunk_id, e) self.passes += 1 def chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info('Rebuilding (container %s, content %s, chunk %s)', container_id, content_id, chunk_id) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') chunk = content.chunks.filter(id=chunk_id).one() if chunk is None: raise OrphanChunk("Chunk not found in content") chunk_size = chunk.size content.rebuild_chunk(chunk_id, allow_same_rawx=self.allow_same_rawx) self.rdir_client.chunk_delete(self.volume, container_id, content_id, chunk_id) self.bytes_processed += chunk_size self.total_bytes_processed += chunk_size
class TestIndexerCrawler(BaseTestCase): def setUp(self): super(TestIndexerCrawler, self).setUp() self.namespace = self.conf['namespace'] self.rawx_conf = self.conf['services']['rawx'][0] self.conf = {"namespace": self.namespace, "volume": self.rawx_conf['path']} self.rdir_client = RdirClient(self.conf) def tearDown(self): super(TestIndexerCrawler, self).tearDown() def _create_chunk(self, rawx_path, alias="toto"): container_id = random_id(64) content_id = random_id(32) chunk_id = random_id(64) chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3]) if not os.path.isdir(chunk_dir): os.makedirs(chunk_dir) chunk_path = "%s/%s" % (chunk_dir, chunk_id) with open(chunk_path, "w") as f: f.write("toto") xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * '0') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], chunk_id) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '0') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '4') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['container_id'], container_id) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_id'], content_id) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_path'], alias) xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_policy'], 'TESTPOLICY') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'], 'plain/nb_copy=3') xattr.setxattr( chunk_path, 'user.' + chunk_xattr_keys['content_version'], '0') return chunk_path, container_id, content_id, chunk_id def _rdir_get(self, rawx_addr, container_id, content_id, chunk_id): data = self.rdir_client.chunk_fetch(rawx_addr) key = (container_id, content_id, chunk_id) for i_container, i_content, i_chunk, i_value in data: if (i_container, i_content, i_chunk) == key: return i_value return None def _test_index_chunk(self, alias="toto"): # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( self.rawx_conf['path'], alias) # index the chunk indexer = BlobIndexer(self.conf) with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=1234)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(self.rawx_conf['addr'], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 1234) # index a chunk already indexed with mock.patch('oio.blob.indexer.time.time', mock.MagicMock(return_value=4567)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(self.rawx_conf['addr'], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value['mtime'], 4567) def test_index_chunk(self): return self._test_index_chunk() def test_index_unicode_chunk(self): return self._test_index_chunk('a%%%s%d%xàç"\r\n{0}€ 1+1=2/\\$\t_') def test_index_chunk_missing_xattr(self): # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk( self.rawx_conf['path']) # remove mandatory xattr xattr.removexattr( chunk_path, 'user.' + chunk_xattr_keys['container_id']) # try to index the chunk indexer = BlobIndexer(self.conf) self.assertRaises(FaultyChunk, indexer.update_index, chunk_path)
class TestRdirClient(unittest.TestCase): def setUp(self): super(TestRdirClient, self).setUp() self.namespace = "dummy" self.rdir_client = RdirClient({'namespace': self.namespace}, endpoint='127.0.0.0:6000') self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567") self.container_id_1 = random_id(64) self.container_id_2 = random_id(64) self.container_id_3 = random_id(64) self.content_id_1 = random_id(32) self.content_id_2 = random_id(32) self.content_id_3 = random_id(32) self.chunk_id_1 = random_id(64) self.chunk_id_2 = random_id(64) self.chunk_id_3 = random_id(64) def tearDown(self): super(TestRdirClient, self).tearDown() del self.rdir_client def test_fetch_one_req_post(self): self.rdir_client._direct_request = Mock( side_effect=[ ( FakeResponse(200), [ ["%s|%s|%s" % (self.container_id_1, self.content_id_1, self.chunk_id_1), {'mtime': 10}], ["%s|%s|%s" % (self.container_id_2, self.content_id_2, self.chunk_id_2), {'mtime': 20}], ] ), (FakeResponse(204), None) ]) gen = self.rdir_client.chunk_fetch("volume", limit=2) items = list(gen) self.assertEqual( items[0], (self.container_id_1, self.content_id_1, self.chunk_id_1, {'mtime': 10})) self.assertEqual( items[1], (self.container_id_2, self.content_id_2, self.chunk_id_2, {'mtime': 20})) self.assertEqual(2, len(items)) self.assertEqual(self.rdir_client._direct_request.call_count, 2) def test_fetch_multi_req(self): self.rdir_client._direct_request = Mock( side_effect=[ ( FakeResponse(200), [ ["%s|%s|%s" % (self.container_id_1, self.content_id_1, self.chunk_id_1), {'mtime': 10}], ["%s|%s|%s" % (self.container_id_2, self.content_id_2, self.chunk_id_2), {'mtime': 20}], ] ), ( FakeResponse(200), [ ["%s|%s|%s" % (self.container_id_3, self.content_id_3, self.chunk_id_3), {'mtime': 30}], ] ), (FakeResponse(204), None) ]) gen = self.rdir_client.chunk_fetch("volume", limit=2) items = list(gen) self.assertEqual( items[0], (self.container_id_1, self.content_id_1, self.chunk_id_1, {'mtime': 10})) self.assertEqual( items[1], (self.container_id_2, self.content_id_2, self.chunk_id_2, {'mtime': 20})) self.assertEqual( items[2], (self.container_id_3, self.content_id_3, self.chunk_id_3, {'mtime': 30})) self.assertEqual(3, len(items)) self.assertEqual(self.rdir_client._direct_request.call_count, 3)
class TestRdirClient(BaseTestCase): def setUp(self): super(TestRdirClient, self).setUp() self.namespace = self.conf['namespace'] self.rdir_client = RdirClient({'namespace': self.namespace}) self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567") self.container_id_1 = random_id(64) self.container_id_2 = random_id(64) self.container_id_3 = random_id(64) self.content_id_1 = random_id(32) self.content_id_2 = random_id(32) self.content_id_3 = random_id(32) self.chunk_id_1 = random_id(64) self.chunk_id_2 = random_id(64) self.chunk_id_3 = random_id(64) def tearDown(self): super(TestRdirClient, self).tearDown() del self.rdir_client def test_fetch_one_req_post(self): self.rdir_client._direct_request = Mock(side_effect=[(Mock(), [ [ "%s|%s|%s" % (self.container_id_1, self.content_id_1, self.chunk_id_1), { 'mtime': 10 } ], [ "%s|%s|%s" % (self.container_id_2, self.content_id_2, self.chunk_id_2), { 'mtime': 20 } ], ])]) gen = self.rdir_client.chunk_fetch("volume", limit=2) self.assertEqual( gen.next(), (self.container_id_1, self.content_id_1, self.chunk_id_1, { 'mtime': 10 })) self.assertEqual( gen.next(), (self.container_id_2, self.content_id_2, self.chunk_id_2, { 'mtime': 20 })) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 2) def test_fetch_multi_req(self): self.rdir_client._direct_request = Mock( side_effect=[(Mock(), [ [ "%s|%s|%s" % (self.container_id_1, self.content_id_1, self.chunk_id_1), { 'mtime': 10 } ], [ "%s|%s|%s" % (self.container_id_2, self.content_id_2, self.chunk_id_2), { 'mtime': 20 } ], ]), (Mock(), [ [ "%s|%s|%s" % (self.container_id_3, self.content_id_3, self.chunk_id_3), { 'mtime': 30 } ], ])]) gen = self.rdir_client.chunk_fetch("volume", limit=2) self.assertEqual( gen.next(), (self.container_id_1, self.content_id_1, self.chunk_id_1, { 'mtime': 10 })) self.assertEqual( gen.next(), (self.container_id_2, self.content_id_2, self.chunk_id_2, { 'mtime': 20 })) self.assertEqual( gen.next(), (self.container_id_3, self.content_id_3, self.chunk_id_3, { 'mtime': 30 })) self.assertRaises(StopIteration, gen.next) self.assertEqual(self.rdir_client._direct_request.call_count, 3)
class Checker(object): def __init__(self, namespace, concurrency=50, error_file=None, rebuild_file=None, full=True, limit_listings=0, request_attempts=1, logger=None, verbose=False, integrity=False): self.pool = GreenPool(concurrency) self.error_file = error_file self.full = bool(full) self.integrity = bool(integrity) # Optimisation for when we are only checking one object # or one container. # 0 -> do not limit # 1 -> limit account listings (list of containers) # 2 -> limit container listings (list of objects) self.limit_listings = limit_listings if self.error_file: outfile = open(self.error_file, 'a') self.error_writer = csv.writer(outfile, delimiter=' ') self.rebuild_file = rebuild_file if self.rebuild_file: fd = open(self.rebuild_file, 'a') self.rebuild_writer = csv.writer(fd, delimiter='|') self.logger = logger or get_logger( {'namespace': namespace}, name='integrity', verbose=verbose) self.api = ObjectStorageApi(namespace, logger=self.logger, max_retries=request_attempts - 1, request_attempts=request_attempts) self.rdir_client = RdirClient({"namespace": namespace}, logger=self.logger) self.accounts_checked = 0 self.containers_checked = 0 self.objects_checked = 0 self.chunks_checked = 0 self.account_not_found = 0 self.container_not_found = 0 self.object_not_found = 0 self.chunk_not_found = 0 self.account_exceptions = 0 self.container_exceptions = 0 self.object_exceptions = 0 self.chunk_exceptions = 0 self.list_cache = {} self.running = {} self.running_lock = Semaphore(1) self.result_queue = Queue() def complete_target_from_chunk_metadata(self, target, xattr_meta): """ Complete a Target object from metadata found in chunk's extended attributes. In case the "fullpath" is not available, try to read legacy metadata, and maybe ask meta1 to resolve the CID into account and container names. """ # pylint: disable=unbalanced-tuple-unpacking try: acct, ct, path, vers, content_id = \ decode_fullpath(xattr_meta['full_path']) target.account = acct target.container = ct target.obj = path target.content_id = content_id target.version = vers except KeyError: # No fullpath header, try legacy headers if 'content_path' in xattr_meta: target.obj = xattr_meta['content_path'] if 'content_id' in xattr_meta: target.content_id = xattr_meta['content_id'] if 'content_version' in xattr_meta: target.version = xattr_meta['content_version'] cid = xattr_meta.get('container_id') if cid: try: md = self.api.directory.show(cid=cid) acct = md.get('account') ct = md.get('name') if acct: target.account = acct if ct: target.container = ct except Exception as err: self.logger.warn( "Failed to resolve CID %s into account " "and container names: %s", cid, err) def recover_and_complete_object_meta(self, target, chunk): _, rawx_service, chunk_id = chunk.rsplit('/', 2) # 1. Fetch chunk list from rdir (could be cached). # Unfortunately we cannot seek for a chunk ID. entries = [ x for x in self.rdir_client.chunk_fetch(rawx_service, limit=-1) if x[2] == chunk_id ] if not entries: self.logger.warn('Chunk %s not found in rdir' % chunk_id) return elif len(entries) > 1: self.logger.info('Chunk %s appears in %d objects', chunk_id, len(entries)) # 2. Find content and container IDs target.cid, target.content_id = entries[0][0:2] meta = self.api.object_get_properties(None, None, None, cid=target.cid, content=target.content_id) target.obj = meta['name'] target.version = meta['version'] target.account, target.container = self.api.resolve_cid(target.cid) def send_result(self, target, errors=None): """ Put an item in the result queue. """ # TODO(FVE): send to an external queue. self.result_queue.put(ItemResult(target, errors)) def write_error(self, target, irreparable=False): if not self.error_file: return error = list() if irreparable: error.append('#IRREPARABLE') error.append(target.account) if target.container: error.append(target.container) if target.obj: error.append(target.obj) if target.chunk: error.append(target.chunk) self.error_writer.writerow(error) def write_rebuilder_input(self, target, irreparable=False): # FIXME(FVE): cid can be computed from account and container names if target.cid: cid = target.cid else: ct_meta = self.list_cache[(target.account, target.container)][1] try: cid = ct_meta['system']['sys.name'].split('.', 1)[0] except KeyError: cid = ct_meta['properties']['sys.name'].split('.', 1)[0] error = list() if irreparable: error.append('#IRREPARABLE') error.append(cid) # FIXME(FVE): ensure we always resolve content_id, # or pass object version along with object name. error.append(target.content_id or target.obj) error.append(target.chunk) self.rebuild_writer.writerow(error) def write_chunk_error(self, target, chunk=None, irreparable=False): if chunk is not None: target = target.copy() target.chunk = chunk self.write_error(target, irreparable=irreparable) if self.rebuild_file: self.write_rebuilder_input(target, irreparable=irreparable) def _check_chunk_xattr(self, target, obj_meta, xattr_meta): """ Check coherency of chunk extended attributes with object metadata. :returns: a list of errors """ errors = list() # Composed position -> erasure coding attr_prefix = 'meta' if '.' in obj_meta['pos'] else '' attr_key = attr_prefix + 'chunk_size' if str(obj_meta['size']) != xattr_meta.get(attr_key): errors.append( "'%s' xattr (%s) differs from size in meta2 (%s)" % (attr_key, xattr_meta.get(attr_key), obj_meta['size'])) attr_key = attr_prefix + 'chunk_hash' if obj_meta['hash'] != xattr_meta.get(attr_key): errors.append( "'%s' xattr (%s) differs from hash in meta2 (%s)" % (attr_key, xattr_meta.get(attr_key), obj_meta['hash'])) return errors def _check_chunk(self, target): """ Execute various checks on a chunk: - does it appear in object's chunk list? - is it reachable? - are its extended attributes coherent? :returns: the list of errors encountered, and the chunk's owner object metadata. """ chunk = target.chunk errors = list() obj_meta = None xattr_meta = None try: xattr_meta = self.api.blob_client.chunk_head( chunk, xattr=self.full, check_hash=self.integrity) except exc.NotFound as err: self.chunk_not_found += 1 errors.append('Not found: %s' % (err, )) except exc.FaultyChunk as err: self.chunk_exceptions += 1 errors.append('Faulty: %r' % (err, )) except Exception as err: self.chunk_exceptions += 1 errors.append('Check failed: %s' % (err, )) if not target.obj: if xattr_meta: self.complete_target_from_chunk_metadata(target, xattr_meta) else: self.recover_and_complete_object_meta(target, chunk) if target.obj: obj_listing, obj_meta = self.check_obj(target.copy_object()) if chunk not in obj_listing: errors.append('Missing from object listing') db_meta = dict() else: db_meta = obj_listing[chunk] if db_meta and xattr_meta and self.full: errors.extend( self._check_chunk_xattr(target, db_meta, xattr_meta)) self.send_result(target, errors) self.chunks_checked += 1 return errors, obj_meta def check_chunk(self, target): errors, _obj_meta = self._check_chunk(target) return errors def _check_metachunk(self, target, stg_met, pos, chunks, recurse=0): """ Check that a metachunk has the right number of chunks. :returns: the list of errors """ required = stg_met.expected_chunks errors = list() if len(chunks) < required: missing_chunks = required - len(chunks) if stg_met.ec: subs = {x['num'] for x in chunks} for sub in range(required): if sub not in subs: errors.append("Missing chunk at position %d.%d" % (pos, sub)) else: for _ in range(missing_chunks): errors.append("Missing chunk at position %d" % pos) if recurse > 0: for chunk in chunks: tcopy = target.copy() tcopy.chunk = chunk['url'] chunk_errors, _ = self._check_chunk(tcopy) if chunk_errors: # The errors have already been reported by _check_chunk, # but we must count this chunk among the unusable chunks # of the current metachunk. errors.append("Unusable chunk %s at position %s" % (chunk['url'], chunk['pos'])) irreparable = required - len(errors) < stg_met.min_chunks_to_read if irreparable: errors.append( "Unavailable metachunk at position %s (%d/%d chunks)" % (pos, required - len(errors), stg_met.expected_chunks)) # Since the "metachunk" is not an official item type, # this method does not report errors itself. Errors will # be reported as object errors. return errors def _check_obj_policy(self, target, obj_meta, chunks, recurse=0): """ Check that the list of chunks of an object matches the object's storage policy. :returns: the list of errors encountered """ stg_met = STORAGE_METHODS.load(obj_meta['chunk_method']) chunks_by_pos = _sort_chunks(chunks, stg_met.ec) tasks = list() for pos, pchunks in chunks_by_pos.iteritems(): tasks.append((pos, self.pool.spawn(self._check_metachunk, target.copy(), stg_met, pos, pchunks, recurse=recurse))) errors = list() for pos, task in tasks: try: errors.extend(task.wait()) except Exception as err: errors.append("Check failed: pos %d: %s" % (pos, err)) return errors def check_obj_versions(self, target, versions, recurse=0): """ Run checks of all versions of the targeted object in parallel. """ tasks = list() for ov in versions: tcopy = target.copy_object() tcopy.content_id = ov['id'] tcopy.version = str(ov['version']) tasks.append((tcopy.version, self.pool.spawn(self.check_obj, tcopy, recurse=recurse))) errors = list() for version, task in tasks: try: task.wait() except Exception as err: errors.append("Check failed: version %s: %s" % (version, err)) if errors: # Send a result with the target without version to tell # we were not able to check all versions of the object. self.send_result(target, errors) def _load_obj_meta(self, target, errors): """ Load object metadata and chunks. :param target: which object to check. :param errors: list of errors that will be appended in case any error occurs. :returns: a tuple with object metadata and a list of chunks. """ try: return self.api.object_locate(target.account, target.container, target.obj, version=target.version, properties=False) except exc.NoSuchObject as err: self.object_not_found += 1 errors.append('Not found: %s' % (err, )) except Exception as err: self.object_exceptions += 1 errors.append('Check failed: %s' % (err, )) return None, [] def _get_cached_or_lock(self, lock_key): # If something is running, wait for it with self.running_lock: event = self.running.get(lock_key) if event: event.wait() event = None # Maybe get a cached result if lock_key in self.list_cache: return self.list_cache[lock_key] # No cached result, try to compute the thing ourselves while True: with self.running_lock: # Another check while locked if lock_key in self.list_cache: return self.list_cache[lock_key] # Still nothing cached event = self.running.get(lock_key) if event is None: self.running[lock_key] = Event() return None event.wait() def _unlock(self, lock_key): with self.running_lock: event = self.running[lock_key] del self.running[lock_key] event.send(True) def check_obj(self, target, recurse=0): """ Check one object version. If no version is specified, all versions of the object will be checked. :returns: the result of the check of the most recent version, or the one that is explicitly targeted. """ account = target.account container = target.container obj = target.obj vers = target.version # can be None cached = self._get_cached_or_lock((account, container, obj, vers)) if cached: return cached self.logger.info('Checking object "%s"', target) container_listing, _ = self.check_container(target.copy_container()) errors = list() if obj not in container_listing: errors.append('Missing from container listing') # checksum = None else: versions = container_listing[obj] if vers is None: if target.content_id is None: # No version specified, check all versions self.check_obj_versions(target.copy_object(), versions, recurse=recurse) # Now return the cached result of the most recent version target.content_id = versions[0]['id'] target.version = str(versions[0]['version']) res = self.check_obj(target, recurse=0) self._unlock((account, container, obj, vers)) return res else: for ov in versions: if ov['id'] == target.content_id: vers = str(ov['version']) target.version = vers break else: errors.append('Missing from container listing') # TODO check checksum match # checksum = container_listing[obj]['hash'] pass meta, chunks = self._load_obj_meta(target, errors) chunk_listing = {c['url']: c for c in chunks} if meta: if target.content_id is None: target.content_id = meta['id'] if target.version is None: target.version = str(meta['version']) self.list_cache[(account, container, obj, vers)] = \ (chunk_listing, meta) self.objects_checked += 1 self._unlock((account, container, obj, vers)) # Skip the check if we could not locate the object if meta: errors.extend( self._check_obj_policy(target, meta, chunks, recurse=recurse)) self.send_result(target, errors) return chunk_listing, meta def check_container(self, target, recurse=0): account = target.account container = target.container cached = self._get_cached_or_lock((account, container)) if cached: return cached self.logger.info('Checking container "%s"', target) account_listing = self.check_account(target.copy_account()) errors = list() if container not in account_listing: errors.append('Missing from account listing') marker = None results = [] ct_meta = dict() extra_args = dict() if self.limit_listings > 1 and target.obj: # When we are explicitly checking one object, start the listing # where this object is supposed to be. Do not use a limit, # but an end marker, in order to fetch all versions of the object. extra_args['prefix'] = target.obj extra_args['end_marker'] = target.obj + '\x00' # HACK while True: try: resp = self.api.object_list(account, container, marker=marker, versions=True, **extra_args) except exc.NoSuchContainer as err: self.container_not_found += 1 errors.append('Not found: %s' % (err, )) break except Exception as err: self.container_exceptions += 1 errors.append('Check failed: %s' % (err, )) break if resp.get('truncated', False): marker = resp['next_marker'] if resp['objects']: # safeguard, probably useless if not marker: marker = resp['objects'][-1]['name'] results.extend(resp['objects']) if self.limit_listings > 1: break else: ct_meta = resp ct_meta.pop('objects') break container_listing = dict() # Save all object versions, with the most recent first for obj in results: container_listing.setdefault(obj['name'], list()).append(obj) for versions in container_listing.values(): versions.sort(key=lambda o: o['version'], reverse=True) if self.limit_listings <= 1: # We just listed the whole container, keep the result in a cache self.containers_checked += 1 self.list_cache[(account, container)] = container_listing, ct_meta self._unlock((account, container)) if recurse > 0: for obj_vers in container_listing.values(): for obj in obj_vers: tcopy = target.copy_object() tcopy.obj = obj['name'] tcopy.content_id = obj['id'] tcopy.version = str(obj['version']) self.pool.spawn_n(self.check_obj, tcopy, recurse - 1) self.send_result(target, errors) return container_listing, ct_meta def check_account(self, target, recurse=0): account = target.account cached = self._get_cached_or_lock(account) if cached: return cached self.logger.info('Checking account "%s"', target) errors = list() marker = None results = [] extra_args = dict() if self.limit_listings > 0 and target.container: # When we are explicitly checking one container, start the listing # where this container is supposed to be, and list only one # container. extra_args['prefix'] = target.container extra_args['limit'] = 1 while True: try: resp = self.api.container_list(account, marker=marker, **extra_args) except Exception as err: self.account_exceptions += 1 errors.append('Check failed: %s' % (err, )) break if resp: marker = resp[-1][0] results.extend(resp) if self.limit_listings > 0: break else: break containers = dict() for container in results: # Name, number of objects, number of bytes containers[container[0]] = (container[1], container[2]) if self.limit_listings <= 0: # We just listed the whole account, keep the result in a cache self.accounts_checked += 1 self.list_cache[account] = containers self._unlock(account) if recurse > 0: for container in containers: tcopy = target.copy_account() tcopy.container = container self.pool.spawn_n(self.check_container, tcopy, recurse - 1) self.send_result(target, errors) return containers def check(self, target, recurse=0): if target.type == 'chunk': self.pool.spawn_n(self.check_chunk, target) elif target.type == 'object': self.pool.spawn_n(self.check_obj, target, recurse) elif target.type == 'container': self.pool.spawn_n(self.check_container, target, recurse) else: self.pool.spawn_n(self.check_account, target, recurse) def check_all_accounts(self, recurse=0): all_accounts = self.api.account_list() for acct in all_accounts: self.check(Target(acct), recurse=recurse) def fetch_results(self): while not self.result_queue.empty(): res = self.result_queue.get(True) yield res def log_result(self, result): if result.errors: if result.target.type == 'chunk': # FIXME(FVE): check error criticity # and set the irreparable flag. self.write_chunk_error(result.target) else: self.write_error(result.target) self.logger.warn('%s:\n%s', result.target, result.errors_to_str(err_format=' %s')) def run(self): """ Fetch results and write logs until all jobs have finished. :returns: a generator yielding check results. """ while self.pool.running() + self.pool.waiting(): for result in self.fetch_results(): self.log_result(result) yield result sleep(0.1) self.pool.waitall() for result in self.fetch_results(): self.log_result(result) yield result def report(self): success = True def _report_stat(name, stat): print("{0:18}: {1}".format(name, stat)) print() print('Report') _report_stat("Accounts checked", self.accounts_checked) if self.account_not_found: success = False _report_stat("Missing accounts", self.account_not_found) if self.account_exceptions: success = False _report_stat("Exceptions", self.account_exceptions) print() _report_stat("Containers checked", self.containers_checked) if self.container_not_found: success = False _report_stat("Missing containers", self.container_not_found) if self.container_exceptions: success = False _report_stat("Exceptions", self.container_exceptions) print() _report_stat("Objects checked", self.objects_checked) if self.object_not_found: success = False _report_stat("Missing objects", self.object_not_found) if self.object_exceptions: success = False _report_stat("Exceptions", self.object_exceptions) print() _report_stat("Chunks checked", self.chunks_checked) if self.chunk_not_found: success = False _report_stat("Missing chunks", self.chunk_not_found) if self.chunk_exceptions: success = False _report_stat("Exceptions", self.chunk_exceptions) return success
class TestBlobIndexer(BaseTestCase): def setUp(self): super(TestBlobIndexer, self).setUp() self.rdir_client = RdirClient(self.conf) self.blob_client = BlobClient(self.conf) _, self.rawx_path, rawx_addr, _ = \ self.get_service_url('rawx') services = self.conscience.all_services('rawx') self.rawx_id = None for rawx in services: if rawx_addr == rawx['addr']: self.rawx_id = rawx['tags'].get('tag.service_id', None) if self.rawx_id is None: self.rawx_id = rawx_addr conf = self.conf.copy() conf['volume'] = self.rawx_path self.blob_indexer = BlobIndexer(conf) # clear rawx/rdir chunk_files = paths_gen(self.rawx_path) for chunk_file in chunk_files: os.remove(chunk_file) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) def _put_chunk(self): account = random_str(16) container = random_str(16) cid = cid_from_name(account, container) content_path = random_str(16) content_version = 1234567890 content_id = random_id(32) fullpath = encode_fullpath(account, container, content_path, content_version, content_id) chunk_id = random_chunk_id() data = random_buffer(string.printable, 100) meta = { 'full_path': fullpath, 'container_id': cid, 'content_path': content_path, 'version': content_version, 'id': content_id, 'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3', 'policy': 'TESTPOLICY', 'chunk_hash': md5(data).hexdigest().upper(), 'oio_version': OIO_VERSION, 'chunk_pos': 0, 'metachunk_hash': md5().hexdigest(), 'metachunk_size': 1024 } self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id, meta, data) sleep(1) # ensure chunk event have been processed return account, container, cid, content_path, content_version, \ content_id, chunk_id def _delete_chunk(self, chunk_id): self.blob_client.chunk_delete('http://' + self.rawx_id + '/' + chunk_id) sleep(1) # ensure chunk event have been processed def _link_chunk(self, target_chunk_id): account = random_str(16) container = random_str(16) cid = cid_from_name(account, container) content_path = random_str(16) content_version = 1234567890 content_id = random_id(32) fullpath = encode_fullpath(account, container, content_path, content_version, content_id) _, link = self.blob_client.chunk_link( 'http://' + self.rawx_id + '/' + target_chunk_id, None, fullpath) chunk_id = link.split('/')[-1] sleep(1) # ensure chunk event have been processed return account, container, cid, content_path, content_version, \ content_id, chunk_id def _chunk_path(self, chunk_id): return self.rawx_path + '/' + chunk_id[:3] + '/' + chunk_id def test_blob_indexer(self): _, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \ self._put_chunk() chunks = list(self.rdir_client.chunk_fetch(self.rawx_id)) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks)) def test_blob_indexer_with_old_chunk(self): expected_account, expected_container, expected_cid, \ expected_content_path, expected_content_version, \ expected_content_id, expected_chunk_id = self._put_chunk() chunks = list(self.rdir_client.chunk_fetch(self.rawx_id)) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) convert_to_old_chunk(self._chunk_path(chunk_id), expected_account, expected_container, expected_content_path, expected_content_version, expected_content_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks)) def test_blob_indexer_with_linked_chunk(self): _, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \ self._put_chunk() chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) _, _, linked_cid, _, _, linked_content_id, linked_chunk_id = \ self._link_chunk(expected_chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(2, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(2, len(chunks)) self.assertNotEqual(chunks[0][2], chunks[1][2]) for chunk in chunks: cid, content_id, chunk_id, _ = chunk if chunk_id == expected_chunk_id: self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) else: self.assertEqual(linked_cid, cid) self.assertEqual(linked_content_id, content_id) self.assertEqual(linked_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(linked_cid, cid) self.assertEqual(linked_content_id, content_id) self.assertEqual(linked_chunk_id, chunk_id) self._delete_chunk(linked_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks))
class TestIndexerCrawler(BaseTestCase): def setUp(self): super(TestIndexerCrawler, self).setUp() self.namespace = self.conf["namespace"] self.gridconf = {"namespace": self.namespace} self.rdir_client = RdirClient(self.gridconf) def tearDown(self): super(TestIndexerCrawler, self).tearDown() def _create_chunk(self, rawx_path): container_id = generate_id(64) content_id = generate_id(64) chunk_id = generate_id(64) chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:2]) if not os.path.isdir(chunk_dir): os.makedirs(chunk_dir) chunk_path = "%s/%s" % (chunk_dir, chunk_id) with open(chunk_path, "w") as f: f.write("toto") xattr.setxattr(chunk_path, "user.grid.chunk.hash", 32 * "0") xattr.setxattr(chunk_path, "user.grid.chunk.id", chunk_id) xattr.setxattr(chunk_path, "user.grid.chunk.position", "0") xattr.setxattr(chunk_path, "user.grid.chunk.size", "4") xattr.setxattr(chunk_path, "user.grid.content.container", container_id) xattr.setxattr(chunk_path, "user.grid.content.id", content_id) xattr.setxattr(chunk_path, "user.grid.content.nbchunk", "1") xattr.setxattr(chunk_path, "user.grid.content.path", "toto") xattr.setxattr(chunk_path, "user.grid.content.size", "4") xattr.setxattr(chunk_path, "user.grid.content.version", "0") return chunk_path, container_id, content_id, chunk_id def _rdir_get(self, rawx_addr, container_id, content_id, chunk_id): data = self.rdir_client.chunk_fetch(rawx_addr) key = (container_id, content_id, chunk_id) for i_container, i_content, i_chunk, i_value in data: if (i_container, i_content, i_chunk) == key: return i_value return None def test_index_chunk(self): rawx_conf = self.conf["rawx"][0] # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk(rawx_conf["path"]) # index the chunk indexer = BlobIndexerWorker(self.gridconf, None, rawx_conf["path"]) with mock.patch("oio.blob.indexer.time.time", mock.MagicMock(return_value=1234)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(rawx_conf["addr"], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value["content_nbchunks"], 1) self.assertEqual(check_value["chunk_hash"], 32 * "0") self.assertEqual(check_value["content_size"], 4) self.assertEqual(check_value["content_path"], "toto") self.assertEqual(check_value["chunk_position"], "0") self.assertEqual(check_value["chunk_size"], 4) self.assertEqual(check_value["mtime"], 1234) self.assertEqual(check_value["content_version"], 0) # index a chunk already indexed with mock.patch("oio.blob.indexer.time.time", mock.MagicMock(return_value=4567)): indexer.update_index(chunk_path) # check rdir check_value = self._rdir_get(rawx_conf["addr"], container_id, content_id, chunk_id) self.assertIsNotNone(check_value) self.assertEqual(check_value["mtime"], 4567) def test_index_chunk_missing_xattr(self): rawx_conf = self.conf["rawx"][0] # create a fake chunk chunk_path, container_id, content_id, chunk_id = self._create_chunk(rawx_conf["path"]) # remove mandatory xattr xattr.removexattr(chunk_path, "user.grid.chunk.hash") # try to index the chunk indexer = BlobIndexerWorker(self.gridconf, None, rawx_conf["path"]) self.assertRaises(FaultyChunk, indexer.update_index, chunk_path)