def _convert_and_check(self, chunk_volume, chunk_path, chunk_id_info, expected_raw_meta=None, expected_errors=0): conf = self.conf conf['volume'] = self.rawx_volumes[chunk_volume] converter = BlobConverter(conf, logger=self.logger) converter.safe_convert_chunk(chunk_path) self.assertEqual(1, converter.total_chunks_processed) self.assertEqual(1, converter.passes) self.assertEqual(expected_errors, converter.errors) checker = Checker(self.ns) for chunk_id, info in chunk_id_info.items(): account, container, path, version, content_id = info fullpath = encode_fullpath(account, container, path, version, content_id) cid = cid_from_name(account, container) meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id) self.assertEqual(meta.get('chunk_id'), chunk_id) self.assertEqual(meta.get('container_id'), cid) self.assertEqual(meta.get('content_path'), path) self.assertEqual(meta.get('content_version'), version) self.assertEqual(meta.get('content_id'), content_id) self.assertEqual(meta.get('full_path'), fullpath) checker.check( Target(account, container=container, obj=path, chunk='http://' + converter.volume_id + '/' + chunk_id)) for _ in checker.run(): pass self.assertTrue(checker.report()) if expected_raw_meta: self.assertDictEqual(expected_raw_meta, raw_meta) continue self.assertNotIn(CHUNK_XATTR_KEYS['chunk_id'], raw_meta) self.assertNotIn(CHUNK_XATTR_KEYS['container_id'], raw_meta) self.assertNotIn(CHUNK_XATTR_KEYS['content_path'], raw_meta) self.assertNotIn(CHUNK_XATTR_KEYS['content_version'], raw_meta) self.assertNotIn(CHUNK_XATTR_KEYS['content_id'], raw_meta) self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id, raw_meta) for k in raw_meta.keys(): if k.startswith('oio:'): self.fail('old fullpath always existing') self.assertEqual(raw_meta[CHUNK_XATTR_KEYS['oio_version']], OIO_VERSION)
class TestIntegrityCrawler(BaseTestCase): def setUp(self): super(TestIntegrityCrawler, self).setUp() self.container = 'ct-' + random_str(8) self.obj = 'obj-' + random_str(8) self.account = 'test-integrity-' + random_str(8) self.storage.object_create(self.account, self.container, obj_name=self.obj, data="chunk") _, self.rebuild_file = tempfile.mkstemp() self.checker = Checker(self.ns, rebuild_file=self.rebuild_file) self.meta, chunks = self.storage.object_locate(self.account, self.container, self.obj) self.chunk = chunks[0] self.irreparable = len(chunks) == 1 self.storage.blob_client.chunk_delete(self.chunk['real_url']) def tearDown(self): super(TestIntegrityCrawler, self).tearDown() os.remove(self.rebuild_file) self.storage.container_flush(self.account, self.container) self.storage.container_delete(self.account, self.container) self.wait_for_event('oio-preserved', type_=EventTypes.CONTAINER_DELETED, fields={'user': self.container}) self.storage.account_delete(self.account) def _verify_rebuilder_input(self): try: line = fileinput.input(self.rebuild_file).next().strip() cid = cid_from_name(self.account, self.container) expected = '|'.join([cid, self.meta['id'], self.chunk['url']]) if self.irreparable: expected = IRREPARABLE_PREFIX + '|' + expected self.assertEqual(expected, line) finally: fileinput.close() def test_account_rebuilder_output(self): self.checker.check(Target(self.account), recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() self._verify_rebuilder_input() def test_container_rebuilder_output(self): self.checker.check(Target(self.account, container=self.container), recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() self._verify_rebuilder_input() def test_object_rebuilder_output(self): self.checker.check(Target(self.account, container=self.container, obj=self.obj), recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() self._verify_rebuilder_input() def test_object_rebuilder_output_with_confirmations(self): """ Check that chunk targets showing errors are reported only after the right number of confirmations. """ self.checker.required_confirmations = 2 tgt = Target(self.account, container=self.container, obj=self.obj, content_id=self.meta['id'], version=self.meta['version']) self.checker.check(tgt, recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() # File is empty self.assertRaises(StopIteration, self._verify_rebuilder_input) self.assertIn(repr(tgt), self.checker.delayed_targets) # 1st confirmation for dtgt in self.checker.delayed_targets.values(): self.checker.check(dtgt, recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() # File is empty self.assertRaises(StopIteration, self._verify_rebuilder_input) self.assertIn(repr(tgt), self.checker.delayed_targets) # 2nd confirmation for dtgt in self.checker.delayed_targets.values(): self.checker.check(dtgt, recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() # File is NOT empty self._verify_rebuilder_input() self.assertNotIn(repr(tgt), self.checker.delayed_targets)
class ItemCheckCommand(lister.Lister): """ Various parameters that apply to all check commands. """ columns = ('Type', 'Item', 'Status', 'Errors') success = True checker = None @property def logger(self): return self.app.client_manager.logger def get_parser(self, prog_name): parser = super(ItemCheckCommand, self).get_parser(prog_name) parser.add_argument( '--attempts', type=int, default=1, help="Number of attempts for listing requests (default: 1).") parser.add_argument('--checksum', action='store_true', help=("Perform checksum comparisons.")) parser.add_argument('--concurrency', '--workers', type=int, default=30, help="Number of concurrent checks (default: 30).") parser.add_argument( '-o', '--output', help=("Output file. Will contain elements in error. " "Can later be passed to stdin of the legacy " "oio-crawler-integrity to re-check only these elements.")) parser.add_argument( '--output-for-chunk-rebuild', help=("Write chunk errors in a file with a format " "suitable as 'openio-admin chunk rebuild' input.")) return parser def _take_action(self, parsed_args): raise NotImplementedError() def take_action(self, parsed_args): self.logger.debug('take_action(%s)', parsed_args) self.checker = Checker( self.app.options.ns, concurrency=parsed_args.concurrency, error_file=parsed_args.output, rebuild_file=parsed_args.output_for_chunk_rebuild, request_attempts=parsed_args.attempts, check_hash=parsed_args.checksum, logger=self.logger) return self.columns, self._take_action(parsed_args) def _format_results(self): for res in self.checker.run(): if not res.errors: status = 'OK' else: self.success = False status = 'error' yield (res.target.type, repr(res.target), status, res.errors_to_str()) def run(self, parsed_args): super(ItemCheckCommand, self).run(parsed_args) if not self.success: return 1
class TestIntegrityCrawler(BaseTestCase): def setUp(self): super(TestIntegrityCrawler, self).setUp() self.container = 'ct-' + random_str(8) self.obj = 'obj-' + random_str(8) self.account = 'test-integrity-' + random_str(8) self.storage.object_create(self.account, self.container, obj_name=self.obj, data="chunk") _, self.rebuild_file = tempfile.mkstemp() self.checker = Checker(self.ns, rebuild_file=self.rebuild_file) self.meta, chunks = self.storage.object_locate(self.account, self.container, self.obj) self.chunk = chunks[0] self.irreparable = len(chunks) == 1 self.storage.blob_client.chunk_delete(self.chunk['real_url']) def tearDown(self): super(TestIntegrityCrawler, self).tearDown() os.remove(self.rebuild_file) self.storage.container_flush(self.account, self.container) self.storage.container_delete(self.account, self.container) self.wait_for_event('oio-preserved', type_=EventTypes.CONTAINER_DELETED, fields={'user': self.container}) self.storage.account_delete(self.account) def _verify_rebuilder_input(self): try: line = fileinput.input(self.rebuild_file).next().strip() cid = cid_from_name(self.account, self.container) expected = '|'.join([cid, self.meta['id'], self.chunk['url']]) if self.irreparable: expected = IRREPARABLE_PREFIX + '|' + expected self.assertEqual(expected, line) finally: fileinput.close() def test_account_rebuilder_output(self): self.checker.check(Target(self.account), recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() self._verify_rebuilder_input() def test_container_rebuilder_output(self): self.checker.check(Target(self.account, container=self.container), recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() self._verify_rebuilder_input() def test_object_rebuilder_output(self): self.checker.check(Target(self.account, container=self.container, obj=self.obj), recurse=DEFAULT_DEPTH) for _ in self.checker.run(): pass self.checker.fd.flush() self._verify_rebuilder_input()