def _convert_and_check(self,
chunk_volume,
chunk_path,
chunk_id_info,
expected_raw_meta=None,
expected_errors=0):
conf = self.conf
conf['volume'] = self.rawx_volumes[chunk_volume]
converter = BlobConverter(conf, logger=self.logger)
converter.safe_convert_chunk(chunk_path)
self.assertEqual(1, converter.total_chunks_processed)
self.assertEqual(1, converter.passes)
self.assertEqual(expected_errors, converter.errors)
checker = Checker(self.ns)
for chunk_id, info in chunk_id_info.items():
account, container, path, version, content_id = info
fullpath = encode_fullpath(account, container, path, version,
content_id)
cid = cid_from_name(account, container)
meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id)
self.assertEqual(meta.get('chunk_id'), chunk_id)
self.assertEqual(meta.get('container_id'), cid)
self.assertEqual(meta.get('content_path'), path)
self.assertEqual(meta.get('content_version'), version)
self.assertEqual(meta.get('content_id'), content_id)
self.assertEqual(meta.get('full_path'), fullpath)
checker.check(
Target(account,
container=container,
obj=path,
chunk='http://' + converter.volume_id + '/' + chunk_id))
for _ in checker.run():
pass
self.assertTrue(checker.report())
if expected_raw_meta:
self.assertDictEqual(expected_raw_meta, raw_meta)
continue
self.assertNotIn(CHUNK_XATTR_KEYS['chunk_id'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['container_id'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_path'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_version'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_id'], raw_meta)
self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id,
raw_meta)
for k in raw_meta.keys():
if k.startswith('oio:'):
self.fail('old fullpath always existing')
self.assertEqual(raw_meta[CHUNK_XATTR_KEYS['oio_version']],
OIO_VERSION)
class TestIntegrityCrawler(BaseTestCase):
def setUp(self):
super(TestIntegrityCrawler, self).setUp()
self.container = 'ct-' + random_str(8)
self.obj = 'obj-' + random_str(8)
self.account = 'test-integrity-' + random_str(8)
self.storage.object_create(self.account,
self.container,
obj_name=self.obj,
data="chunk")
_, self.rebuild_file = tempfile.mkstemp()
self.checker = Checker(self.ns, rebuild_file=self.rebuild_file)
self.meta, chunks = self.storage.object_locate(self.account,
self.container,
self.obj)
self.chunk = chunks[0]
self.irreparable = len(chunks) == 1
self.storage.blob_client.chunk_delete(self.chunk['real_url'])
def tearDown(self):
super(TestIntegrityCrawler, self).tearDown()
os.remove(self.rebuild_file)
self.storage.container_flush(self.account, self.container)
self.storage.container_delete(self.account, self.container)
self.wait_for_event('oio-preserved',
type_=EventTypes.CONTAINER_DELETED,
fields={'user': self.container})
self.storage.account_delete(self.account)
def _verify_rebuilder_input(self):
try:
line = fileinput.input(self.rebuild_file).next().strip()
cid = cid_from_name(self.account, self.container)
expected = '|'.join([cid, self.meta['id'], self.chunk['url']])
if self.irreparable:
expected = IRREPARABLE_PREFIX + '|' + expected
self.assertEqual(expected, line)
finally:
fileinput.close()
def test_account_rebuilder_output(self):
self.checker.check(Target(self.account), recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
self._verify_rebuilder_input()
def test_container_rebuilder_output(self):
self.checker.check(Target(self.account, container=self.container),
recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
self._verify_rebuilder_input()
def test_object_rebuilder_output(self):
self.checker.check(Target(self.account,
container=self.container,
obj=self.obj),
recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
self._verify_rebuilder_input()
def test_object_rebuilder_output_with_confirmations(self):
"""
Check that chunk targets showing errors are reported only after
the right number of confirmations.
"""
self.checker.required_confirmations = 2
tgt = Target(self.account,
container=self.container,
obj=self.obj,
content_id=self.meta['id'],
version=self.meta['version'])
self.checker.check(tgt, recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
# File is empty
self.assertRaises(StopIteration, self._verify_rebuilder_input)
self.assertIn(repr(tgt), self.checker.delayed_targets)
# 1st confirmation
for dtgt in self.checker.delayed_targets.values():
self.checker.check(dtgt, recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
# File is empty
self.assertRaises(StopIteration, self._verify_rebuilder_input)
self.assertIn(repr(tgt), self.checker.delayed_targets)
# 2nd confirmation
for dtgt in self.checker.delayed_targets.values():
self.checker.check(dtgt, recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
# File is NOT empty
self._verify_rebuilder_input()
self.assertNotIn(repr(tgt), self.checker.delayed_targets)
class ItemCheckCommand(lister.Lister):
"""
Various parameters that apply to all check commands.
"""
columns = ('Type', 'Item', 'Status', 'Errors')
success = True
checker = None
@property
def logger(self):
return self.app.client_manager.logger
def get_parser(self, prog_name):
parser = super(ItemCheckCommand, self).get_parser(prog_name)
parser.add_argument(
'--attempts',
type=int,
default=1,
help="Number of attempts for listing requests (default: 1).")
parser.add_argument('--checksum',
action='store_true',
help=("Perform checksum comparisons."))
parser.add_argument('--concurrency',
'--workers',
type=int,
default=30,
help="Number of concurrent checks (default: 30).")
parser.add_argument(
'-o',
'--output',
help=("Output file. Will contain elements in error. "
"Can later be passed to stdin of the legacy "
"oio-crawler-integrity to re-check only these elements."))
parser.add_argument(
'--output-for-chunk-rebuild',
help=("Write chunk errors in a file with a format "
"suitable as 'openio-admin chunk rebuild' input."))
return parser
def _take_action(self, parsed_args):
raise NotImplementedError()
def take_action(self, parsed_args):
self.logger.debug('take_action(%s)', parsed_args)
self.checker = Checker(
self.app.options.ns,
concurrency=parsed_args.concurrency,
error_file=parsed_args.output,
rebuild_file=parsed_args.output_for_chunk_rebuild,
request_attempts=parsed_args.attempts,
check_hash=parsed_args.checksum,
logger=self.logger)
return self.columns, self._take_action(parsed_args)
def _format_results(self):
for res in self.checker.run():
if not res.errors:
status = 'OK'
else:
self.success = False
status = 'error'
yield (res.target.type, repr(res.target), status,
res.errors_to_str())
def run(self, parsed_args):
super(ItemCheckCommand, self).run(parsed_args)
if not self.success:
return 1
class TestIntegrityCrawler(BaseTestCase):
def setUp(self):
super(TestIntegrityCrawler, self).setUp()
self.container = 'ct-' + random_str(8)
self.obj = 'obj-' + random_str(8)
self.account = 'test-integrity-' + random_str(8)
self.storage.object_create(self.account,
self.container,
obj_name=self.obj,
data="chunk")
_, self.rebuild_file = tempfile.mkstemp()
self.checker = Checker(self.ns, rebuild_file=self.rebuild_file)
self.meta, chunks = self.storage.object_locate(self.account,
self.container,
self.obj)
self.chunk = chunks[0]
self.irreparable = len(chunks) == 1
self.storage.blob_client.chunk_delete(self.chunk['real_url'])
def tearDown(self):
super(TestIntegrityCrawler, self).tearDown()
os.remove(self.rebuild_file)
self.storage.container_flush(self.account, self.container)
self.storage.container_delete(self.account, self.container)
self.wait_for_event('oio-preserved',
type_=EventTypes.CONTAINER_DELETED,
fields={'user': self.container})
self.storage.account_delete(self.account)
def _verify_rebuilder_input(self):
try:
line = fileinput.input(self.rebuild_file).next().strip()
cid = cid_from_name(self.account, self.container)
expected = '|'.join([cid, self.meta['id'], self.chunk['url']])
if self.irreparable:
expected = IRREPARABLE_PREFIX + '|' + expected
self.assertEqual(expected, line)
finally:
fileinput.close()
def test_account_rebuilder_output(self):
self.checker.check(Target(self.account), recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
self._verify_rebuilder_input()
def test_container_rebuilder_output(self):
self.checker.check(Target(self.account, container=self.container),
recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
self._verify_rebuilder_input()
def test_object_rebuilder_output(self):
self.checker.check(Target(self.account,
container=self.container,
obj=self.obj),
recurse=DEFAULT_DEPTH)
for _ in self.checker.run():
pass
self.checker.fd.flush()
self._verify_rebuilder_input()