def _make_composite_builder(self, builders):
# helper to compose a ring, save it and sanity check it
builder_files = self.save_builders(builders)
cb = CompositeRingBuilder(builder_files)
cb.compose().save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
return cb, builder_files
def test_load_errors(self):
bad_file = os.path.join(self.tmpdir, 'bad_file.json')
with self.assertRaises(IOError):
CompositeRingBuilder.load(bad_file)
def check_bad_content(content):
with open(bad_file, 'wb') as fp:
fp.write(content)
try:
with self.assertRaises(ValueError) as cm:
CompositeRingBuilder.load(bad_file)
self.assertIn(
"File does not contain valid composite ring data",
cm.exception.message)
except AssertionError as err:
raise AssertionError('With content %r: %s' % (content, err))
for content in ('', 'not json', json.dumps({}), json.dumps([])):
check_bad_content(content)
good_content = {
'components': [
{'version': 1, 'id': 'uuid_x', 'replicas': 12},
{'version': 2, 'id': 'uuid_y', 'replicas': 12}
],
'builder_files': {'uuid_x': '/path/to/file_x',
'uuid_y': '/path/to/file_y'},
'version': 99}
for missing in good_content:
bad_content = dict(good_content)
bad_content.pop(missing)
check_bad_content(json.dumps(bad_content))
def test_save_errors(self):
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
def do_test(cb):
with self.assertRaises(ValueError) as cm:
cb.save(cb_file)
self.assertIn("No composed ring to save", cm.exception.message)
do_test(CompositeRingBuilder())
do_test(CompositeRingBuilder([]))
do_test(CompositeRingBuilder(['file1', 'file2']))
def check_bad_content(content):
with open(bad_file, 'wb') as fp:
fp.write(content)
try:
with self.assertRaises(ValueError) as cm:
CompositeRingBuilder.load(bad_file)
self.assertIn(
"File does not contain valid composite ring data",
cm.exception.message)
except AssertionError as err:
raise AssertionError('With content %r: %s' % (content, err))
def test_compose_duplicate_builder_ids(self):
builders = self.create_sample_ringbuilders(3)
builders[2]._id = builders[0]._id
cb = CompositeRingBuilder(self.save_builders(builders))
with self.assertRaises(ValueError) as cm:
cb.compose()
error_lines = cm.exception.message.split('\n')
self.assertIn("Builder id %r used at indexes 0, 2" % builders[0].id,
error_lines[0])
self.assertFalse(error_lines[1:])
self.assertEqual(0, cb.version)
def test_compose_override_component_builders(self):
# check passing different builder files to the compose() method
# overrides loaded builder files
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
builders = self.create_sample_ringbuilders(2)
cb, builder_files = self._make_composite_builder(builders)
# modify builders and save in different files
self.add_dev_and_rebalance(builders[1])
with self.assertRaises(ValueError):
cb.compose(builder_files) # sanity check - originals are unchanged
other_files = self.save_builders(builders, prefix='other')
cb.compose(other_files).save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
# check composite builder persists ok
cb.save(cb_file)
self.assertTrue(os.path.exists(cb_file))
self.check_composite_meta(cb_file, other_files, version=2)
# and reloads ok
cb = CompositeRingBuilder.load(cb_file)
# and composes ok after reload
cb.compose(force=True).save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
# check composite builder persists ok again
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2')
cb.save(cb_file)
self.assertTrue(os.path.exists(cb_file))
self.check_composite_meta(cb_file, other_files, version=3)
def test_compose_modified_component_builders(self):
# check it's ok to compose again with same but modified builders
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
builders = self.create_sample_ringbuilders(2)
cb, builder_files = self._make_composite_builder(builders)
ring = Ring(self.output_ring)
orig_devs = [dev for dev in ring.devs if dev]
self.assertEqual(10, len(orig_devs)) # sanity check
self.add_dev_and_rebalance(builders[1])
builder_files = self.save_builders(builders)
cb.compose().save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
ring = Ring(self.output_ring)
modified_devs = [dev for dev in ring.devs if dev]
self.assertEqual(len(orig_devs) + 1, len(modified_devs))
# check composite builder persists ok
cb.save(cb_file)
self.assertTrue(os.path.exists(cb_file))
self.check_composite_meta(cb_file, builder_files, version=2)
# and reloads ok
cb = CompositeRingBuilder.load(cb_file)
# and composes ok after reload
cb.compose(force=True).save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
# check composite builder persists ok again
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2')
cb.save(cb_file)
self.assertTrue(os.path.exists(cb_file))
self.check_composite_meta(cb_file, builder_files, version=3)
def test_compose_ok(self):
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
builders = self.create_sample_ringbuilders(2)
# make first version of composite ring
cb, builder_files = self._make_composite_builder(builders)
# check composite builder persists ok
cb.save(cb_file)
self.assertTrue(os.path.exists(cb_file))
self.check_composite_meta(cb_file, builder_files)
# and reloads ok
cb = CompositeRingBuilder.load(cb_file)
self.assertEqual(1, cb.version)
# composes after with no component builder changes will fail...
with self.assertRaises(ValueError) as cm:
cb.compose()
self.assertIn('None of the component builders has been modified',
cm.exception.message)
self.assertEqual(1, cb.version)
# ...unless we force it
cb.compose(force=True).save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
self.assertEqual(2, cb.version)
# check composite builder persists ok again
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2')
cb.save(cb_file)
self.assertTrue(os.path.exists(cb_file))
self.check_composite_meta(cb_file, builder_files, version=2)
def test_compose_with_builder_files(self):
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
builders = self.create_sample_ringbuilders(2)
cb = CompositeRingBuilder(self.save_builders(builders))
cb.compose().save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
cb.save(cb_file)
for i, b in enumerate(builders):
self.add_dev_and_rebalance(b)
self.save_builders(builders)
cb = CompositeRingBuilder.load(cb_file)
cb.compose().save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
def do_test(builder_files):
cb = CompositeRingBuilder(builder_files)
with self.assertRaises(ValueError) as cm:
cb.compose()
self.assertIn('Two or more component builders are required',
cm.exception.message)
cb = CompositeRingBuilder()
with self.assertRaises(ValueError) as cm:
cb.compose(builder_files)
self.assertIn('Two or more component builders are required',
cm.exception.message)
def test_compose_missing_builder_id(self):
def check_missing_id(cb, builders):
# not ok to compose with builder_files that have no id assigned
orig_version = cb.version
no_id = random.randint(0, len(builders) - 1)
# rewrite the builder files so that one has missing id
self.save_builders(builders, missing_ids=[no_id])
with self.assertRaises(ValueError) as cm:
cb.compose()
error_lines = cm.exception.message.split('\n')
self.assertIn("Problem with builder at index %s" % no_id,
error_lines[0])
self.assertIn("id attribute has not been initialised",
error_lines[0])
self.assertFalse(error_lines[1:])
self.assertEqual(orig_version, cb.version)
# check with compose not previously called, cb has no existing metadata
builders = self.create_sample_ringbuilders(3)
builder_files = self.save_builders(builders)
cb = CompositeRingBuilder(builder_files)
check_missing_id(cb, builders)
# now save good copies of builders and compose so this cb has
# existing component metadata
builder_files = self.save_builders(builders)
cb = CompositeRingBuilder(builder_files)
cb.compose() # cb now has component metadata
check_missing_id(cb, builders)
def main(arguments=None):
if arguments is not None:
argv = arguments
else:
argv = sys.argv
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
'composite_builder_file',
metavar='composite_builder_file', type=str,
help='Name of composite builder file')
subparsers = parser.add_subparsers(
help='subcommand help', title='subcommands')
# show
show_parser = subparsers.add_parser(
'show', help='show composite ring builder metadata')
show_parser.set_defaults(func=show)
# compose
compose_parser = subparsers.add_parser(
'compose', help='compose composite ring',
usage='%(prog)s [-h] '
'[builder_file builder_file [builder_file ...] '
'--output ring_file [--force]')
bf_help = ('Paths to component ring builder files to include in composite '
'ring')
compose_parser.add_argument('builder_files', metavar='builder_file',
nargs='*', type=str, help=bf_help)
compose_parser.add_argument('--output', metavar='output_file', type=str,
required=True, help='Name of output ring file')
compose_parser.add_argument(
'--force', action='store_true',
help='Force new composite ring file to be written')
compose_parser.set_defaults(func=compose)
_print_to_stderr(WARNING)
args = parser.parse_args(argv[1:])
composite_builder = None
if args.func != compose or os.path.exists(args.composite_builder_file):
try:
composite_builder = CompositeRingBuilder.load(
args.composite_builder_file)
except Exception as err:
_print_err(
'An error occurred while loading the composite builder file.',
err)
exit(EXIT_ERROR)
exit(args.func(composite_builder, args))
def test_abs_paths_persisted(self):
cwd = os.getcwd()
try:
os.chdir(self.tmpdir)
builders = self.create_sample_ringbuilders(2)
builder_files = self.save_builders(builders)
rel_builder_files = [os.path.basename(bf) for bf in builder_files]
cb = CompositeRingBuilder(rel_builder_files)
cb.compose().save(self.output_ring)
self.check_composite_ring(self.output_ring, builders)
cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json')
rel_cb_file = os.path.basename(cb_file)
cb.save(rel_cb_file)
self.check_composite_meta(rel_cb_file, rel_builder_files)
finally:
os.chdir(cwd)
def compose(composite_builder, args):
composite_builder = composite_builder or CompositeRingBuilder()
try:
ring_data = composite_builder.compose(
args.builder_files, force=args.force, require_modified=True)
except Exception as err:
_print_err(
'An error occurred while composing the ring.', err)
return EXIT_ERROR
try:
ring_data.save(args.output)
except Exception as err:
_print_err(
'An error occurred while writing the composite ring file.', err)
return EXIT_ERROR
try:
composite_builder.save(args.composite_builder_file)
except Exception as err:
_print_err(
'An error occurred while writing the composite builder file.', err)
return EXIT_ERROR
return EXIT_SUCCESS