def _make_composite_builder(self, builders): # helper to compose a ring, save it and sanity check it builder_files = self.save_builders(builders) cb = CompositeRingBuilder(builder_files) cb.compose().save(self.output_ring) self.check_composite_ring(self.output_ring, builders) return cb, builder_files
def test_load_errors(self): bad_file = os.path.join(self.tmpdir, 'bad_file.json') with self.assertRaises(IOError): CompositeRingBuilder.load(bad_file) def check_bad_content(content): with open(bad_file, 'wb') as fp: fp.write(content) try: with self.assertRaises(ValueError) as cm: CompositeRingBuilder.load(bad_file) self.assertIn( "File does not contain valid composite ring data", cm.exception.message) except AssertionError as err: raise AssertionError('With content %r: %s' % (content, err)) for content in ('', 'not json', json.dumps({}), json.dumps([])): check_bad_content(content) good_content = { 'components': [ {'version': 1, 'id': 'uuid_x', 'replicas': 12}, {'version': 2, 'id': 'uuid_y', 'replicas': 12} ], 'builder_files': {'uuid_x': '/path/to/file_x', 'uuid_y': '/path/to/file_y'}, 'version': 99} for missing in good_content: bad_content = dict(good_content) bad_content.pop(missing) check_bad_content(json.dumps(bad_content))
def test_save_errors(self): cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json') def do_test(cb): with self.assertRaises(ValueError) as cm: cb.save(cb_file) self.assertIn("No composed ring to save", cm.exception.message) do_test(CompositeRingBuilder()) do_test(CompositeRingBuilder([])) do_test(CompositeRingBuilder(['file1', 'file2']))
def check_bad_content(content): with open(bad_file, 'wb') as fp: fp.write(content) try: with self.assertRaises(ValueError) as cm: CompositeRingBuilder.load(bad_file) self.assertIn( "File does not contain valid composite ring data", cm.exception.message) except AssertionError as err: raise AssertionError('With content %r: %s' % (content, err))
def test_compose_duplicate_builder_ids(self): builders = self.create_sample_ringbuilders(3) builders[2]._id = builders[0]._id cb = CompositeRingBuilder(self.save_builders(builders)) with self.assertRaises(ValueError) as cm: cb.compose() error_lines = cm.exception.message.split('\n') self.assertIn("Builder id %r used at indexes 0, 2" % builders[0].id, error_lines[0]) self.assertFalse(error_lines[1:]) self.assertEqual(0, cb.version)
def test_compose_override_component_builders(self): # check passing different builder files to the compose() method # overrides loaded builder files cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json') builders = self.create_sample_ringbuilders(2) cb, builder_files = self._make_composite_builder(builders) # modify builders and save in different files self.add_dev_and_rebalance(builders[1]) with self.assertRaises(ValueError): cb.compose(builder_files) # sanity check - originals are unchanged other_files = self.save_builders(builders, prefix='other') cb.compose(other_files).save(self.output_ring) self.check_composite_ring(self.output_ring, builders) # check composite builder persists ok cb.save(cb_file) self.assertTrue(os.path.exists(cb_file)) self.check_composite_meta(cb_file, other_files, version=2) # and reloads ok cb = CompositeRingBuilder.load(cb_file) # and composes ok after reload cb.compose(force=True).save(self.output_ring) self.check_composite_ring(self.output_ring, builders) # check composite builder persists ok again cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2') cb.save(cb_file) self.assertTrue(os.path.exists(cb_file)) self.check_composite_meta(cb_file, other_files, version=3)
def test_compose_modified_component_builders(self): # check it's ok to compose again with same but modified builders cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json') builders = self.create_sample_ringbuilders(2) cb, builder_files = self._make_composite_builder(builders) ring = Ring(self.output_ring) orig_devs = [dev for dev in ring.devs if dev] self.assertEqual(10, len(orig_devs)) # sanity check self.add_dev_and_rebalance(builders[1]) builder_files = self.save_builders(builders) cb.compose().save(self.output_ring) self.check_composite_ring(self.output_ring, builders) ring = Ring(self.output_ring) modified_devs = [dev for dev in ring.devs if dev] self.assertEqual(len(orig_devs) + 1, len(modified_devs)) # check composite builder persists ok cb.save(cb_file) self.assertTrue(os.path.exists(cb_file)) self.check_composite_meta(cb_file, builder_files, version=2) # and reloads ok cb = CompositeRingBuilder.load(cb_file) # and composes ok after reload cb.compose(force=True).save(self.output_ring) self.check_composite_ring(self.output_ring, builders) # check composite builder persists ok again cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2') cb.save(cb_file) self.assertTrue(os.path.exists(cb_file)) self.check_composite_meta(cb_file, builder_files, version=3)
def test_compose_ok(self): cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json') builders = self.create_sample_ringbuilders(2) # make first version of composite ring cb, builder_files = self._make_composite_builder(builders) # check composite builder persists ok cb.save(cb_file) self.assertTrue(os.path.exists(cb_file)) self.check_composite_meta(cb_file, builder_files) # and reloads ok cb = CompositeRingBuilder.load(cb_file) self.assertEqual(1, cb.version) # composes after with no component builder changes will fail... with self.assertRaises(ValueError) as cm: cb.compose() self.assertIn('None of the component builders has been modified', cm.exception.message) self.assertEqual(1, cb.version) # ...unless we force it cb.compose(force=True).save(self.output_ring) self.check_composite_ring(self.output_ring, builders) self.assertEqual(2, cb.version) # check composite builder persists ok again cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json2') cb.save(cb_file) self.assertTrue(os.path.exists(cb_file)) self.check_composite_meta(cb_file, builder_files, version=2)
def test_compose_with_builder_files(self): cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json') builders = self.create_sample_ringbuilders(2) cb = CompositeRingBuilder(self.save_builders(builders)) cb.compose().save(self.output_ring) self.check_composite_ring(self.output_ring, builders) cb.save(cb_file) for i, b in enumerate(builders): self.add_dev_and_rebalance(b) self.save_builders(builders) cb = CompositeRingBuilder.load(cb_file) cb.compose().save(self.output_ring) self.check_composite_ring(self.output_ring, builders)
def do_test(builder_files): cb = CompositeRingBuilder(builder_files) with self.assertRaises(ValueError) as cm: cb.compose() self.assertIn('Two or more component builders are required', cm.exception.message) cb = CompositeRingBuilder() with self.assertRaises(ValueError) as cm: cb.compose(builder_files) self.assertIn('Two or more component builders are required', cm.exception.message)
def test_compose_missing_builder_id(self): def check_missing_id(cb, builders): # not ok to compose with builder_files that have no id assigned orig_version = cb.version no_id = random.randint(0, len(builders) - 1) # rewrite the builder files so that one has missing id self.save_builders(builders, missing_ids=[no_id]) with self.assertRaises(ValueError) as cm: cb.compose() error_lines = cm.exception.message.split('\n') self.assertIn("Problem with builder at index %s" % no_id, error_lines[0]) self.assertIn("id attribute has not been initialised", error_lines[0]) self.assertFalse(error_lines[1:]) self.assertEqual(orig_version, cb.version) # check with compose not previously called, cb has no existing metadata builders = self.create_sample_ringbuilders(3) builder_files = self.save_builders(builders) cb = CompositeRingBuilder(builder_files) check_missing_id(cb, builders) # now save good copies of builders and compose so this cb has # existing component metadata builder_files = self.save_builders(builders) cb = CompositeRingBuilder(builder_files) cb.compose() # cb now has component metadata check_missing_id(cb, builders)
def main(arguments=None): if arguments is not None: argv = arguments else: argv = sys.argv parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument( 'composite_builder_file', metavar='composite_builder_file', type=str, help='Name of composite builder file') subparsers = parser.add_subparsers( help='subcommand help', title='subcommands') # show show_parser = subparsers.add_parser( 'show', help='show composite ring builder metadata') show_parser.set_defaults(func=show) # compose compose_parser = subparsers.add_parser( 'compose', help='compose composite ring', usage='%(prog)s [-h] ' '[builder_file builder_file [builder_file ...] ' '--output ring_file [--force]') bf_help = ('Paths to component ring builder files to include in composite ' 'ring') compose_parser.add_argument('builder_files', metavar='builder_file', nargs='*', type=str, help=bf_help) compose_parser.add_argument('--output', metavar='output_file', type=str, required=True, help='Name of output ring file') compose_parser.add_argument( '--force', action='store_true', help='Force new composite ring file to be written') compose_parser.set_defaults(func=compose) _print_to_stderr(WARNING) args = parser.parse_args(argv[1:]) composite_builder = None if args.func != compose or os.path.exists(args.composite_builder_file): try: composite_builder = CompositeRingBuilder.load( args.composite_builder_file) except Exception as err: _print_err( 'An error occurred while loading the composite builder file.', err) exit(EXIT_ERROR) exit(args.func(composite_builder, args))
def test_abs_paths_persisted(self): cwd = os.getcwd() try: os.chdir(self.tmpdir) builders = self.create_sample_ringbuilders(2) builder_files = self.save_builders(builders) rel_builder_files = [os.path.basename(bf) for bf in builder_files] cb = CompositeRingBuilder(rel_builder_files) cb.compose().save(self.output_ring) self.check_composite_ring(self.output_ring, builders) cb_file = os.path.join(self.tmpdir, 'test-composite-ring.json') rel_cb_file = os.path.basename(cb_file) cb.save(rel_cb_file) self.check_composite_meta(rel_cb_file, rel_builder_files) finally: os.chdir(cwd)
def compose(composite_builder, args): composite_builder = composite_builder or CompositeRingBuilder() try: ring_data = composite_builder.compose( args.builder_files, force=args.force, require_modified=True) except Exception as err: _print_err( 'An error occurred while composing the ring.', err) return EXIT_ERROR try: ring_data.save(args.output) except Exception as err: _print_err( 'An error occurred while writing the composite ring file.', err) return EXIT_ERROR try: composite_builder.save(args.composite_builder_file) except Exception as err: _print_err( 'An error occurred while writing the composite builder file.', err) return EXIT_ERROR return EXIT_SUCCESS